From bbae3e01771501ae8ec0dcbca5ef30a4f0fb5ad8 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 08:52:11 +0200 Subject: [PATCH 01/13] feat(greenfield): replace generic exec with service actions --- .../application-architecture.md | 34 +- .../greenfield-rewrite/data-and-security.md | 24 +- .../greenfield-rewrite/implementation-plan.md | 8 + .../greenfield-rewrite/progress.md | 55 +- .../runtime-and-delivery.md | 17 + greenfield/docs/generated/procedures.md | 2 + ...Security.createPrincipal.input.schema.json | 4 +- ...ecurity.createPrincipal.output.schema.json | 8 +- ...curity.disablePrincipal.output.schema.json | 8 +- ...Security.listPrincipals.output.schema.json | 8 +- ...rity.replaceCapabilities.input.schema.json | 4 +- ...ity.replaceCapabilities.output.schema.json | 8 +- ...ecurityAudit.listEvents.output.schema.json | 8 +- ...serviceActions.getStatus.input.schema.json | 8 + ...erviceActions.getStatus.output.schema.json | 462 +++++++++++++ .../serviceActions.request.input.schema.json | 98 +++ .../serviceActions.request.output.schema.json | 32 + .../migration.sql | 29 +- .../snapshot.json | 18 +- .../fixtures/2026.7.2-beta.7/manifest.json | 46 ++ .../fixtures/2026.7.2-beta.7/operations.json | 169 +++++ .../audits/openclaw/reviewedFixtures.ts | 7 + .../scripts/audits/openclaw/sourceAudit.ts | 653 ++++++++++++++++++ .../audits/openclaw/sourceAuditSchemas.ts | 205 +++++- .../scripts/documentation/artifacts.test.ts | 14 + .../scripts/documentation/jsonSchema.test.ts | 23 +- .../scripts/documentation/jsonSchema.ts | 5 + .../sourceBoundaries/sourceTopologyPolicy.ts | 1 + greenfield/src/app/dashboardServer.test.ts | 1 + greenfield/src/app/dashboardServer.ts | 69 ++ greenfield/src/app/developmentWorker.ts | 13 +- greenfield/src/app/server.ts | 3 + greenfield/src/app/trpcHttpHandler.ts | 3 + greenfield/src/app/trpcRequestPolicy.test.ts | 4 + greenfield/src/app/worker.test.ts | 25 +- greenfield/src/app/worker.ts | 31 +- greenfield/src/browser/api/trpcClient.ts | 4 + .../browser/overview/OverviewRoute.test.tsx | 27 +- .../src/browser/overview/OverviewRoute.tsx | 4 + .../overview/OverviewServiceActionsCard.tsx | 258 +++++++ .../OverviewServiceActionsSection.test.tsx | 463 +++++++++++++ .../OverviewServiceActionsSection.tsx | 177 +++++ .../overview/serviceActionsOperations.test.ts | 107 +++ .../overview/serviceActionsOperations.ts | 204 ++++++ greenfield/src/contracts/contractRegistry.ts | 2 + greenfield/src/contracts/security.test.ts | 2 + greenfield/src/contracts/security.ts | 2 + .../src/contracts/serviceActions.test.ts | 195 ++++++ greenfield/src/contracts/serviceActions.ts | 185 +++++ .../database/migrations/jobsSchema.test.ts | 19 + .../migrations/migrationGraph.test.ts | 1 + .../schema/automationPrincipalCapabilities.ts | 2 +- .../server/database/schema/workerInstances.ts | 7 + .../database/validation/rowSchemas.test.ts | 16 + .../database/validation/workerActionKeys.ts | 61 ++ .../database/validation/workerInstances.ts | 20 +- .../server/database/workerActionKeyPolicy.ts | 4 + .../server/domains/cache/repository.test.ts | 1 + .../domains/jobs/actionExecutors.test.ts | 174 +++++ .../server/domains/jobs/actionExecutors.ts | 147 ++++ .../domains/jobs/actionRegistry.test.ts | 44 ++ .../src/server/domains/jobs/actionRegistry.ts | 124 ++++ .../server/domains/jobs/coordinator.test.ts | 37 + .../src/server/domains/jobs/coordinator.ts | 14 + .../server/domains/jobs/repository.test.ts | 172 ++++- .../src/server/domains/jobs/repository.ts | 68 +- .../src/server/domains/jobs/service.test.ts | 2 + .../domains/jobs/serviceActionQueue.test.ts | 315 +++++++++ .../server/domains/jobs/serviceActionQueue.ts | 313 +++++++++ .../server/domains/jobs/workerRuntime.test.ts | 61 ++ .../src/server/domains/jobs/workerRuntime.ts | 53 +- .../serviceActions/operationAudit.test.ts | 72 ++ .../domains/serviceActions/operationAudit.ts | 86 +++ .../domains/serviceActions/procedures.test.ts | 288 ++++++++ .../domains/serviceActions/procedures.ts | 10 + .../server/domains/serviceActions/routes.ts | 125 ++++ .../domains/serviceActions/service.test.ts | 287 ++++++++ .../server/domains/serviceActions/service.ts | 225 ++++++ .../serviceActions/statusReader.test.ts | 143 ++++ .../domains/serviceActions/statusReader.ts | 92 +++ ...ewayOpenClawServiceActionsProvider.test.ts | 202 ++++++ ...ntGatewayOpenClawServiceActionsProvider.ts | 159 +++++ .../gateway/persistentGatewayProtocol.test.ts | 202 ++++++ .../gateway/persistentGatewayProtocol.ts | 239 +++++++ .../persistentGatewayTransport.test.ts | 342 +++++++++ .../gateway/persistentGatewayTransport.ts | 69 +- .../observability/structuredLogger.test.ts | 37 + .../observability/structuredLogger.ts | 25 + .../src/server/test/support/requestContext.ts | 22 + greenfield/src/server/trpc/appRouter.ts | 6 + greenfield/src/server/trpc/context.test.ts | 6 + greenfield/src/server/trpc/context.ts | 4 + .../src/server/trpc/procedureErrorPolicy.ts | 7 + .../src/shared/databaseMigrationManifest.ts | 4 +- greenfield/src/shared/hostOperations.ts | 24 + .../src/shared/openClawServiceActions.ts | 45 ++ .../integration/openclaw/sourceAudit.test.ts | 526 +++++++++++++- .../parity/fixtures/greenfield-contracts.json | 8 + .../parity/fixtures/legacy-endpoints.json | 39 +- .../src/test/parity/parityInventory.test.ts | 63 +- 100 files changed, 8663 insertions(+), 86 deletions(-) create mode 100644 greenfield/docs/generated/schemas/serviceActions.getStatus.input.schema.json create mode 100644 greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json create mode 100644 greenfield/docs/generated/schemas/serviceActions.request.input.schema.json create mode 100644 greenfield/docs/generated/schemas/serviceActions.request.output.schema.json create mode 100644 greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json create mode 100644 greenfield/src/browser/overview/OverviewServiceActionsCard.tsx create mode 100644 greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx create mode 100644 greenfield/src/browser/overview/OverviewServiceActionsSection.tsx create mode 100644 greenfield/src/browser/overview/serviceActionsOperations.test.ts create mode 100644 greenfield/src/browser/overview/serviceActionsOperations.ts create mode 100644 greenfield/src/contracts/serviceActions.test.ts create mode 100644 greenfield/src/contracts/serviceActions.ts create mode 100644 greenfield/src/server/database/validation/workerActionKeys.ts create mode 100644 greenfield/src/server/database/workerActionKeyPolicy.ts create mode 100644 greenfield/src/server/domains/jobs/serviceActionQueue.test.ts create mode 100644 greenfield/src/server/domains/jobs/serviceActionQueue.ts create mode 100644 greenfield/src/server/domains/serviceActions/operationAudit.test.ts create mode 100644 greenfield/src/server/domains/serviceActions/operationAudit.ts create mode 100644 greenfield/src/server/domains/serviceActions/procedures.test.ts create mode 100644 greenfield/src/server/domains/serviceActions/procedures.ts create mode 100644 greenfield/src/server/domains/serviceActions/routes.ts create mode 100644 greenfield/src/server/domains/serviceActions/service.test.ts create mode 100644 greenfield/src/server/domains/serviceActions/service.ts create mode 100644 greenfield/src/server/domains/serviceActions/statusReader.test.ts create mode 100644 greenfield/src/server/domains/serviceActions/statusReader.ts create mode 100644 greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts create mode 100644 greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts create mode 100644 greenfield/src/shared/hostOperations.ts create mode 100644 greenfield/src/shared/openClawServiceActions.ts diff --git a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md index e46341684..a2e1f4b6e 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md +++ b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md @@ -213,7 +213,7 @@ Every application operation controlled by this repository becomes a tRPC procedu - settings, authentication, MFA, WebAuthn, and session administration; - Docker inventory, updater policy, and actions; - database, cache, quota, backup, and log-rotation operations; -- Moltbook, files, logs, terminal helpers, and exec jobs; and +- Moltbook, files, logs, terminal sessions, and purpose-built Service Actions; and - TypeScript automation calls from OpenClaw scripts. The browser uses `@trpc/tanstack-react-query`, a singleton `QueryClient`, and a singleton @@ -464,6 +464,38 @@ is exclusive, caller-idempotent, single-attempt, non-retry-safe, and non-cancell worker owns its fixed no-shell lifecycle command. Ambiguous enqueue or terminal settlement is reconciled by durable run identity and never blindly dispatches a second restart. +### Purpose-built Service Actions replace generic exec + +The Overview exposes exactly four fixed Service Actions through +`serviceActions.getStatus` and `serviceActions.request`: OpenClaw session cleanup, OpenClaw +installation update, host restart, and host update. The browser submits only a fixed action ID and +a caller-owned idempotency key. The web process commits a sanitized attempt audit, checks a fresh +exact-release worker advertisement, and revalidates the browser session plus recent MFA at the +durable enqueue handoff. It returns a job-run ID rather than waiting for a privileged effect and +links all progress and terminal state to the existing Jobs surface. + +OpenClaw cleanup and update are implemented worker-only through the hash-pinned +`sessions.cleanup` and `update.run` Gateway methods. Their providers accept no browser parameters, +persist only bounded schema-validated summaries, never return raw Gateway results, and never +blindly replay a post-dispatch unknown outcome. Cleanup deliberately uses OpenClaw's source-owned +session/artifact maintenance instead of reproducing legacy recursive deletion. The legacy broad +`system_cleanup` behavior is not restored: package, journal, and Docker deletion cross separate +ownership domains, and Docker cleanup remains part of the Docker domain. + +The contract and Overview retain fixed rows for host restart and host update, but production marks +both unavailable. The current web and worker processes share one Unix identity, so a group- or +shared-user polkit rule would also give a compromised web process the worker's root authority. No +such broker, polkit rule, root helper, or host-operation unit ships in this slice. Enabling either +host action later requires a distinct worker OS identity, a root-owned immutable worker boundary, +and separately reviewed provisioning and rollback before the worker may advertise the action key. + +The interactive PTY remains the sole terminal boundary. Shell `cd` and completion are owned by the +connected shell/readline protocol, termination uses the bounded terminal session control, and no +new generic command, cwd, or completion API is introduced. The unused synchronous `POST /api/exec` +endpoint is a reviewed removal because no current browser or scoped automation consumer depends on +it; legacy long-running exec consumers map to either the PTY or the fixed durable Service Actions +queue. + ### Current-protocol Control UI projections The 2026-08-06 OpenClaw audit separates protocol authority from Control UI projection through 23 diff --git a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md index fde7dcc80..fb0af76fa 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md +++ b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md @@ -281,6 +281,24 @@ deploys, builds, Git mutations, Docker mutations, backups, restores, systemd cha restarts, or unbounded shell commands. Those operations become durable `job_runs` consumed by the worker. +Service Actions are a separate fixed-intent boundary, not a generic exec facade. The contract +contains exactly `openclaw-cleanup`, `openclaw-update`, `system-restart`, and `system-update`; a +caller can supply only one of those IDs plus an actor-bound idempotency key. Reads and requests are +session-only under dedicated capabilities, requests require recent MFA, and audit attempt must +commit before the durable enqueue handoff. That handoff rechecks exact-release worker +availability, the current browser session, and recent MFA. Enqueue uncertainty is reconciled by +the same principal/idempotency intent, and post-dispatch uncertainty never authorizes a replay. + +The production worker advertises only actions for which its composition owns an exact executor. +OpenClaw cleanup and update are worker-only, fixed-parameter Gateway operations with bounded, +sanitized results. Host restart and host update remain canonical contract/UI rows but are +unavailable in production because the web and worker currently share one Unix identity. A shared +group or polkit grant would therefore collapse the web/worker trust boundary. This rewrite ships +no shared-user host broker, polkit rule, root helper, or host-operation systemd unit. Future host +enablement requires a distinct worker OS identity, root-owned immutable worker execution, exact +subject and operation policy, and reviewed install/rollback evidence before either action key can +be advertised. + The `cache:read` automation heartbeat is a separate sanitized projection, not a shortcut around session, task, job, or cron detail authorization. It reads process-local validated Gateway summaries plus bounded payload-free cache status and purpose-built SQLite task/Dashboard-job @@ -367,7 +385,7 @@ Retain the current security behavior while simplifying its structure: - durable browser sessions use random opaque validators, store only their hashes, and enforce idle and absolute expiry; - recent high-assurance verification is required for secrets, credentials, deploy, rollback, - restore, exec, Docker mutation, and security administration; + restore, Service Actions, Docker mutation, and security administration; - the process Effect runtime bounds Gateway, password/Argon2, TOTP AES/HMAC, and WebAuthn parsing/signature work with separate concurrency and queue limits; rolling in-memory budgets stop parallel requests before expensive work can outrun durable cooldowns, and a failed authentication attempt retains its active-work @@ -589,8 +607,8 @@ proxy mode names exact proxies and requires them to overwrite forwarded identity units. - Markdown and HTML are sanitized at the rendering boundary. A raw HTML feature is not an authorization boundary. -- Exec, terminal, Git, Docker, systemd, backup, restore, and OpenClaw adapters each have a - command/operation allowlist and a structured audit record. +- Terminal, Service Actions, Git, Docker, systemd, backup, restore, and OpenClaw adapters each have + a command/operation allowlist and a structured audit record. No generic exec adapter is retained. - Logs and audit details pass a central redactor before persistence and again before browser output. - CSP, frame denial, MIME-sniff prevention, referrer policy, permissions policy, and request ID diff --git a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md index 18914f622..3b033d126 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md +++ b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md @@ -101,6 +101,14 @@ including restart during streaming. - treat the configured Terminal workspace root as an initial working-directory catalog only. A real interactive shell can change directory and access anything permitted by its OS identity; filesystem isolation requires a separate mount, namespace, or container sandbox. +- keep shell `cd`, completion, and termination inside the implemented bounded PTY. Replace consumed + legacy exec behavior only with purpose-built durable Service Actions; do not restore a generic + command, shell, or cwd API for the unused synchronous exec route. +- expose the four fixed Service Action intents in contract/UI, but advertise only exact executors + owned by a fresh worker on the current release. OpenClaw cleanup/update use reviewed worker-only + Gateway methods. Host restart/update remain unavailable until web and worker have distinct OS + identities and a root-owned immutable worker boundary with reviewed provisioning and rollback; + a shared-user/group polkit grant is forbidden. **Exit gate:** capability, step-up, audit, cancellation, resource-limit, and failure-recovery tests pass for every privileged operation. diff --git a/greenfield/docs/architecture/greenfield-rewrite/progress.md b/greenfield/docs/architecture/greenfield-rewrite/progress.md index 7e8db1f94..dce0ebe09 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/progress.md +++ b/greenfield/docs/architecture/greenfield-rewrite/progress.md @@ -7,15 +7,15 @@ This matrix is the living phase status. Update it in the same change that materially advances or closes a phase; dated entries below provide the evidence, not a second status source. -| Phase | Status | Current evidence and remaining gate | -| ----------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | -| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | -| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | -| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | -| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | -| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Docker control, database, GitHub, deployment, database backup/restore, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | -| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | +| Phase | Status | Current evidence and remaining gate | +| ----------------------------------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | +| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | +| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | +| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | +| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | +| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Overview Service Actions replace the consumed legacy exec flows with four fixed intents; OpenClaw cleanup/update are worker-owned, while host restart/update remain explicitly unavailable pending a distinct worker OS identity and reviewed root boundary. Docker control, database, GitHub, deployment, database backup/restore, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | +| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | ### 2026-08-03 — Phase 0 started @@ -1548,7 +1548,38 @@ full-browser parity, production rehearsal, cutover, and legacy deletion remain o content stays download-only. No media inventory, host path, path-query API, or additional browser route is introduced. - The secure narrowing implements the legacy `GET /api/media` behavior through the existing Chat - raw route. The living endpoint inventory is now **108 implemented, 47 planned, and two reviewed - removals** out of 157; browser routes remain **12 implemented and four planned**. Phase 4 live - Gateway smoke/restart evidence and the remaining Phase 5 domains still keep their aggregate exit + raw route. At this checkpoint the inventory was **108 implemented, 47 planned, and two reviewed + removals** out of 157; browser routes remained **12 implemented and four planned**. Phase 4 live + Gateway smoke/restart evidence and the remaining Phase 5 domains still kept their aggregate exit gates open. + +### 2026-08-12 — Purpose-built Service Actions close consumed exec parity + +- The Overview now exposes exactly four fixed Service Actions through session-only + `serviceActions.getStatus` and recent-MFA `serviceActions.request`: OpenClaw cleanup, OpenClaw + update, host restart, and host update. Requests carry a caller-owned idempotency key, commit a + fail-closed attempted audit record, recheck fresh exact-release worker availability, and + revalidate session and recent MFA at durable enqueue. The browser receives only a durable run ID + and follows progress through `/jobs`; no stdout, command, environment, or provider response + crosses the contract. +- OpenClaw cleanup and update are implemented as exact worker-only, hash-pinned + `sessions.cleanup` and `update.run` calls. Cleanup uses OpenClaw's own bounded maintenance policy + instead of reintroducing legacy recursive deletion, and update preserves the managed handoff. + Both actions are single-attempt, non-retry-safe, non-cancellable, resource-locked jobs with + sanitized results and explicit unknown-outcome handling. +- Host restart and host update remain fixed contract/UI rows but production reports both + `unavailable`. Web and worker currently share one Unix identity, so a shared-user or group-based + polkit broker would collapse the intended privilege boundary. No unsafe host broker, root helper, + polkit rule, or operation unit ships. Future enablement requires a distinct worker OS identity, + root-owned immutable worker code/configuration, exact-principal authorization, and reviewed + provisioning plus rollback before those worker action keys may be advertised. +- The interactive PTY already owns shell `cd`, completion, and bounded termination. Legacy + long-running exec consumers map to either that PTY or the purpose-built durable Service Actions + queue. The unused synchronous `POST /api/exec` route is a reviewed removal with no current + browser or scoped automation consumer; no generic shell/command replacement was added. The broad + legacy `system_cleanup` behavior is not restored because it mixed package, journal, and Docker + deletion across separately owned domains. +- The living inventory is now **113 implemented, 41 planned, and three reviewed removals** out of + 157 legacy endpoints. Browser routes remain **12 implemented and four planned**. This advances + Phase 5 without claiming host-operation enablement, Docker/database/delivery parity, or the + aggregate Phase 5 exit gate. diff --git a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md index 7bdb39857..177e463f8 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md +++ b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md @@ -423,6 +423,23 @@ target-directory write access for its private stage file, `renameat2` exchange, at runtime. Descriptor validation, per-file bounds, CAS, and the fixed worker manifest are the write boundary. +The fixed Service Actions contract and Overview include host restart and host update, but the +production release does not install or compose authority for them. Web and worker still run as the +same Unix user, so granting that identity a root helper, polkit action, or root-owned operation unit +would also grant the web process the same authority. Consequently both host actions remain +`unavailable`, the worker does not advertise their action keys, and enqueue rechecks fail closed. +No host-operation helper, polkit rule, or operation unit is included in release staging. + +Future host-action enablement is a delivery/topology change rather than an application toggle. It +must introduce a distinct worker OS identity, keep the web principal outside that identity and its +groups, execute only root-owned immutable worker code/configuration, constrain authorization to +the exact worker principal and fixed operation, and ship manifest-verified provisioning plus +explicit rollback. Only after that boundary has executable installation, identity, availability, +and rollback evidence may production compose a host broker and advertise either host action. +OpenClaw cleanup and update do not use this deferred host authority: their exact worker-only +Gateway operations are already implemented and remain available only when a fresh exact-release +worker advertises them. + The web process also derives the fixed `/media` descriptor boundary from that same reviewed root. It exposes no configurable media directory, recursive listing, or browser-supplied path route. Local-history transcript carriers become opaque session/message-bound diff --git a/greenfield/docs/generated/procedures.md b/greenfield/docs/generated/procedures.md index 2e3672caf..f840e972a 100644 --- a/greenfield/docs/generated/procedures.md +++ b/greenfield/docs/generated/procedures.md @@ -122,6 +122,8 @@ | `schedules.run` | mutation | schedules | Authenticated: jobs:write | [input](./schemas/schedules.run.input.schema.json) | [output](./schemas/schedules.run.output.schema.json) | `CONFLICT`, `FORBIDDEN`, `NOT_FOUND`, `SERVICE_UNAVAILABLE`, `UNAUTHORIZED` | None | Enqueues one caller-scoped idempotent manual schedule run. | | `schedules.update` | mutation | schedules | Authenticated browser session: jobs:write | [input](./schemas/schedules.update.input.schema.json) | [output](./schemas/schedules.update.output.schema.json) | `BAD_REQUEST`, `CONFLICT`, `FORBIDDEN`, `NOT_FOUND`, `SERVICE_UNAVAILABLE`, `UNAUTHORIZED` | None | Updates one schedule or its explicit disable intent by version. | | `securityAudit.listEvents` | query | securityAudit | Authenticated browser session | [input](./schemas/securityAudit.listEvents.input.schema.json) | [output](./schemas/securityAudit.listEvents.output.schema.json) | `FORBIDDEN`, `UNAUTHORIZED` | None | Lists redacted immutable security events in stable newest-first order. | +| `serviceActions.getStatus` | query | service-actions | Authenticated browser session: service-actions:read | [input](./schemas/serviceActions.getStatus.input.schema.json) | [output](./schemas/serviceActions.getStatus.output.schema.json) | `FORBIDDEN`, `SERVICE_UNAVAILABLE`, `UNAUTHORIZED` | None | Returns bounded availability and durable-run observations for fixed privileged service actions. | +| `serviceActions.request` | mutation | service-actions | Authenticated browser session: service-actions:write; MFA enrollment required; recent MFA when enabled | [input](./schemas/serviceActions.request.input.schema.json) | [output](./schemas/serviceActions.request.output.schema.json) | `CONFLICT`, `FORBIDDEN`, `SERVICE_UNAVAILABLE`, `UNAUTHORIZED` | `mfa_enrollment_required`, `operation_outcome_unknown`, `step_up_required` | Queues one exact worker-owned service action after recent-MFA authorization and durable audit admission. | | `system.healthDiagnostics` | query | system | Authenticated browser session | [input](./schemas/system.healthDiagnostics.input.schema.json) | [output](./schemas/system.healthDiagnostics.output.schema.json) | `FORBIDDEN`, `UNAUTHORIZED` | None | Returns bounded readiness, dependency, and queue diagnostics without operational identities. | | `system.metrics` | query | system | Authenticated browser session | [input](./schemas/system.metrics.input.schema.json) | [output](./schemas/system.metrics.output.schema.json) | `FORBIDDEN`, `SERVICE_UNAVAILABLE`, `UNAUTHORIZED` | None | Returns bounded host gauges and throughput without host identity or control authority. | | `system.runtimeIdentity` | query | system | Public | [input](./schemas/system.runtimeIdentity.input.schema.json) | [output](./schemas/system.runtimeIdentity.output.schema.json) | None | None | Returns the Bun runtime identity of the serving process. | diff --git a/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.input.schema.json b/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.input.schema.json index a64cc856a..c9501ca31 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.input.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.input.schema.json @@ -29,6 +29,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -36,7 +38,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "id": { diff --git a/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.output.schema.json b/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.output.schema.json index 41acc68b8..e1324de12 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.output.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.createPrincipal.output.schema.json @@ -98,6 +98,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -105,7 +107,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { @@ -192,6 +194,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -199,7 +203,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { diff --git a/greenfield/docs/generated/schemas/automationSecurity.disablePrincipal.output.schema.json b/greenfield/docs/generated/schemas/automationSecurity.disablePrincipal.output.schema.json index 54777ada2..054771927 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.disablePrincipal.output.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.disablePrincipal.output.schema.json @@ -47,6 +47,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -54,7 +56,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { @@ -141,6 +143,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -148,7 +152,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { diff --git a/greenfield/docs/generated/schemas/automationSecurity.listPrincipals.output.schema.json b/greenfield/docs/generated/schemas/automationSecurity.listPrincipals.output.schema.json index 48caad2d4..a029c75fe 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.listPrincipals.output.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.listPrincipals.output.schema.json @@ -72,6 +72,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -79,7 +81,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { @@ -166,6 +168,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -173,7 +177,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { diff --git a/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.input.schema.json b/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.input.schema.json index bdd390541..cb91fda8a 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.input.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.input.schema.json @@ -40,6 +40,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -47,7 +49,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true } }, diff --git a/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.output.schema.json b/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.output.schema.json index e7328eb06..f9a6dbd53 100644 --- a/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.output.schema.json +++ b/greenfield/docs/generated/schemas/automationSecurity.replaceCapabilities.output.schema.json @@ -47,6 +47,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -54,7 +56,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { @@ -141,6 +143,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -148,7 +152,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "createdAtMs": { diff --git a/greenfield/docs/generated/schemas/securityAudit.listEvents.output.schema.json b/greenfield/docs/generated/schemas/securityAudit.listEvents.output.schema.json index 37746d900..ec5796218 100644 --- a/greenfield/docs/generated/schemas/securityAudit.listEvents.output.schema.json +++ b/greenfield/docs/generated/schemas/securityAudit.listEvents.output.schema.json @@ -156,6 +156,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -163,7 +165,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "method": { @@ -229,6 +231,8 @@ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", @@ -236,7 +240,7 @@ ], "type": "string" }, - "maxItems": 27, + "maxItems": 29, "uniqueItems": true }, "replacementCredentialId": { diff --git a/greenfield/docs/generated/schemas/serviceActions.getStatus.input.schema.json b/greenfield/docs/generated/schemas/serviceActions.getStatus.input.schema.json new file mode 100644 index 000000000..c0fa84ff1 --- /dev/null +++ b/greenfield/docs/generated/schemas/serviceActions.getStatus.input.schema.json @@ -0,0 +1,8 @@ +{ + "$id": "urn:mira-dashboard:serviceActions.getStatus.input", + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} diff --git a/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json b/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json new file mode 100644 index 000000000..b5aa0e085 --- /dev/null +++ b/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json @@ -0,0 +1,462 @@ +{ + "$id": "urn:mira-dashboard:serviceActions.getStatus.output", + "type": "object", + "properties": { + "actions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "activeRun": { + "type": "object", + "properties": { + "actionKey": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "attemptCount": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, + "attemptLimit": { + "type": "integer", + "minimum": 1, + "maximum": 10 + }, + "availableAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "cancellationPolicy": { + "enum": [ + "cooperative", + "never", + "queued-only" + ], + "type": "string" + }, + "cancelRequestedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "displayName": { + "type": "string", + "maxLength": 160, + "minLength": 1, + "pattern": "\\S", + "allOf": [ + { + "pattern": "^[^\\u0000]*$" + }, + { + "pattern": "^(?![\\s\\S]*(?:[\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890-\\u0891\\u08E2\\u180E\\u200B-\\u200F\\u2028-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFEFF\\uFFF9-\\uFFFB]|\\uD804[\\uDCBD\\uDCCD]|\\uD80D[\\uDC30-\\uDC3F]|\\uD82F[\\uDCA0-\\uDCA3]|\\uD834[\\uDD73-\\uDD7A]|\\uDB40(?:\\uDC01|[\\uDC20-\\uDC7F])))[\\s\\S]*$" + } + ] + }, + "eventCount": { + "type": "integer", + "minimum": 0, + "maximum": 1000 + }, + "finishedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "firstStartedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "id": { + "type": "string", + "minLength": 36, + "maxLength": 36, + "format": "uuid", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "lastAttemptStartedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "priority": { + "type": "integer", + "minimum": -100, + "maximum": 100 + }, + "queuedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "resourceClass": { + "enum": [ + "exclusive", + "host-heavy", + "interactive", + "light", + "network" + ], + "type": "string" + }, + "resourceKeys": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "maxItems": 32, + "$comment": "Live Valibot validation additionally requires resource keys to be unique, strictly sorted, and within their aggregate UTF-8 byte budget." + }, + "retrySafe": { + "type": "boolean" + }, + "scheduledForAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "scheduledJobId": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "scheduledJobVersion": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "state": { + "enum": [ + "cancelled", + "failed", + "queued", + "running", + "succeeded", + "timed-out" + ], + "type": "string" + }, + "stateVersion": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "terminalCode": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._/-]*$" + }, + "terminalMessage": { + "type": "string", + "maxLength": 2000, + "minLength": 1, + "pattern": "\\S", + "allOf": [ + { + "pattern": "^[^\\u0000]*$" + }, + { + "pattern": "^(?![\\s\\S]*(?:[\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890-\\u0891\\u08E2\\u180E\\u200B-\\u200F\\u2028-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFEFF\\uFFF9-\\uFFFB]|\\uD804[\\uDCBD\\uDCCD]|\\uD80D[\\uDC30-\\uDC3F]|\\uD82F[\\uDCA0-\\uDCA3]|\\uD834[\\uDD73-\\uDD7A]|\\uDB40(?:\\uDC01|[\\uDC20-\\uDC7F])))[\\s\\S]*$" + } + ] + }, + "timeoutMs": { + "type": "integer", + "minimum": 1000, + "maximum": 86400000 + }, + "triggerType": { + "enum": [ + "manual", + "schedule", + "startup", + "system" + ], + "type": "string" + }, + "updatedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + } + }, + "required": [ + "actionKey", + "attemptCount", + "attemptLimit", + "availableAtMs", + "cancellationPolicy", + "displayName", + "eventCount", + "id", + "priority", + "queuedAtMs", + "resourceClass", + "resourceKeys", + "retrySafe", + "state", + "stateVersion", + "timeoutMs", + "triggerType", + "updatedAtMs" + ], + "additionalProperties": false, + "$comment": "Live Valibot validation additionally requires run provenance, attempts, state, cancellation, and timestamps to agree." + }, + "availability": { + "enum": [ + "available", + "unavailable" + ], + "type": "string" + }, + "id": { + "enum": [ + "openclaw-cleanup", + "openclaw-update", + "system-restart", + "system-update" + ], + "type": "string" + }, + "latestRun": { + "type": "object", + "properties": { + "actionKey": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "attemptCount": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, + "attemptLimit": { + "type": "integer", + "minimum": 1, + "maximum": 10 + }, + "availableAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "cancellationPolicy": { + "enum": [ + "cooperative", + "never", + "queued-only" + ], + "type": "string" + }, + "cancelRequestedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "displayName": { + "type": "string", + "maxLength": 160, + "minLength": 1, + "pattern": "\\S", + "allOf": [ + { + "pattern": "^[^\\u0000]*$" + }, + { + "pattern": "^(?![\\s\\S]*(?:[\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890-\\u0891\\u08E2\\u180E\\u200B-\\u200F\\u2028-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFEFF\\uFFF9-\\uFFFB]|\\uD804[\\uDCBD\\uDCCD]|\\uD80D[\\uDC30-\\uDC3F]|\\uD82F[\\uDCA0-\\uDCA3]|\\uD834[\\uDD73-\\uDD7A]|\\uDB40(?:\\uDC01|[\\uDC20-\\uDC7F])))[\\s\\S]*$" + } + ] + }, + "eventCount": { + "type": "integer", + "minimum": 0, + "maximum": 1000 + }, + "finishedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "firstStartedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "id": { + "type": "string", + "minLength": 36, + "maxLength": 36, + "format": "uuid", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "lastAttemptStartedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "priority": { + "type": "integer", + "minimum": -100, + "maximum": 100 + }, + "queuedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "resourceClass": { + "enum": [ + "exclusive", + "host-heavy", + "interactive", + "light", + "network" + ], + "type": "string" + }, + "resourceKeys": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "maxItems": 32, + "$comment": "Live Valibot validation additionally requires resource keys to be unique, strictly sorted, and within their aggregate UTF-8 byte budget." + }, + "retrySafe": { + "type": "boolean" + }, + "scheduledForAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + }, + "scheduledJobId": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "scheduledJobVersion": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "state": { + "enum": [ + "cancelled", + "failed", + "queued", + "running", + "succeeded", + "timed-out" + ], + "type": "string" + }, + "stateVersion": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "terminalCode": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9][a-z0-9._/-]*$" + }, + "terminalMessage": { + "type": "string", + "maxLength": 2000, + "minLength": 1, + "pattern": "\\S", + "allOf": [ + { + "pattern": "^[^\\u0000]*$" + }, + { + "pattern": "^(?![\\s\\S]*(?:[\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890-\\u0891\\u08E2\\u180E\\u200B-\\u200F\\u2028-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFEFF\\uFFF9-\\uFFFB]|\\uD804[\\uDCBD\\uDCCD]|\\uD80D[\\uDC30-\\uDC3F]|\\uD82F[\\uDCA0-\\uDCA3]|\\uD834[\\uDD73-\\uDD7A]|\\uDB40(?:\\uDC01|[\\uDC20-\\uDC7F])))[\\s\\S]*$" + } + ] + }, + "timeoutMs": { + "type": "integer", + "minimum": 1000, + "maximum": 86400000 + }, + "triggerType": { + "enum": [ + "manual", + "schedule", + "startup", + "system" + ], + "type": "string" + }, + "updatedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + } + }, + "required": [ + "actionKey", + "attemptCount", + "attemptLimit", + "availableAtMs", + "cancellationPolicy", + "displayName", + "eventCount", + "id", + "priority", + "queuedAtMs", + "resourceClass", + "resourceKeys", + "retrySafe", + "state", + "stateVersion", + "timeoutMs", + "triggerType", + "updatedAtMs" + ], + "additionalProperties": false, + "$comment": "Live Valibot validation additionally requires run provenance, attempts, state, cancellation, and timestamps to agree." + } + }, + "required": [ + "availability", + "id" + ], + "additionalProperties": false + }, + "maxItems": 4, + "$comment": "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered." + }, + "observedAtMs": { + "type": "integer", + "minimum": 0, + "maximum": 8640000000000000 + } + }, + "required": [ + "actions", + "observedAtMs" + ], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} diff --git a/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json b/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json new file mode 100644 index 000000000..e25b87cfc --- /dev/null +++ b/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json @@ -0,0 +1,98 @@ +{ + "$id": "urn:mira-dashboard:serviceActions.request.input", + "oneOf": [ + { + "type": "object", + "properties": { + "actionId": { + "const": "openclaw-cleanup" + }, + "confirmation": { + "const": "cleanup-openclaw" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "actionId": { + "const": "openclaw-update" + }, + "confirmation": { + "const": "update-openclaw" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "actionId": { + "const": "system-restart" + }, + "confirmation": { + "const": "restart-system" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "actionId": { + "const": "system-update" + }, + "confirmation": { + "const": "update-system" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + } + ], + "$schema": "https://json-schema.org/draft/2020-12/schema" +} diff --git a/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json b/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json new file mode 100644 index 000000000..c9688b937 --- /dev/null +++ b/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json @@ -0,0 +1,32 @@ +{ + "$id": "urn:mira-dashboard:serviceActions.request.output", + "type": "object", + "properties": { + "actionId": { + "enum": [ + "openclaw-cleanup", + "openclaw-update", + "system-restart", + "system-update" + ], + "type": "string" + }, + "jobRunId": { + "type": "string", + "minLength": 36, + "maxLength": 36, + "format": "uuid", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "queued": { + "const": true + } + }, + "required": [ + "actionId", + "jobRunId", + "queued" + ], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql index cf1bddca3..59ea2c7b8 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql +++ b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql @@ -139,7 +139,7 @@ CREATE TABLE `automation_principal_capabilities` ( `principal_id` text NOT NULL, CONSTRAINT `automation_principal_capabilities_pk` PRIMARY KEY(`principal_id`, `capability`), CONSTRAINT `fk_automation_principal_capabilities_principal_id_automation_principals_id_fk` FOREIGN KEY (`principal_id`) REFERENCES `automation_principals`(`id`) ON DELETE CASCADE, - CONSTRAINT "automation_principal_capabilities_capability_check" CHECK("capability" IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')), + CONSTRAINT "automation_principal_capabilities_capability_check" CHECK("capability" IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'service-actions:read', 'service-actions:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')), CONSTRAINT "automation_principal_capabilities_granted_at_check" CHECK("granted_at" BETWEEN 0 AND 8640000000000000) ) STRICT; --> statement-breakpoint @@ -1084,6 +1084,7 @@ CREATE TABLE `scheduled_jobs` ( ) STRICT, WITHOUT ROWID; --> statement-breakpoint CREATE TABLE `worker_instances` ( + `action_keys_json` text DEFAULT '[]' NOT NULL, `capacity` integer NOT NULL, `draining_at` integer, `heartbeat_at` integer NOT NULL, @@ -1093,6 +1094,7 @@ CREATE TABLE `worker_instances` ( `started_at` integer NOT NULL, `state` text NOT NULL, `stopped_at` integer, + CONSTRAINT "worker_instances_action_keys_json_check" CHECK(length(CAST("action_keys_json" AS BLOB)) <= 4096 AND CASE WHEN json_valid("action_keys_json") THEN json_type("action_keys_json") = 'array' ELSE 0 END), CONSTRAINT "worker_instances_capacity_check" CHECK("capacity" BETWEEN 1 AND 16), CONSTRAINT "worker_instances_id_check" CHECK(length("id") = 36 AND instr("id", char(0)) = 0 AND length(replace("id", '-', '')) = 32 AND replace("id", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr("id", 9, 1) = '-' AND substr("id", 14, 1) = '-' AND substr("id", 15, 1) = '7' AND substr("id", 19, 1) = '-' AND substr("id", 20, 1) GLOB '[89ab]' AND substr("id", 24, 1) = '-'), CONSTRAINT "worker_instances_pid_check" CHECK("pid" BETWEEN 1 AND 2147483647), @@ -1808,8 +1810,31 @@ BEGIN SELECT RAISE(ABORT, 'worker_instances identity is immutable'); END; --> statement-breakpoint +CREATE TRIGGER worker_instances_validate_action_keys_insert +BEFORE INSERT ON worker_instances +WHEN json_array_length(NEW.action_keys_json) > 32 + OR EXISTS ( + SELECT 1 + FROM json_each(NEW.action_keys_json) AS entry + WHERE entry.type <> 'text' + OR length(CAST(entry.value AS TEXT)) NOT BETWEEN 1 AND 128 + OR CAST(entry.value AS TEXT) <> lower(CAST(entry.value AS TEXT)) + OR substr(CAST(entry.value AS TEXT), 1, 1) NOT GLOB '[a-z0-9]' + OR CAST(entry.value AS TEXT) GLOB '*[^a-z0-9._-]*' + ) + OR EXISTS ( + SELECT 1 + FROM json_each(NEW.action_keys_json) AS current + JOIN json_each(NEW.action_keys_json) AS previous + ON previous.key = current.key - 1 + WHERE CAST(current.value AS TEXT) <= CAST(previous.value AS TEXT) + ) +BEGIN + SELECT RAISE(ABORT, 'worker_instances action keys must be canonical'); +END; +--> statement-breakpoint CREATE TRIGGER worker_instances_reject_identity_update -BEFORE UPDATE OF id, release_id, pid, capacity, started_at ON worker_instances +BEFORE UPDATE OF id, release_id, pid, capacity, started_at, action_keys_json ON worker_instances BEGIN SELECT RAISE(ABORT, 'worker_instances identity is immutable'); END; diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json index 608e1f1de..f8e324254 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json +++ b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json @@ -4138,6 +4138,16 @@ "entityType": "columns", "table": "users" }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'[]'", + "generated": null, + "name": "action_keys_json", + "entityType": "columns", + "table": "worker_instances" + }, { "type": "integer", "notNull": true, @@ -7097,7 +7107,7 @@ "table": "automation_credentials" }, { - "value": "\"capability\" IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')", + "value": "\"capability\" IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'service-actions:read', 'service-actions:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')", "name": "automation_principal_capabilities_capability_check", "entityType": "checks", "table": "automation_principal_capabilities" @@ -8416,6 +8426,12 @@ "entityType": "checks", "table": "users" }, + { + "value": "length(CAST(\"action_keys_json\" AS BLOB)) <= 4096 AND CASE WHEN json_valid(\"action_keys_json\") THEN json_type(\"action_keys_json\") = 'array' ELSE 0 END", + "name": "worker_instances_action_keys_json_check", + "entityType": "checks", + "table": "worker_instances" + }, { "value": "\"capacity\" BETWEEN 1 AND 16", "name": "worker_instances_capacity_check", diff --git a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json index 4592fbdcc..3cf27b03c 100644 --- a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json +++ b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json @@ -16,6 +16,10 @@ "file": "gateway.json", "sha256": "dbd4311e4e31b71855527459aa0f7f389c99dba58ba79c066f77334c1d1cf6e3" }, + { + "file": "operations.json", + "sha256": "bfb91bffba35131f3bdcde1900e6a33904ee9188ac32318b28aa78bc07ed69c7" + }, { "file": "sessions.json", "sha256": "91785d6d4ee208b2dace06c48473341195c5782b8a371317cc63e463e75757a1" @@ -381,12 +385,24 @@ "role": "runtime-subscriptions", "sha256": "171c52bd6a3d10e8555fdaf00aed3ce225fd465911ef8958ce8235751a4c6f7b" }, + { + "bytes": 199139, + "path": "dist/session-accessor.sqlite-2Th60fFl.js", + "role": "session-accessor-sqlite-maintenance", + "sha256": "a028cf6a72edd7fd19d51985eb32111b836dd6336c98843c0ec1291e28d3c11a" + }, { "bytes": 2077, "path": "dist/session-change-event-B7AM9yTQ.js", "role": "session-change-event", "sha256": "3fa61e8f0664d4943c474f22d680dc391cff689044328129a02e9fc21b7ce0ea" }, + { + "bytes": 15719, + "path": "dist/cleanup-service-BAH9Mem2.js", + "role": "session-cleanup-service", + "sha256": "281f338cbd533c6438d345c2eed7c74f72f9f97cdf12cb5f988d783ca478678a" + }, { "bytes": 3364, "path": "dist/session-companion-rpc-BItcEiDG.js", @@ -417,6 +433,12 @@ "role": "session-list-projection", "sha256": "f0025237779a8fd3ea2ac3f79b9f45c12ccf05850995dd525014b9321b1d5328" }, + { + "bytes": 106305, + "path": "dist/session-entry-slot-keys-DRaa8e03.js", + "role": "session-maintenance-policy", + "sha256": "da1a930f1d759f8648c5786c0a921cb623c60cfd871d4ffaa4375170d552cf96" + }, { "bytes": 7598, "path": "dist/sessions-shared-DiPcjD6h.js", @@ -531,6 +553,30 @@ "role": "transcript-media-persistence", "sha256": "73fef1dff84be8783531dae1528e25dbfd2cfe2d3d7abdaa1eefea0320529f0e" }, + { + "bytes": 17875, + "path": "dist/update-CkwHKf6r.js", + "role": "update-handlers", + "sha256": "b7ee1ea9336b1b8d496a2610b041d3af8849fe92a0ccf09b3ca559965061c330" + }, + { + "bytes": 50211, + "path": "dist/update-startup-uZ_Jn11N.js", + "role": "update-managed-handoff", + "sha256": "77e3141db24e6eea7cb10ee10725c4feed72f22db6d2dfc57520ec894747e852" + }, + { + "bytes": 118137, + "path": "dist/update-runner-DBlb9fqo.js", + "role": "update-runner", + "sha256": "547191d6df5f35a1dcf930881a2d931febd56a09c4057af487eab943dd8f22b9" + }, + { + "bytes": 5490, + "path": "dist/update-control-plane-sentinel-CC-WfVRy.js", + "role": "update-sentinel", + "sha256": "d74fed63f194afdf67fefbac58cdfefd69bf679fa9355bedcd2e5eb1181dc804" + }, { "bytes": 6633, "path": "dist/runtime-Etb7L8Ef.js", diff --git a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json new file mode 100644 index 000000000..8532f1911 --- /dev/null +++ b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json @@ -0,0 +1,169 @@ +{ + "domain": "operations", + "methodAccess": [ + { + "controlPlaneWrite": false, + "lane": "one-shot-admin", + "method": "sessions.cleanup", + "scope": "operator.admin" + }, + { + "controlPlaneWrite": true, + "lane": "one-shot-admin", + "method": "update.run", + "scope": "operator.admin" + } + ], + "methods": ["sessions.cleanup", "update.run"], + "schemaVersion": 1, + "sessionsCleanup": { + "handlerValidatesParams": true, + "method": "sessions.cleanup", + "mutation": { + "diskBudgetEnforcedAfterEntryMaintenance": true, + "entryStateRecheckedBeforeRemoval": true, + "unreferencedArtifactsPrunedOutsideWarnMode": true, + "usesSqliteLifecycleMutation": true + }, + "outcome": { + "automaticReplaySafe": false, + "handlerTimeoutParameter": false, + "idempotencyParameter": false, + "postDispatchTransportTimeout": "outcome-unknown" + }, + "preservation": { + "activeKeyAndParentsPreserved": true, + "activeWorkAdmissionsPreserved": true, + "archivedEntriesPreserved": true, + "groupChannelAndThreadEntriesPreserved": true, + "modelSelectionLockedEntriesPreserved": true, + "primarySessionsPreserved": true, + "registeredRuntimeKeysPreserved": true + }, + "request": { + "acceptedParams": [ + "activeKey", + "agent", + "allAgents", + "enforce", + "fixDmScope", + "fixMissing" + ], + "closedObject": true, + "requiredParams": [] + }, + "response": { + "appliedStoreFields": [ + "agentId", + "storePath", + "mode", + "dryRun", + "beforeCount", + "afterCount", + "missing", + "dmScopeRetired", + "modelRunPruned", + "pruned", + "capped", + "unreferencedArtifacts", + "diskBudget", + "wouldMutate", + "applied", + "appliedCount" + ], + "diskBudgetFields": [ + "totalBytesBefore", + "totalBytesAfter", + "removedFiles", + "removedEntries", + "freedBytes", + "maxBytes", + "highWaterBytes", + "overBudget" + ], + "formattedUpstreamErrorMustBeSanitized": true, + "multiStoreFields": ["allAgents", "mode", "dryRun", "stores"], + "sensitivePaths": ["storePath", "stores[].storePath"], + "unreferencedArtifactFields": [ + "scannedFiles", + "removedFiles", + "freedBytes", + "olderThanMs" + ] + }, + "semantics": { + "activeKeyOptional": true, + "enforceTrueOverridesConfiguredMode": true, + "fixDmScopeDefaultsFalse": true, + "fixMissingDefaultsFalse": true, + "maintenanceConfigSource": "session.maintenance", + "rpcAlwaysAppliesRatherThanDryRuns": true + } + }, + "updateRun": { + "handlerValidatesParams": true, + "managedHandoff": { + "activeFlightJoinedWithoutSecondSpawn": true, + "detachedChild": true, + "gitRequiresSupervisor": true, + "globalInstallRequiresHandoff": true, + "readyMarkerTimeoutMs": 30000, + "sensitiveTemporaryFilesRemoved": true, + "startedHandoffCountsAsAccepted": true, + "systemdMinimumRestartDelayMs": 2000, + "systemdRequiresUnitContext": true, + "systemdUsesUserScope": true + }, + "method": "update.run", + "outcome": { + "automaticReplaySafe": false, + "handlerAbortSignal": false, + "idempotencyParameter": false, + "operationalErrorsUseRpcSuccess": true, + "postDispatchTransportTimeout": "outcome-unknown" + }, + "request": { + "acceptedParams": [ + "continuationMessage", + "deliveryContext", + "note", + "restartDelayMs", + "sessionKey", + "timeoutMs" + ], + "closedObject": true, + "requiredParams": [], + "restartDelayMinimumMs": 0, + "timeoutMinimumMs": 1 + }, + "response": { + "okWhenHandoffStarted": true, + "okWhenResultStatusOk": true, + "resultStatuses": ["error", "ok", "skipped"], + "sentinelPersistenceBestEffort": true, + "sensitivePaths": [ + "handoff.command", + "handoff.message", + "handoff.pid", + "result.root", + "result.steps[].command", + "result.steps[].cwd", + "result.steps[].stderrTail", + "result.steps[].stdoutTail", + "restart.pid", + "sentinel.payload" + ], + "topLevelFields": ["ok", "result", "handoff", "restart", "sentinel"] + }, + "restart": { + "directSuccessSchedulesSigusr1": true, + "managedSystemdSkipsCooldownAndDeferral": true, + "packageSwapSkipsCooldownAndDeferral": true + }, + "timeout": { + "defaultRunnerPerStepMs": 1200000, + "handlerFloorMs": 1000, + "perStepRatherThanWholeOperation": true + } + } +} diff --git a/greenfield/scripts/audits/openclaw/reviewedFixtures.ts b/greenfield/scripts/audits/openclaw/reviewedFixtures.ts index dec29d3a8..7d975d96b 100644 --- a/greenfield/scripts/audits/openclaw/reviewedFixtures.ts +++ b/greenfield/scripts/audits/openclaw/reviewedFixtures.ts @@ -9,6 +9,7 @@ import { chatFixtureSchema, cronFixtureSchema, gatewayFixtureSchema, + operationsFixtureSchema, parseFixtureDocument, parseFixtureManifest, parseSourceAuditResult, @@ -26,6 +27,7 @@ const reviewedFixtureFileNames = [ "cron.json", "gateway.json", "manifest.json", + "operations.json", "sessions.json", "settings.json", "tasks.json", @@ -132,6 +134,10 @@ export async function loadReviewedOpenClawFixtures( chat: parseFixtureDocument(chatFixtureSchema, required("chat.json")), cron: parseFixtureDocument(cronFixtureSchema, required("cron.json")), gateway: parseFixtureDocument(gatewayFixtureSchema, required("gateway.json")), + operations: parseFixtureDocument( + operationsFixtureSchema, + required("operations.json") + ), sessions: parseFixtureDocument(sessionsFixtureSchema, required("sessions.json")), settings: parseFixtureDocument(settingsFixtureSchema, required("settings.json")), tasks: parseFixtureDocument(tasksFixtureSchema, required("tasks.json")), @@ -169,6 +175,7 @@ function fixtureComponents(audit: SourceAuditResult): readonly [string, unknown] ["chat.json", audit.chat], ["cron.json", audit.cron], ["gateway.json", audit.gateway], + ["operations.json", audit.operations], ["sessions.json", audit.sessions], ["settings.json", audit.settings], ["tasks.json", audit.tasks], diff --git a/greenfield/scripts/audits/openclaw/sourceAudit.ts b/greenfield/scripts/audits/openclaw/sourceAudit.ts index 55f466fbb..45a47d45f 100644 --- a/greenfield/scripts/audits/openclaw/sourceAudit.ts +++ b/greenfield/scripts/audits/openclaw/sourceAudit.ts @@ -488,6 +488,24 @@ const distributionArtifactSpecs: readonly DistributionArtifactSpec[] = [ ], role: "runtime-subscriptions", }, + { + fileNamePattern: /^session-accessor\.sqlite-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "function collectSqliteSessionMaintenanceBaseKeys(store, activeSessionKey)", + "function applySqliteSessionEntryMaintenance(database, params)", + "async function applySqliteSessionEntryLifecycleMutation(params)", + ], + role: "session-accessor-sqlite-maintenance", + }, + { + fileNamePattern: /^cleanup-service-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "function serializeSessionCleanupResult(params)", + "async function previewStoreCleanup(params)", + "async function runSessionsCleanup(params)", + ], + role: "session-cleanup-service", + }, { fileNamePattern: /^session-companion-rpc-[A-Za-z0-9_-]+\.js$/u, markers: [ @@ -561,6 +579,15 @@ const distributionArtifactSpecs: readonly DistributionArtifactSpec[] = [ ], role: "session-list-projection", }, + { + fileNamePattern: /^session-entry-slot-keys-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "function collectSessionMaintenancePreserveKeysForStore(params)", + "function shouldPreserveMaintenanceEntry(params)", + "function resolveMaintenanceConfig()", + ], + role: "session-maintenance-policy", + }, { fileNamePattern: /^session-reset-service-[A-Za-z0-9_-]+\.js$/u, markers: [ @@ -674,6 +701,43 @@ const distributionArtifactSpecs: readonly DistributionArtifactSpec[] = [ ], role: "subagent-control", }, + { + fileNamePattern: /^update-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "const updateHandlers = {", + '"update.run": async', + "startManagedServiceUpdateHandoff", + "buildUpdateRestartSentinelPayload", + ], + role: "update-handlers", + }, + { + fileNamePattern: /^update-startup-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "const HANDOFF_READY_MARKER", + "function formatManagedServiceUpdateCommand(params)", + "async function startManagedServiceUpdateHandoff(params)", + ], + role: "update-managed-handoff", + }, + { + fileNamePattern: /^update-runner-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "const MAX_LOG_CHARS = 8e3", + "async function runStep(opts)", + "async function runGatewayUpdate(opts = {})", + ], + role: "update-runner", + }, + { + fileNamePattern: /^update-control-plane-sentinel-[A-Za-z0-9_-]+\.js$/u, + markers: [ + "function buildUpdateRestartSentinelPayload(params)", + "CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON", + "function isPendingControlPlaneUpdateRestartSentinel(payload)", + ], + role: "update-sentinel", + }, { fileNamePattern: /^task-registry-[A-Za-z0-9_-]+\.js$/u, markers: [ @@ -2935,6 +2999,593 @@ function assertPhase4SessionsSemantics( }; } +function assertOpenClawOperationsSemantics( + artifacts: readonly LoadedSourceArtifact[] +): SourceAuditResult["operations"] { + const protocol = artifactByRole(artifacts, "protocol-schemas").contents; + const cleanupParams = boundedSourceRegion( + protocol, + "/** Repairs or removes invalid session records from the selected agent scope. */", + "/** Reads short previews for selected session keys. */", + 2 * 1024, + "sessions.cleanup params" + ); + assertExactIndentedFields( + cleanupParams, + 1, + ["activeKey", "agent", "allAgents", "enforce", "fixDmScope", "fixMissing"], + "sessions.cleanup params" + ); + assertRequiredMarkers(cleanupParams, "sessions.cleanup optional params", [ + "agent: Type.Optional(NonEmptyString)", + "allAgents: Type.Optional(Type.Boolean())", + "enforce: Type.Optional(Type.Boolean())", + "activeKey: Type.Optional(NonEmptyString)", + "fixMissing: Type.Optional(Type.Boolean())", + "fixDmScope: Type.Optional(Type.Boolean())", + ]); + assertForbiddenMarkers(cleanupParams, "sessions.cleanup request authority", [ + "idempotencyKey", + "timeoutMs", + ]); + + const updateParams = boundedSourceRegion( + protocol, + "/** Request payload for running an update/restart flow with optional channel delivery context. */", + "/** UI metadata attached to config schema paths. */", + 2 * 1024, + "update.run params" + ); + assertExactIndentedFields( + updateParams, + 1, + [ + "continuationMessage", + "deliveryContext", + "note", + "restartDelayMs", + "sessionKey", + "timeoutMs", + ], + "update.run params" + ); + assertRequiredMarkers(updateParams, "update.run optional params", [ + "sessionKey: Type.Optional(Type.String())", + "deliveryContext: Type.Optional(ConfigDeliveryContextSchema)", + "note: Type.Optional(Type.String())", + "continuationMessage: Type.Optional(Type.String())", + "restartDelayMs: Type.Optional(Type.Integer({ minimum: 0 }))", + "timeoutMs: Type.Optional(Type.Integer({ minimum: 1 }))", + ]); + assertForbiddenMarkers(updateParams, "update.run replay authority", [ + "idempotencyKey", + "abortSignal", + ]); + + const descriptors = artifactByRole(artifacts, "method-descriptors").contents; + assertMethodPermission(descriptors, "sessions.cleanup", "operator.admin", false); + assertMethodPermission(descriptors, "update.run", "operator.admin", true); + + const sessionHandlers = artifactByRole(artifacts, "sessions-handlers").contents; + const cleanupHandler = boundedSourceRegion( + sessionHandlers, + '"sessions.cleanup": async', + '"sessions.preview":', + 8 * 1024, + "sessions.cleanup handler" + ); + assertRequiredMarkers(cleanupHandler, "sessions.cleanup handler", [ + 'assertValidParams(params, validateSessionsCleanupParams, "sessions.cleanup", respond)', + "const { mode, appliedSummaries } = await runSessionsCleanup({", + "agent: params.agent", + "allAgents: params.allAgents", + "enforce: params.enforce", + "activeKey: params.activeKey", + "fixMissing: params.fixMissing", + "fixDmScope: params.fixDmScope", + "serializeSessionCleanupResult({", + "summaries: appliedSummaries", + 'reason: "cleanup"', + "errorShape(ErrorCodes.INVALID_REQUEST, formatErrorMessage(error))", + ]); + + const cleanupService = artifactByRole(artifacts, "session-cleanup-service").contents; + const cleanupSerialization = boundedSourceRegion( + cleanupService, + "function serializeSessionCleanupResult(params) {", + "function pruneMissingTranscriptEntries(params) {", + 2 * 1024, + "sessions.cleanup serialization" + ); + assertRequiredMarkers(cleanupSerialization, "sessions.cleanup serialization", [ + "if (params.summaries.length === 1) return params.summaries[0] ?? {}", + "allAgents: true", + "mode: params.mode", + "dryRun: params.dryRun", + "stores: params.summaries", + ]); + const cleanupExecution = boundedSourceRegion( + cleanupService, + "async function runSessionsCleanup(params) {", + "/** Purge session store entries for a deleted agent", + 32 * 1024, + "sessions.cleanup execution" + ); + assertRequiredMarkers(cleanupExecution, "sessions.cleanup execution", [ + "const maintenance = resolveMaintenanceConfig()", + 'const mode = opts.enforce ? "enforce" : maintenance.mode', + "fixMissing: Boolean(opts.fixMissing)", + "fixDmScope: Boolean(opts.fixDmScope)", + "const lifecycleResult = await applySqliteSessionEntryLifecycleMutation({", + "activeSessionKey: opts.activeKey", + "maintenanceOverride: {", + 'const appliedUnreferencedArtifacts = mode === "warn" ? null : await pruneUnreferencedSessionArtifacts({', + "const appliedDiskBudget = await enforceSqliteSessionHistoryDiskBudget({", + "agentId: target.agentId", + "storePath: target.storePath", + "mode: appliedReport.mode", + "dryRun: false", + "beforeCount: appliedReport.beforeCount", + "afterCount: appliedReport.afterCount", + "missing: missingApplied", + "dmScopeRetired: dmScopeRetiredApplied", + "modelRunPruned: appliedReport.modelRunPruned", + "pruned: appliedReport.pruned", + "capped: appliedReport.capped", + "unreferencedArtifacts,", + "diskBudget: appliedDiskBudget", + "wouldMutate:", + "applied: true", + "appliedCount: lifecycleResult.afterCount", + ]); + + const maintenancePolicy = artifactByRole( + artifacts, + "session-maintenance-policy" + ).contents; + const activePreservation = boundedSourceRegion( + maintenancePolicy, + "/** Collects every runtime and active-work key protected from automatic maintenance. */", + "//#endregion", + 2 * 1024, + "session cleanup active preservation" + ); + assertRequiredMarkers(activePreservation, "session cleanup active preservation", [ + "collectSessionMaintenancePreserveKeys(params.baseKeys)", + "collectActiveSessionWorkAdmissionKeys({", + "storePath: params.storePath", + "store: params.store", + ]); + const entryPreservation = boundedSourceRegion( + maintenancePolicy, + "function isProtectedSessionMaintenanceEntry(sessionKey, entry) {", + "function getActiveSessionMaintenanceWarning(params) {", + 4 * 1024, + "session cleanup entry preservation" + ); + assertRequiredMarkers(entryPreservation, "session cleanup entry preservation", [ + "if (isPrimarySessionMaintenanceKey(sessionKey)) return true", + "if (parseSessionThreadInfoFast(sessionKey).threadId) return true", + "if (isTelegramTopicSessionKey(sessionKey)) return true", + "if (isExternalGroupOrChannelSessionKey(sessionKey)) return true", + 'return chatType === "group" || chatType === "channel" || chatType === "thread"', + "if (params.entry?.archivedAt !== void 0) return true", + "params.entry?.modelSelectionLocked === true", + "params.preserveKeys?.has(params.key) === true", + ]); + const maintenanceConfig = boundedSourceRegion( + maintenancePolicy, + "function resolveMaintenanceConfig() {", + "//#endregion", + 2 * 1024, + "session cleanup maintenance config" + ); + assertRequiredMarkers(maintenanceConfig, "session cleanup maintenance config", [ + "getRuntimeConfig().session?.maintenance", + "return resolveMaintenanceConfigFromInput(maintenance)", + ]); + const artifactPruning = boundedSourceRegion( + maintenancePolicy, + "async function pruneUnreferencedSessionArtifacts(params) {", + "async function enforceSessionDiskBudget(params) {", + 16 * 1024, + "session cleanup unreferenced artifact result" + ); + assertRequiredMarkers( + artifactPruning, + "session cleanup unreferenced artifact result", + [ + "scannedFiles: files.length + promptBlobFiles.length", + "removedFiles,", + "freedBytes,", + "olderThanMs", + ] + ); + const diskBudget = boundedSourceRegion( + maintenancePolicy, + "async function enforceSessionDiskBudget(params) {", + "//#endregion", + 32 * 1024, + "session cleanup disk budget result" + ); + assertRequiredMarkers(diskBudget, "session cleanup disk budget result", [ + "totalBytesBefore: totalBefore", + "totalBytesAfter: total", + "removedFiles,", + "removedEntries,", + "freedBytes,", + "maxBytes,", + "highWaterBytes,", + "overBudget: true", + ]); + + const sqliteMaintenance = artifactByRole( + artifacts, + "session-accessor-sqlite-maintenance" + ).contents; + const sqliteEntryMaintenance = boundedSourceRegion( + sqliteMaintenance, + "function collectSqliteSessionMaintenanceBaseKeys(store, activeSessionKey) {", + "function finalizeSqliteSessionEntryMaintenancePlansBestEffort(scope, plans) {", + 24 * 1024, + "SQLite session cleanup preservation" + ); + assertRequiredMarkers(sqliteEntryMaintenance, "SQLite session cleanup preservation", [ + 'currentKey = normalizeStoreSessionKey(store[currentKey]?.parentSessionKey ?? "")', + "collectSessionMaintenancePreserveKeysForStore({", + "baseKeys: collectSqliteSessionMaintenanceBaseKeys(store, params.activeSessionKey)", + "pruneStaleEntries(store, maintenance.pruneAfterMs", + "capEntryCount(store, maintenance.maxEntries", + "preserveKeys", + ]); + const lifecycleMutation = boundedSourceRegion( + sqliteMaintenance, + "async function applySqliteSessionEntryLifecycleMutation(params) {", + "/** Purges entries owned by a deleted agent from SQLite session rows. */", + 32 * 1024, + "SQLite cleanup lifecycle mutation" + ); + assertRequiredMarkers(lifecycleMutation, "SQLite cleanup lifecycle mutation", [ + "if (!sqliteSessionEntriesEqual(entry, removal.expectedEntry))", + 'activeSessionKey: params.activeSessionKey ?? ""', + "forceMaintenance: params.maintenanceOverride !== void 0", + "maintenanceConfig: params.maintenanceOverride ? {", + ]); + + const updateHandlers = artifactByRole(artifacts, "update-handlers").contents; + const managedRestartPolicy = boundedSourceRegion( + updateHandlers, + "const MANAGED_HANDOFF_RESTART_DELAY_MS = 2e3;", + "const updateHandlers = {", + 8 * 1024, + "managed update restart policy" + ); + assertRequiredMarkers(managedRestartPolicy, "managed update restart policy", [ + "const resolvedDelayMs = restartDelayMs ?? MANAGED_HANDOFF_RESTART_DELAY_MS", + 'if (supervisor !== "systemd") return resolvedDelayMs', + "return Math.max(resolvedDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS)", + 'if (supervisor === "systemd") return Boolean(env.OPENCLAW_SYSTEMD_UNIT?.trim())', + ]); + const updateRunHandler = boundedSourceRegion( + updateHandlers, + '"update.run": async', + "//#endregion", + 48 * 1024, + "update.run handler" + ); + assertRequiredMarkers(updateRunHandler, "update.run handler", [ + 'assertValidParams(params, validateUpdateRunParams, "update.run", respond)', + "const timeoutMsRaw = params.timeoutMs", + "Math.max(1e3, Math.floor(timeoutMsRaw))", + 'const requiresManagedServiceHandoff = installSurface.kind === "global" || installSurface.kind === "git" && supervisor !== null', + "const hasHandoffContext = supervisor ? hasManagedServiceHandoffContext(process.env, supervisor) : false", + "const started = await startManagedServiceUpdateHandoff({", + 'ownsManagedServiceHandoff = started.status === "started"', + "...started.pid ? { pid: started.pid } : {}", + "command: started.command", + 'message: "Another managed update is already running; retry after it completes."', + "managedHandoffRestart = scheduleGatewaySigusr1Restart({", + 'reason: "update.run"', + "skipDeferral: true", + "skipCooldown: true", + "result = await runGatewayUpdate({", + "allowGatewayServiceRepair: false", + "allowGatewayActivation: false", + "const payload = buildUpdateRestartSentinelPayload({", + "await writeRestartSentinel(payload)", + 'const updateWasPackageSwap = result.status === "ok" && result.mode !== "git"', + 'ok: result.status === "ok" || handoff?.status === "started"', + "result,", + "...handoff ? { handoff } : {}", + "restart,", + "sentinel: {", + "persisted: sentinelPersisted", + "payload", + ]); + if (updateRunHandler.includes("respond(false")) { + throw new Error( + "OpenClaw update.run operational settlement changed outside the reviewed shape" + ); + } + assertForbiddenMarkers(updateRunHandler, "update.run replay authority", [ + "abortSignal", + "idempotencyKey", + ]); + + const managedHandoff = artifactByRole(artifacts, "update-managed-handoff").contents; + assertRequiredMarkers(managedHandoff, "managed update readiness deadline", [ + "HANDOFF_READY_TIMEOUT_MS = 3e4", + ]); + const handoffCommand = boundedSourceRegion( + managedHandoff, + "function resolveUpdateCliArgv(params) {", + "function resolveGatewayServiceRecovery(supervisor, env) {", + 8 * 1024, + "managed update command" + ); + assertRequiredMarkers(handoffCommand, "managed update command", [ + '"update"', + '"--yes"', + '"--json"', + 'updateArgs.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1e3))))', + 'args.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1e3))))', + ]); + const handoffSpawn = boundedSourceRegion( + managedHandoff, + "async function waitForHandoffReady(child) {", + "function buildManagedServiceHandoffUnavailableMessage(command) {", + 24 * 1024, + "managed update handoff" + ); + assertRequiredMarkers(handoffSpawn, "managed update handoff", [ + "buffered.includes(HANDOFF_READY_MARKER)", + 'new Error("managed update handoff did not signal readiness within 30 seconds")', + '"--user"', + '"--scope"', + '"--collect"', + "detached: true", + "sensitivePaths: [", + "scriptPath", + "paramsPath", + "metaPath", + "child.unref()", + "if (active) return {", + "...await active", + 'status: "joined"', + ]); + assertRequiredMarkers(managedHandoff, "managed update sensitive cleanup", [ + "function cleanupSensitiveFiles()", + "cleanupSensitiveFiles();", + ]); + + const updateRunner = artifactByRole(artifacts, "update-runner").contents; + assertRequiredMarkers(updateRunner, "update result status", ['status: "ok"']); + const updateStep = boundedSourceRegion( + updateRunner, + "async function runStep(opts) {", + "function normalizeFallbackFailureReason(stepName) {", + 8 * 1024, + "update command step" + ); + assertRequiredMarkers(updateStep, "update command step", [ + "const { runCommand, name, argv, cwd, timeoutMs", + "const result = await runCommand(argv, {", + "timeoutMs,", + "command,", + "cwd,", + "stdoutTail: trimLogTail(result.stdout, MAX_LOG_CHARS)", + "stderrTail,", + ]); + const updateRunnerEntry = boundedSourceRegion( + updateRunner, + "async function runGatewayUpdate(opts = {}) {", + "//#endregion", + 8 * 1024, + "update runner entry" + ); + assertRequiredMarkers(updateRunnerEntry, "update runner entry", [ + "const timeoutMs = opts.timeoutMs ?? 12e5", + "return await runGitUpdate({", + "return await runGlobalUpdate({", + 'status: "skipped"', + 'reason: "not-git-install"', + ]); + + const updateSentinel = artifactByRole(artifacts, "update-sentinel").contents; + const sentinelPayload = boundedSourceRegion( + updateSentinel, + "function buildUpdateRestartSentinelPayload(params) {", + "//#endregion", + 8 * 1024, + "update restart sentinel" + ); + assertRequiredMarkers(sentinelPayload, "update restart sentinel", [ + 'kind: "update"', + "status: result.status", + "message: meta.note ?? null", + "doctorHint: formatDoctorNonInteractiveHint()", + "root: result.root", + "handoffId: meta.handoffId", + "before: result.before ?? null", + "after: result.after ?? null", + "steps: result.steps.map((step) => ({", + "command: step.command", + "cwd: step.cwd", + "stdoutTail: step.stdoutTail ?? null", + "stderrTail: step.stderrTail ?? null", + ]); + + return { + domain: "operations", + methodAccess: [ + { + controlPlaneWrite: false, + lane: "one-shot-admin", + method: "sessions.cleanup", + scope: "operator.admin", + }, + { + controlPlaneWrite: true, + lane: "one-shot-admin", + method: "update.run", + scope: "operator.admin", + }, + ], + methods: ["sessions.cleanup", "update.run"], + schemaVersion: 1, + sessionsCleanup: { + handlerValidatesParams: true, + method: "sessions.cleanup", + mutation: { + diskBudgetEnforcedAfterEntryMaintenance: true, + entryStateRecheckedBeforeRemoval: true, + unreferencedArtifactsPrunedOutsideWarnMode: true, + usesSqliteLifecycleMutation: true, + }, + outcome: { + automaticReplaySafe: false, + handlerTimeoutParameter: false, + idempotencyParameter: false, + postDispatchTransportTimeout: "outcome-unknown", + }, + preservation: { + activeKeyAndParentsPreserved: true, + activeWorkAdmissionsPreserved: true, + archivedEntriesPreserved: true, + groupChannelAndThreadEntriesPreserved: true, + modelSelectionLockedEntriesPreserved: true, + primarySessionsPreserved: true, + registeredRuntimeKeysPreserved: true, + }, + request: { + acceptedParams: [ + "activeKey", + "agent", + "allAgents", + "enforce", + "fixDmScope", + "fixMissing", + ], + closedObject: true, + requiredParams: [], + }, + response: { + appliedStoreFields: [ + "agentId", + "storePath", + "mode", + "dryRun", + "beforeCount", + "afterCount", + "missing", + "dmScopeRetired", + "modelRunPruned", + "pruned", + "capped", + "unreferencedArtifacts", + "diskBudget", + "wouldMutate", + "applied", + "appliedCount", + ], + diskBudgetFields: [ + "totalBytesBefore", + "totalBytesAfter", + "removedFiles", + "removedEntries", + "freedBytes", + "maxBytes", + "highWaterBytes", + "overBudget", + ], + formattedUpstreamErrorMustBeSanitized: true, + multiStoreFields: ["allAgents", "mode", "dryRun", "stores"], + sensitivePaths: ["storePath", "stores[].storePath"], + unreferencedArtifactFields: [ + "scannedFiles", + "removedFiles", + "freedBytes", + "olderThanMs", + ], + }, + semantics: { + activeKeyOptional: true, + enforceTrueOverridesConfiguredMode: true, + fixDmScopeDefaultsFalse: true, + fixMissingDefaultsFalse: true, + maintenanceConfigSource: "session.maintenance", + rpcAlwaysAppliesRatherThanDryRuns: true, + }, + }, + updateRun: { + handlerValidatesParams: true, + managedHandoff: { + activeFlightJoinedWithoutSecondSpawn: true, + detachedChild: true, + gitRequiresSupervisor: true, + globalInstallRequiresHandoff: true, + readyMarkerTimeoutMs: 30_000, + sensitiveTemporaryFilesRemoved: true, + startedHandoffCountsAsAccepted: true, + systemdMinimumRestartDelayMs: 2000, + systemdRequiresUnitContext: true, + systemdUsesUserScope: true, + }, + method: "update.run", + outcome: { + automaticReplaySafe: false, + handlerAbortSignal: false, + idempotencyParameter: false, + operationalErrorsUseRpcSuccess: true, + postDispatchTransportTimeout: "outcome-unknown", + }, + request: { + acceptedParams: [ + "continuationMessage", + "deliveryContext", + "note", + "restartDelayMs", + "sessionKey", + "timeoutMs", + ], + closedObject: true, + requiredParams: [], + restartDelayMinimumMs: 0, + timeoutMinimumMs: 1, + }, + response: { + okWhenHandoffStarted: true, + okWhenResultStatusOk: true, + resultStatuses: ["error", "ok", "skipped"], + sentinelPersistenceBestEffort: true, + sensitivePaths: [ + "handoff.command", + "handoff.message", + "handoff.pid", + "result.root", + "result.steps[].command", + "result.steps[].cwd", + "result.steps[].stderrTail", + "result.steps[].stdoutTail", + "restart.pid", + "sentinel.payload", + ], + topLevelFields: ["ok", "result", "handoff", "restart", "sentinel"], + }, + restart: { + directSuccessSchedulesSigusr1: true, + managedSystemdSkipsCooldownAndDeferral: true, + packageSwapSkipsCooldownAndDeferral: true, + }, + timeout: { + defaultRunnerPerStepMs: 1_200_000, + handlerFloorMs: 1000, + perStepRatherThanWholeOperation: true, + }, + }, + }; +} + function assertPhase4CronSemantics( artifacts: readonly LoadedSourceArtifact[] ): SourceAuditResult["cron"]["adapter"] { @@ -5669,6 +6320,7 @@ export async function auditInstalledOpenClaw( const taskPromptChars = assertPlanCompanionAndTasks(artifacts); const tasksAdapter = assertPhase4TaskAdapterSemantics(artifacts); const sessionsAdapter = assertPhase4SessionsSemantics(artifacts); + const operations = assertOpenClawOperationsSemantics(artifacts); const cronAdapter = assertPhase4CronSemantics(artifacts); const settings = assertSettingsSemantics(artifacts); @@ -5809,6 +6461,7 @@ export async function auditInstalledOpenClaw( schemaVersion: 1, sessionScopedEvents, }, + operations, sessions: { adapter: sessionsAdapter, companion: { diff --git a/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts b/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts index f1eb5752f..79ef6e8e0 100644 --- a/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts +++ b/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts @@ -902,6 +902,197 @@ export const sessionsFixtureSchema = v.strictObject({ plan: planSchema, }); +const operationsMethodAccessSchema = v.tuple([ + v.strictObject({ + controlPlaneWrite: v.literal(false), + lane: v.literal("one-shot-admin"), + method: v.literal("sessions.cleanup"), + scope: v.literal("operator.admin"), + }), + v.strictObject({ + controlPlaneWrite: v.literal(true), + lane: v.literal("one-shot-admin"), + method: v.literal("update.run"), + scope: v.literal("operator.admin"), + }), +]); + +/** Exact installed-OpenClaw privileged operation and RPC source facts. */ +export const operationsFixtureSchema = v.strictObject({ + domain: v.literal("operations"), + methodAccess: operationsMethodAccessSchema, + methods: v.tuple([v.literal("sessions.cleanup"), v.literal("update.run")]), + schemaVersion: fixtureSchemaVersion, + sessionsCleanup: v.strictObject({ + handlerValidatesParams: v.literal(true), + method: v.literal("sessions.cleanup"), + mutation: v.strictObject({ + diskBudgetEnforcedAfterEntryMaintenance: v.literal(true), + entryStateRecheckedBeforeRemoval: v.literal(true), + unreferencedArtifactsPrunedOutsideWarnMode: v.literal(true), + usesSqliteLifecycleMutation: v.literal(true), + }), + outcome: v.strictObject({ + automaticReplaySafe: v.literal(false), + handlerTimeoutParameter: v.literal(false), + idempotencyParameter: v.literal(false), + postDispatchTransportTimeout: v.literal("outcome-unknown"), + }), + preservation: v.strictObject({ + activeKeyAndParentsPreserved: v.literal(true), + activeWorkAdmissionsPreserved: v.literal(true), + archivedEntriesPreserved: v.literal(true), + groupChannelAndThreadEntriesPreserved: v.literal(true), + modelSelectionLockedEntriesPreserved: v.literal(true), + primarySessionsPreserved: v.literal(true), + registeredRuntimeKeysPreserved: v.literal(true), + }), + request: v.strictObject({ + acceptedParams: v.tuple([ + v.literal("activeKey"), + v.literal("agent"), + v.literal("allAgents"), + v.literal("enforce"), + v.literal("fixDmScope"), + v.literal("fixMissing"), + ]), + closedObject: v.literal(true), + requiredParams: v.tuple([]), + }), + response: v.strictObject({ + appliedStoreFields: v.tuple([ + v.literal("agentId"), + v.literal("storePath"), + v.literal("mode"), + v.literal("dryRun"), + v.literal("beforeCount"), + v.literal("afterCount"), + v.literal("missing"), + v.literal("dmScopeRetired"), + v.literal("modelRunPruned"), + v.literal("pruned"), + v.literal("capped"), + v.literal("unreferencedArtifacts"), + v.literal("diskBudget"), + v.literal("wouldMutate"), + v.literal("applied"), + v.literal("appliedCount"), + ]), + diskBudgetFields: v.tuple([ + v.literal("totalBytesBefore"), + v.literal("totalBytesAfter"), + v.literal("removedFiles"), + v.literal("removedEntries"), + v.literal("freedBytes"), + v.literal("maxBytes"), + v.literal("highWaterBytes"), + v.literal("overBudget"), + ]), + formattedUpstreamErrorMustBeSanitized: v.literal(true), + multiStoreFields: v.tuple([ + v.literal("allAgents"), + v.literal("mode"), + v.literal("dryRun"), + v.literal("stores"), + ]), + sensitivePaths: v.tuple([ + v.literal("storePath"), + v.literal("stores[].storePath"), + ]), + unreferencedArtifactFields: v.tuple([ + v.literal("scannedFiles"), + v.literal("removedFiles"), + v.literal("freedBytes"), + v.literal("olderThanMs"), + ]), + }), + semantics: v.strictObject({ + activeKeyOptional: v.literal(true), + enforceTrueOverridesConfiguredMode: v.literal(true), + fixDmScopeDefaultsFalse: v.literal(true), + fixMissingDefaultsFalse: v.literal(true), + maintenanceConfigSource: v.literal("session.maintenance"), + rpcAlwaysAppliesRatherThanDryRuns: v.literal(true), + }), + }), + updateRun: v.strictObject({ + handlerValidatesParams: v.literal(true), + managedHandoff: v.strictObject({ + activeFlightJoinedWithoutSecondSpawn: v.literal(true), + detachedChild: v.literal(true), + gitRequiresSupervisor: v.literal(true), + globalInstallRequiresHandoff: v.literal(true), + readyMarkerTimeoutMs: v.literal(30_000), + sensitiveTemporaryFilesRemoved: v.literal(true), + startedHandoffCountsAsAccepted: v.literal(true), + systemdMinimumRestartDelayMs: v.literal(2000), + systemdRequiresUnitContext: v.literal(true), + systemdUsesUserScope: v.literal(true), + }), + method: v.literal("update.run"), + outcome: v.strictObject({ + automaticReplaySafe: v.literal(false), + handlerAbortSignal: v.literal(false), + idempotencyParameter: v.literal(false), + operationalErrorsUseRpcSuccess: v.literal(true), + postDispatchTransportTimeout: v.literal("outcome-unknown"), + }), + request: v.strictObject({ + acceptedParams: v.tuple([ + v.literal("continuationMessage"), + v.literal("deliveryContext"), + v.literal("note"), + v.literal("restartDelayMs"), + v.literal("sessionKey"), + v.literal("timeoutMs"), + ]), + closedObject: v.literal(true), + requiredParams: v.tuple([]), + restartDelayMinimumMs: v.literal(0), + timeoutMinimumMs: v.literal(1), + }), + response: v.strictObject({ + okWhenHandoffStarted: v.literal(true), + okWhenResultStatusOk: v.literal(true), + resultStatuses: v.tuple([ + v.literal("error"), + v.literal("ok"), + v.literal("skipped"), + ]), + sentinelPersistenceBestEffort: v.literal(true), + sensitivePaths: v.tuple([ + v.literal("handoff.command"), + v.literal("handoff.message"), + v.literal("handoff.pid"), + v.literal("result.root"), + v.literal("result.steps[].command"), + v.literal("result.steps[].cwd"), + v.literal("result.steps[].stderrTail"), + v.literal("result.steps[].stdoutTail"), + v.literal("restart.pid"), + v.literal("sentinel.payload"), + ]), + topLevelFields: v.tuple([ + v.literal("ok"), + v.literal("result"), + v.literal("handoff"), + v.literal("restart"), + v.literal("sentinel"), + ]), + }), + restart: v.strictObject({ + directSuccessSchedulesSigusr1: v.literal(true), + managedSystemdSkipsCooldownAndDeferral: v.literal(true), + packageSwapSkipsCooldownAndDeferral: v.literal(true), + }), + timeout: v.strictObject({ + defaultRunnerPerStepMs: v.literal(1_200_000), + handlerFloorMs: v.literal(1000), + perStepRatherThanWholeOperation: v.literal(true), + }), + }), +}); + export const agentsFixtureSchema = v.strictObject({ ...domainFixtureEntries, domain: v.literal("agents"), @@ -2006,12 +2197,15 @@ export const sourceArtifactSchema = v.strictObject({ "protocol-version", "provider-model-id-normalization", "runtime-subscriptions", + "session-accessor-sqlite-maintenance", + "session-cleanup-service", "session-companion-rpc", "session-companion-runtime", "session-change-event", "session-event-payload", "session-lifecycle", "session-list-projection", + "session-maintenance-policy", "session-operation-event", "session-reset-policy", "session-reset-service", @@ -2031,6 +2225,10 @@ export const sourceArtifactSchema = v.strictObject({ "tasks-handlers", "tool-policy-normalization", "transcript-media-persistence", + "update-handlers", + "update-managed-handoff", + "update-runner", + "update-sentinel", "web-fetch-runtime", "web-search-runtime", ]), @@ -2039,7 +2237,7 @@ export const sourceArtifactSchema = v.strictObject({ const sourceArtifactsSchema = v.pipe( v.array(sourceArtifactSchema), - v.length(83), + v.length(90), v.check( (artifacts) => isSortedAndUnique(artifacts.map((artifact) => artifact.role)), "Source artifact roles must be sorted and unique" @@ -2057,6 +2255,7 @@ const fixtureManifestEntrySchema = v.strictObject({ "chat.json", "cron.json", "gateway.json", + "operations.json", "sessions.json", "settings.json", "tasks.json", @@ -2067,7 +2266,7 @@ const fixtureManifestEntrySchema = v.strictObject({ export const fixtureManifestSchema = v.strictObject({ components: v.pipe( v.array(fixtureManifestEntrySchema), - v.length(7), + v.length(8), v.check( (components) => isSortedAndUnique(components.map((component) => component.file)), @@ -2092,6 +2291,7 @@ export const sourceAuditResultSchema = v.pipe( chat: chatFixtureSchema, cron: cronFixtureSchema, gateway: gatewayFixtureSchema, + operations: operationsFixtureSchema, sessions: sessionsFixtureSchema, settings: settingsFixtureSchema, tasks: tasksFixtureSchema, @@ -2109,6 +2309,7 @@ export type ChatFixture = v.InferOutput; export type CronFixture = v.InferOutput; export type FixtureManifest = v.InferOutput; export type GatewayFixture = v.InferOutput; +export type OperationsFixture = v.InferOutput; export type SessionsFixture = v.InferOutput; export type SettingsFixture = v.InferOutput; export type TasksFixture = v.InferOutput; diff --git a/greenfield/scripts/documentation/artifacts.test.ts b/greenfield/scripts/documentation/artifacts.test.ts index 7e079a98e..0de9f590b 100644 --- a/greenfield/scripts/documentation/artifacts.test.ts +++ b/greenfield/scripts/documentation/artifacts.test.ts @@ -113,6 +113,12 @@ describe("generated contract documentation", () => { expect(procedureDocumentation).toContain( "| `terminal.prepareSession` | mutation | terminal | Authenticated browser session: terminal:write; MFA enrollment required; recent MFA when enabled |" ); + expect(procedureDocumentation).toContain( + "| `serviceActions.getStatus` | query | service-actions | Authenticated browser session: service-actions:read |" + ); + expect(procedureDocumentation).toContain( + "| `serviceActions.request` | mutation | service-actions | Authenticated browser session: service-actions:write; MFA enrollment required; recent MFA when enabled |" + ); expect(procedureDocumentation).toContain( "| None | None | Returns bootstrap, pending MFA" ); @@ -188,6 +194,14 @@ describe("generated contract documentation", () => { expect(first.has("schemas/terminal.prepareSession.output.schema.json")).toBe( true ); + for (const artifact of [ + "schemas/serviceActions.getStatus.input.schema.json", + "schemas/serviceActions.getStatus.output.schema.json", + "schemas/serviceActions.request.input.schema.json", + "schemas/serviceActions.request.output.schema.json", + ]) { + expect(first.has(artifact)).toBe(true); + } expect(first.get("schemas/files.list.output.schema.json")).toContain( "rejects traversal names and path separators" ); diff --git a/greenfield/scripts/documentation/jsonSchema.test.ts b/greenfield/scripts/documentation/jsonSchema.test.ts index ec76e1092..583038f8b 100644 --- a/greenfield/scripts/documentation/jsonSchema.test.ts +++ b/greenfield/scripts/documentation/jsonSchema.test.ts @@ -49,6 +49,7 @@ import { openClawTaskGetOutputSchema } from "../../src/contracts/openClawTasks.t import { listReportsResultSchema } from "../../src/contracts/reports.ts"; import { applicationCapabilityListSchema } from "../../src/contracts/security.ts"; import { listSecurityAuditEventsResultSchema } from "../../src/contracts/securityAudit.ts"; +import { getServiceActionsStatusResultSchema } from "../../src/contracts/serviceActions.ts"; import { taskDetailSchema, taskLabelInputSchema, @@ -73,6 +74,24 @@ import { convertContractSchema } from "./jsonSchema.ts"; const parseHexadecimalCodePoint = (value: string): number => Number.parseInt(value, 16); describe("contract JSON Schema conversion", () => { + test("documents the fixed canonical Service Actions inventory", () => { + expect( + convertContractSchema( + getServiceActionsStatusResultSchema, + "test.serviceActionsStatus", + "output" + ) + ).toMatchObject({ + properties: { + actions: { + $comment: + "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered.", + maxItems: 4, + }, + }, + }); + }); + test("documents ASCII-bounded Unicode-mode regular expressions", () => { const schema = v.pipe(v.string(), v.regex(/^[a-z]+$/u)); @@ -283,13 +302,15 @@ describe("contract JSON Schema conversion", () => { "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", "terminal:write", ], }, - maxItems: 27, + maxItems: 29, type: "array", uniqueItems: true, }); diff --git a/greenfield/scripts/documentation/jsonSchema.ts b/greenfield/scripts/documentation/jsonSchema.ts index 195edd25d..f2b5acd6b 100644 --- a/greenfield/scripts/documentation/jsonSchema.ts +++ b/greenfield/scripts/documentation/jsonSchema.ts @@ -217,6 +217,7 @@ import { securityAuditEventsHaveStableOrder, securityAuditPageCursorIsConsistent, } from "../../src/contracts/securityAudit.ts"; +import { serviceActionStatusesAreCanonical } from "../../src/contracts/serviceActions.ts"; import { systemHealthDiagnosticsGatewayIsConsistent, systemHealthDiagnosticsIsConsistent, @@ -279,6 +280,10 @@ const controlSafeTextJsonSchemaPattern = `^(?![\\s\\S]*(?:${controlSafeTextExclu const noNulJsonSchemaPattern = String.raw`^[^\u0000]*$`; const runtimeCheckComments = new Map([ + [ + serviceActionStatusesAreCanonical, + "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered.", + ], [ workspaceFileContentTicketIsConsistent, "Live Valibot validation additionally requires truncated workspace-file representations to carry a larger source size and stay within the bounded text-prefix budget, while full representations omit source-size metadata.", diff --git a/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts b/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts index c17b5f9a1..7df3f9842 100644 --- a/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts +++ b/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts @@ -56,6 +56,7 @@ const reviewedApplicationServerTargets: ReadonlyMap< "src/server/domains/moltbook/provider.ts", "src/server/platform/configuration/workerConfiguration.ts", "src/server/platform/filesystem/projectLayout.ts", + "src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts", "src/server/platform/gateway/persistentGatewayTransport.ts", "src/server/platform/observability/projectFileLogSink.ts", "src/server/platform/observability/structuredLogger.ts", diff --git a/greenfield/src/app/dashboardServer.test.ts b/greenfield/src/app/dashboardServer.test.ts index c14e04d43..2e5ad6ee8 100644 --- a/greenfield/src/app/dashboardServer.test.ts +++ b/greenfield/src/app/dashboardServer.test.ts @@ -1241,6 +1241,7 @@ describe("Dashboard OpenClaw operations composition", () => { await jobRepository.registerWorker({ ...noJobSideEffects, worker: { + actionKeysJson: '["openclaw.gateway.restart"]', capacity: 1, drainingAt: null, heartbeatAt: authenticationTestNow, diff --git a/greenfield/src/app/dashboardServer.ts b/greenfield/src/app/dashboardServer.ts index b6d7f46e3..b497e2f03 100644 --- a/greenfield/src/app/dashboardServer.ts +++ b/greenfield/src/app/dashboardServer.ts @@ -8,6 +8,7 @@ import { chatHistoryPageMaximum, chatHistoryRetainedPageMaximum, } from "../contracts/chatModel.ts"; +import { serviceActionIds } from "../contracts/serviceActions.ts"; import { createAgentRepository } from "../server/domains/agents/repository.ts"; import { createAgentService } from "../server/domains/agents/service.ts"; import { @@ -44,11 +45,18 @@ import { createGatewaySessionsService, type GatewaySessionsService, } from "../server/domains/gatewaySessions/service.ts"; +import { + hostSystemRestartJobActionDefinition, + hostSystemUpdateJobActionDefinition, + openClawInstallationUpdateJobActionDefinition, + openClawSessionsCleanupJobActionDefinition, +} from "../server/domains/jobs/actionRegistry.ts"; import { createJobRepository } from "../server/domains/jobs/repository.ts"; import { createJobService, reconcileJobSchedules, } from "../server/domains/jobs/service.ts"; +import { createServiceActionQueue } from "../server/domains/jobs/serviceActionQueue.ts"; import { createMonitoringCatalogService } from "../server/domains/monitoring/catalogService.ts"; import { createMonitoringRepository } from "../server/domains/monitoring/repository.ts"; import { createMonitoringService } from "../server/domains/monitoring/service.ts"; @@ -111,6 +119,9 @@ import { createRequestAuthenticator } from "../server/domains/security/requestAu import { createRequestAuthenticationRepository } from "../server/domains/security/requestAuthenticationRepository.ts"; import { createSecurityAuditLifecycleService } from "../server/domains/security/securityAuditLifecycle.ts"; import { createSecurityAuditLifecycleRepository } from "../server/domains/security/securityAuditLifecycleRepository.ts"; +import { createSqliteServiceActionAuditWriter } from "../server/domains/serviceActions/operationAudit.ts"; +import { createServiceActionsService } from "../server/domains/serviceActions/service.ts"; +import { createSqliteServiceActionStatusReader } from "../server/domains/serviceActions/statusReader.ts"; import { createSystemHealthDiagnosticsService } from "../server/domains/system/healthDiagnosticsService.ts"; import { createTaskRepository } from "../server/domains/tasks/repository.ts"; import { createTaskService } from "../server/domains/tasks/service.ts"; @@ -223,6 +234,7 @@ export interface DashboardServerOptions extends Omit< | "openClawSettingsService" | "openClawTasksService" | "securityAuditLifecycle" + | "serviceActionsService" | "systemHealthDiagnosticsService" | "taskService" | "terminalService" @@ -747,6 +759,62 @@ export async function createDashboardServer( repository: jobRepository, wakeEventPump, }); + const serviceActionDefinitions = Object.freeze({ + "openclaw-cleanup": openClawSessionsCleanupJobActionDefinition, + "openclaw-update": openClawInstallationUpdateJobActionDefinition, + "system-restart": hostSystemRestartJobActionDefinition, + "system-update": hostSystemUpdateJobActionDefinition, + }); + const serviceActionsService = createServiceActionsService({ + auditWriter: createSqliteServiceActionAuditWriter({ + ...(domainNow === undefined ? {} : { clock: domainNow }), + database, + writeAdmission: databaseRuntime, + }), + ...(domainNow === undefined ? {} : { nowMs: () => domainNow().getTime() }), + onAuditSettlementFailure: ({ actionId, cause, settlement }) => + options.applicationRuntime.logger.error({ + component: "service-actions-audit", + event: "service_actions.audit_settlement.failed", + failure: cause, + fields: { + actionId, + kind: "service-actions-audit-settlement", + settlement, + }, + outcome: "server-error", + }), + queue: createServiceActionQueue({ + definitions: serviceActionDefinitions, + ...(domainNow === undefined + ? {} + : { nowMs: () => domainNow().getTime() }), + repository: jobRepository, + wakeEventPump, + }), + statusReader: + options.verifiedReleaseId === undefined + ? Object.freeze({ + read(signal?: AbortSignal) { + signal?.throwIfAborted(); + return Promise.resolve( + serviceActionIds.map((id) => + Object.freeze({ + availability: "unavailable" as const, + id, + }) + ) + ); + }, + }) + : createSqliteServiceActionStatusReader({ + expectedReleaseId: options.verifiedReleaseId, + ...(domainNow === undefined + ? {} + : { nowMs: () => domainNow().getTime() }), + repository: jobRepository, + }), + }); const logsService = options.dashboardLogsRoot === undefined || options.dashboardLogMaintenanceRoot === undefined @@ -1354,6 +1422,7 @@ export async function createDashboardServer( port: options.port, readiness: options.readiness, securityAuditLifecycle, + serviceActionsService, systemHealthDiagnosticsService, taskService, ...(terminalComposition === undefined diff --git a/greenfield/src/app/developmentWorker.ts b/greenfield/src/app/developmentWorker.ts index 3db4dfd8a..8bddb5044 100644 --- a/greenfield/src/app/developmentWorker.ts +++ b/greenfield/src/app/developmentWorker.ts @@ -36,17 +36,22 @@ export async function runDevelopmentWorkerProcess( const defaults = createDefaultDashboardWorkerProcessDependencies(); const dependencies = Object.freeze({ ...defaults, + createHostOperations: () => void 0, createLogMaintenanceExecutor: createDevelopmentLogMaintenanceExecutor, + createOpenClawGatewayLifecycle: () => void 0, + createOpenClawServiceActions: () => void 0, createRuntime: ( layout, source, _logger, gatewayTransport, openClawGateway, + openClawServiceActions, workspaceRoot, openClawRoot, logMaintenance, - moltbook + moltbook, + hostOperations ) => { const writer = createDescriptorWorkspaceFileStructuralWriter({ roots: [workspaceRoot, openClawRoot], @@ -61,7 +66,11 @@ export async function runDevelopmentWorkerProcess( }, logMaintenance, moltbook, - openClawGateway, + ...(openClawGateway === undefined ? {} : { openClawGateway }), + ...(openClawServiceActions === undefined + ? {} + : { openClawServiceActions }), + ...(hostOperations === undefined ? {} : { hostOperations }), persistentGatewayTransport: gatewayTransport, pid: process.pid, releaseId: source.manifest.source.commitSha, diff --git a/greenfield/src/app/server.ts b/greenfield/src/app/server.ts index 6972cfc34..c0e070d3d 100644 --- a/greenfield/src/app/server.ts +++ b/greenfield/src/app/server.ts @@ -24,6 +24,7 @@ import type { AutomationSecurityLifecycleService } from "../server/domains/secur import type { MfaAccountLifecycleService } from "../server/domains/security/mfa/accountLifecycle.ts"; import type { MfaLoginLifecycleService } from "../server/domains/security/mfa/loginLifecycle.ts"; import type { SecurityAuditLifecycleService } from "../server/domains/security/securityAuditLifecycle.ts"; +import type { ServiceActionsService } from "../server/domains/serviceActions/service.ts"; import type { SystemHealthDiagnosticsService } from "../server/domains/system/healthDiagnosticsService.ts"; import type { TaskService } from "../server/domains/tasks/service.ts"; import type { TerminalService } from "../server/domains/terminal/service.ts"; @@ -223,6 +224,7 @@ export interface ServerOptions { readonly port: number; readonly readiness: ReadinessController; readonly securityAuditLifecycle: SecurityAuditLifecycleService; + readonly serviceActionsService: ServiceActionsService; readonly systemHealthDiagnosticsService: SystemHealthDiagnosticsService; readonly taskService: TaskService["Service"]; readonly terminalService?: TerminalService; @@ -280,6 +282,7 @@ export async function createServer(options: ServerOptions): Promise { rejectsBatch: false, requestBodyMaximumBytes: trpcRequestBodyMaximumBytes, }); + expect(policy("/trpc/serviceActions.request?batch=1")).toEqual({ + rejectsBatch: true, + requestBodyMaximumBytes: trpcRequestBodyMaximumBytes, + }); expect(policy("/trpc/tasks.create")).toEqual({ rejectsBatch: false, requestBodyMaximumBytes: taskContentRequestBodyMaximumBytes, diff --git a/greenfield/src/app/worker.test.ts b/greenfield/src/app/worker.test.ts index 30a449dd5..96bb185c4 100644 --- a/greenfield/src/app/worker.test.ts +++ b/greenfield/src/app/worker.test.ts @@ -16,6 +16,7 @@ import { import type { ManagedLogManifest } from "../worker/logs/managedLogManifest.ts"; import type { DashboardWorkerRuntime } from "../worker/runtime.ts"; import { + createDefaultDashboardWorkerProcessDependencies, createWorkerLogMaintenanceExecutor, type DashboardWorkerProcessDependencies, runDashboardWorkerProcess, @@ -92,6 +93,8 @@ function processFixture( }), } satisfies ProjectFileLogDestination); const gatewayTransport = Object.freeze({ + requestOpenClawServiceAction: () => + Promise.reject(new Error("OpenClaw operations are unavailable in fixture")), start() { events.push("gateway-start"); }, @@ -124,6 +127,10 @@ function processFixture( const openClawGateway = Object.freeze({ restart: () => Promise.resolve(), }); + const openClawServiceActions = Object.freeze({ + cleanupSessions: () => Promise.reject(new Error("fixture cleanup unavailable")), + updateInstallation: () => Promise.reject(new Error("fixture update unavailable")), + }); const runtime: DashboardWorkerRuntime = Object.freeze({ completion, dispose(forceSignal?: AbortSignal) { @@ -190,21 +197,29 @@ function processFixture( expect(observedOpenClawRoot).toBe(openClawRoot); return openClawGateway; }, + createOpenClawServiceActions(observedGatewayTransport) { + expect(observedGatewayTransport).toBe(gatewayTransport); + return openClawServiceActions; + }, createRuntime( observedLayout, observedRelease, logger, observedGatewayTransport, observedOpenClawGateway, + observedOpenClawServiceActions, observedWorkspaceRoot, observedOpenClawRoot, - observedLogMaintenance + observedLogMaintenance, + _observedMoltbook, + observedHostOperations ) { expect(observedLayout).toBe(layout); expect(observedRelease).toBe(release); expect(logger).toBeDefined(); expect(observedGatewayTransport).toBe(gatewayTransport); expect(observedOpenClawGateway).toBe(openClawGateway); + expect(observedOpenClawServiceActions).toBe(openClawServiceActions); expect(observedWorkspaceRoot).toEqual({ id: "workspace", path: workspaceRoot, @@ -228,7 +243,9 @@ function processFixture( writable: true, }); expect(observedLogMaintenance).toBe(logMaintenance); + expect(observedHostOperations).toBeUndefined(); expect(Object.keys(observedGatewayTransport).toSorted()).toEqual([ + "requestOpenClawServiceAction", "start", "stop", "taskNotificationSender", @@ -360,6 +377,12 @@ const processOptions = Object.freeze({ }); describe("Dashboard worker process", () => { + test("does not compose shared-identity host-operation authority", () => { + expect( + createDefaultDashboardWorkerProcessDependencies().createHostOperations + ).toBeUndefined(); + }); + test("binds managed rotation state to protected project-local paths", () => { let observedManifest: ManagedLogManifest | undefined; const executor = createWorkerLogMaintenanceExecutor(layout, { diff --git a/greenfield/src/app/worker.ts b/greenfield/src/app/worker.ts index bf91beca3..b7e24cf37 100644 --- a/greenfield/src/app/worker.ts +++ b/greenfield/src/app/worker.ts @@ -17,6 +17,7 @@ import { type DashboardProjectLayout, resolveDashboardProjectLayout, } from "../server/platform/filesystem/projectLayout.ts"; +import { createPersistentGatewayOpenClawServiceActionsProvider } from "../server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts"; import { createPersistentGatewayTaskNotificationTransport, type PersistentGatewayTaskNotificationTransport, @@ -38,7 +39,9 @@ import { createProcessTerminationController, type ProcessTerminationController, } from "../server/platform/runtime/processSignals.ts"; +import type { FixedHostOperationsExecutionPort } from "../shared/hostOperations.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../shared/openClawGatewayLifecycle.ts"; +import type { OpenClawServiceActionsExecutionPort } from "../shared/openClawServiceActions.ts"; import { createDescriptorWorkspaceFileStructuralWriter, type WorkerWorkspaceFileRootConfiguration, @@ -93,19 +96,25 @@ export interface DashboardWorkerProcessDependencies { readonly createLogMaintenanceExecutor: ( layout: DashboardProjectLayout ) => LogMaintenanceExecutor; + readonly createHostOperations?: () => FixedHostOperationsExecutionPort | undefined; readonly createOpenClawGatewayLifecycle: ( openClawRoot: string - ) => OpenClawGatewayLifecycleExecutionPort; + ) => OpenClawGatewayLifecycleExecutionPort | undefined; + readonly createOpenClawServiceActions: ( + transport: PersistentGatewayTaskNotificationTransport + ) => OpenClawServiceActionsExecutionPort | undefined; readonly createRuntime: ( layout: DashboardProjectLayout, release: RuntimeRelease, logger: StructuredLogger, persistentGatewayTransport: PersistentGatewayTaskNotificationTransport, - openClawGateway: OpenClawGatewayLifecycleExecutionPort, + openClawGateway: OpenClawGatewayLifecycleExecutionPort | undefined, + openClawServiceActions: OpenClawServiceActionsExecutionPort | undefined, workspaceRoot: WorkerWorkspaceFileRootConfiguration, openClawRoot: WorkerWorkspaceFileRootConfiguration, logMaintenance: LogMaintenanceExecutor, - moltbook: MoltbookDashboardCollector + moltbook: MoltbookDashboardCollector, + hostOperations: FixedHostOperationsExecutionPort | undefined ) => DashboardWorkerRuntime; readonly createTerminationController: () => ProcessTerminationController; readonly loadRelease: ( @@ -180,16 +189,19 @@ const defaultDependencies = Object.freeze({ createLogMaintenanceExecutor: createWorkerLogMaintenanceExecutor, createOpenClawGatewayLifecycle: (openClawRoot) => createFixedOpenClawGatewayLifecycle({ openClawRoot }), + createOpenClawServiceActions: createPersistentGatewayOpenClawServiceActionsProvider, createRuntime: ( layout, release, _logger, gatewayTransport, openClawGateway, + openClawServiceActions, workspaceRoot, openClawRoot, logMaintenance, - moltbook + moltbook, + hostOperations ) => { const writer = createDescriptorWorkspaceFileStructuralWriter({ roots: [workspaceRoot, openClawRoot], @@ -204,7 +216,9 @@ const defaultDependencies = Object.freeze({ }, logMaintenance, moltbook, - openClawGateway, + ...(openClawGateway === undefined ? {} : { openClawGateway }), + ...(openClawServiceActions === undefined ? {} : { openClawServiceActions }), + ...(hostOperations === undefined ? {} : { hostOperations }), persistentGatewayTransport: gatewayTransport, pid: process.pid, releaseId: release.manifest.source.commitSha, @@ -313,6 +327,9 @@ export async function runDashboardWorkerProcess( const openClawGateway = dependencies.createOpenClawGatewayLifecycle( openClawRoot.path ); + const openClawServiceActions = + dependencies.createOpenClawServiceActions(gatewayTransport); + const hostOperations = dependencies.createHostOperations?.(); const moltbook = createMoltbookDashboardCollector({ agentName: configuration.moltbookAgentName, apiKey: configuration.moltbookApiKey, @@ -323,10 +340,12 @@ export async function runDashboardWorkerProcess( logger, gatewayTransport, openClawGateway, + openClawServiceActions, workspaceRoot, openClawRoot, logMaintenance, - moltbook + moltbook, + hostOperations ); const runtimeCompletion = runtime.completion.then( () => ({ kind: "stopped" as const }), diff --git a/greenfield/src/browser/api/trpcClient.ts b/greenfield/src/browser/api/trpcClient.ts index 84713ee17..f5a827206 100644 --- a/greenfield/src/browser/api/trpcClient.ts +++ b/greenfield/src/browser/api/trpcClient.ts @@ -148,6 +148,10 @@ async function procedureContractsFor( const module = await import("../../contracts/securityAudit.ts"); return module.securityAuditProcedureContracts; } + case "serviceActions": { + const module = await import("../../contracts/serviceActions.ts"); + return module.serviceActionProcedureContracts; + } case "system": { const module = await import("../../contracts/system.ts"); return module.systemProcedureContracts; diff --git a/greenfield/src/browser/overview/OverviewRoute.test.tsx b/greenfield/src/browser/overview/OverviewRoute.test.tsx index 5c7697dca..fc84a6f24 100644 --- a/greenfield/src/browser/overview/OverviewRoute.test.tsx +++ b/greenfield/src/browser/overview/OverviewRoute.test.tsx @@ -24,6 +24,7 @@ import type { } from "../../contracts/monitoring.ts"; import type { ListNotificationsResult } from "../../contracts/notifications.ts"; import type { ListReportsResult } from "../../contracts/reports.ts"; +import type { GetServiceActionsStatusResult } from "../../contracts/serviceActions.ts"; import type { SystemMetrics } from "../../contracts/system.ts"; import type { TaskSummary } from "../../contracts/taskModel.ts"; import type { ListTasksResult } from "../../contracts/tasks.ts"; @@ -228,6 +229,16 @@ const jobRunPage = Object.freeze({ summary: overviewQueueSummary, } satisfies ListJobRunsResult); +const serviceActionsStatus = Object.freeze({ + actions: [ + { availability: "unavailable", id: "openclaw-cleanup" }, + { availability: "unavailable", id: "openclaw-update" }, + { availability: "unavailable", id: "system-restart" }, + { availability: "unavailable", id: "system-update" }, + ], + observedAtMs: timestampMs, +} satisfies GetServiceActionsStatusResult); + const overviewTask = Object.freeze({ assignee: "mira-2026", createdAtMs: timestampMs - 3000, @@ -417,6 +428,9 @@ class OverviewTransport implements DashboardTrpcTransport { case "reports.list": { return transportOutput(this.#reportOutputs, callIndex, path); } + case "serviceActions.getStatus": { + return Promise.resolve(serviceActionsStatus); + } case "system.metrics": { return transportOutput(this.#systemMetricsOutputs, callIndex, path); } @@ -541,10 +555,9 @@ describe("Dashboard operational overview foundation", () => { }) ).toBeTruthy(); expect(screen.getByText("Accepting new jobs")).toBeTruthy(); - expect(screen.getByRole("link", { name: "View Dashboard jobs" })).toHaveAttribute( - "href", - "/jobs" - ); + expect( + screen.getAllByRole("link", { name: "View Dashboard jobs" }) + ).not.toHaveLength(0); const jobSummaryCalls = transport.queryCalls.filter( ({ path }) => path === "jobs.listRuns" ); @@ -552,6 +565,12 @@ describe("Dashboard operational overview foundation", () => { for (const call of jobSummaryCalls) { expect(call).toEqual({ input: { limit: 1 }, path: "jobs.listRuns" }); } + expect( + await screen.findByRole("heading", { level: 2, name: "Service actions" }) + ).toBeTruthy(); + expect( + transport.queryCalls.filter(({ path }) => path === "serviceActions.getStatus") + ).toEqual([{ input: {}, path: "serviceActions.getStatus" }]); expect( await screen.findByRole("heading", { level: 2, diff --git a/greenfield/src/browser/overview/OverviewRoute.tsx b/greenfield/src/browser/overview/OverviewRoute.tsx index d8383ea27..b7e2de3e8 100644 --- a/greenfield/src/browser/overview/OverviewRoute.tsx +++ b/greenfield/src/browser/overview/OverviewRoute.tsx @@ -5,6 +5,7 @@ import { OverviewIncidentsSection } from "./OverviewIncidentsSection.tsx"; import { OverviewJobsSection } from "./OverviewJobsSection.tsx"; import { OverviewNotificationsSection } from "./OverviewNotificationsSection.tsx"; import { OverviewReportsSection } from "./OverviewReportsSection.tsx"; +import { OverviewServiceActionsSection } from "./OverviewServiceActionsSection.tsx"; import { OverviewTasksSection } from "./OverviewTasksSection.tsx"; import { SystemMetricsSection } from "./SystemMetricsSection.tsx"; @@ -29,6 +30,9 @@ export function OverviewRoute() {
+
+ +
diff --git a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx new file mode 100644 index 000000000..2edca8591 --- /dev/null +++ b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx @@ -0,0 +1,258 @@ +import { Wrench } from "lucide-react"; +import { useId, useState } from "react"; + +import type { JobRunSummary } from "../../contracts/jobModel.ts"; +import type { + GetServiceActionsStatusResult, + ServiceActionId, + ServiceActionStatus, +} from "../../contracts/serviceActions.ts"; +import { jobRunStateBadgeVariant, jobRunStateLabel } from "../jobs/jobRunPresentation.ts"; +import { formatDashboardDateTime } from "../lib/formatDateTime.ts"; +import { ActionLink } from "../ui/ActionLink.tsx"; +import { Alert } from "../ui/Alert.tsx"; +import { Badge } from "../ui/Badge.tsx"; +import { Button } from "../ui/Button.tsx"; +import { Card } from "../ui/Card.tsx"; +import { ConfirmModal } from "../ui/ConfirmModal.tsx"; +import { Heading } from "../ui/Heading.tsx"; +import { Icon } from "../ui/Icon.tsx"; +import { Text } from "../ui/Text.tsx"; +import { serviceActionPresentations } from "./serviceActionsOperations.ts"; + +interface RunObservationProps { + readonly label: string; + readonly run: JobRunSummary | undefined; +} + +function RunObservation({ label, run }: RunObservationProps) { + if (run === undefined) { + return ( +
+
{label}
+
None
+
+ ); + } + return ( +
+
{label}
+
+ + {jobRunStateLabel(run.state)} + + +
+
Run {run.id}
+
+ ); +} + +interface ServiceActionRowProps { + readonly action: ServiceActionStatus; + readonly globalBusy: boolean; + readonly onSelect: (actionId: ServiceActionId) => void; + readonly recoveryPending: boolean; +} + +function ServiceActionRow({ + action, + globalBusy, + onSelect, + recoveryPending, +}: ServiceActionRowProps) { + const presentation = serviceActionPresentations[action.id]; + const active = action.activeRun !== undefined; + const disabled = action.availability === "unavailable" || active || globalBusy; + return ( +
  • +
    +
    +
    + {presentation.actionLabel} + + {action.availability} + + {active && Active job} +
    + + {presentation.description} + + {action.availability === "unavailable" && ( + + No fresh worker currently advertises this fixed operation. + + )} + {recoveryPending && ( + + This browser session retains the request identity. An explicit + retry reuses it; review Dashboard jobs first. + + )} +
    + +
    +
    + + +
    +
    • + ); +} + +export interface OverviewServiceActionsCardProps { + readonly actions: GetServiceActionsStatusResult["actions"]; + readonly error?: string; + readonly notice?: string; + readonly observedAtMs: number; + readonly onClearError: () => void; + readonly onClearNotice: () => void; + readonly onRequest: (actionId: ServiceActionId, onConfirmed: () => void) => void; + readonly recoveryPending: (actionId: ServiceActionId) => boolean; + readonly requestActionId: ServiceActionId | undefined; + readonly requestBusy: boolean; +} + +/** + * Renders exact fixed service actions without command, payload, or provider details. + * @param properties Validated status rows and one session-bound request controller. + * @returns Fixed-action status rows and an accessible confirmation boundary. + */ +export function OverviewServiceActionsCard({ + actions, + error, + notice, + observedAtMs, + onClearError, + onClearNotice, + onRequest, + recoveryPending, + requestActionId, + requestBusy, +}: OverviewServiceActionsCardProps) { + const headingId = useId(); + const [selectedActionId, setSelectedActionId] = useState(); + const selectedAction = actions.find(({ id }) => id === selectedActionId); + const selectedPresentation = + selectedActionId === undefined + ? undefined + : serviceActionPresentations[selectedActionId]; + const selectedRecoveryPending = + selectedActionId === undefined ? false : recoveryPending(selectedActionId); + + return ( + +
    +
    + + + +
    + + Service actions + + + Queue four fixed, audited worker operations. Recent + multi-factor authentication is required; arbitrary commands + are not accepted. + +
    +
    + + View Dashboard jobs + +
    + + + + +
      + {actions.map((action) => ( + { + onClearError(); + onClearNotice(); + setSelectedActionId(actionId); + }} + recoveryPending={recoveryPending(action.id)} + /> + ))} +
    + + + Status observed{" "} + + . A queued response confirms only the durable Dashboard job run. + + + + {selectedPresentation?.warning} + {selectedRecoveryPending && ( + + This retry uses the retained request identity and does not + create a new intent. + + )} + + } + error={requestActionId === selectedActionId ? error : undefined} + onCancel={() => { + if (!requestBusy) setSelectedActionId(undefined); + }} + onConfirm={() => { + if (selectedActionId === undefined) return; + onRequest(selectedActionId, () => setSelectedActionId(undefined)); + }} + open={selectedActionId !== undefined} + title={selectedPresentation?.confirmationTitle ?? "Queue service action?"} + /> +     + ); +} diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx new file mode 100644 index 000000000..e2929e564 --- /dev/null +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx @@ -0,0 +1,463 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { QueryClientProvider } from "@tanstack/react-query"; +import { + createMemoryHistory, + createRootRoute, + createRoute, + createRouter, + RouterProvider, +} from "@tanstack/react-router"; + +import type { AuthStatus } from "../../contracts/auth.ts"; +import type { JobRunSummary } from "../../contracts/jobModel.ts"; +import type { + GetServiceActionsStatusResult, + RequestServiceActionResult, +} from "../../contracts/serviceActions.ts"; +import { createDashboardQueryClient } from "../api/queryClient.ts"; +import { + createDashboardTrpcClient, + type DashboardTrpcTransport, +} from "../api/trpcClient.ts"; +import { DashboardTrpcProvider } from "../api/trpcContext.tsx"; +import { authStatusQueryKey } from "../auth/authQueries.ts"; +import { OverviewServiceActionsSection } from "./OverviewServiceActionsSection.tsx"; + +const { render, screen, waitFor } = await import("@testing-library/react"); +const userEventModule = await import("@testing-library/user-event"); +const userEvent = userEventModule.default; + +const timestampMs = 1_800_000_000_000; +const authenticatedStatus = Object.freeze({ + session: { + authenticatedAtMs: timestampMs, + authMethod: "password", + createdAtMs: timestampMs, + expiresAtMs: timestampMs + 86_400_000, + id: "a".repeat(32), + isCurrent: true, + lastSeenAtMs: timestampMs, + userAgent: "Service actions browser test", + }, + state: "authenticated", + user: { + id: "019fd974-54a2-74dd-a64b-d4186f8d8828", + username: "operator", + }, +} satisfies AuthStatus); + +const queuedRun = Object.freeze({ + actionKey: "openclaw.sessions.cleanup", + attemptCount: 0, + attemptLimit: 1, + availableAtMs: timestampMs, + cancellationPolicy: "never", + displayName: "OpenClaw session cleanup", + eventCount: 1, + id: "019fe000-0000-7000-8000-000000000001", + priority: 0, + queuedAtMs: timestampMs, + resourceClass: "exclusive", + resourceKeys: ["host.mutation"], + retrySafe: false, + state: "queued", + stateVersion: 1, + timeoutMs: 600_000, + triggerType: "manual", + updatedAtMs: timestampMs, +} satisfies JobRunSummary); + +const runningRun = Object.freeze({ + ...queuedRun, + actionKey: "host.system.restart", + attemptCount: 1, + displayName: "System restart", + eventCount: 2, + firstStartedAtMs: timestampMs + 100, + id: "019fe000-0000-7000-8000-000000000002", + lastAttemptStartedAtMs: timestampMs + 100, + state: "running", + stateVersion: 2, + updatedAtMs: timestampMs + 100, +} satisfies JobRunSummary); + +const succeededRun = Object.freeze({ + ...queuedRun, + actionKey: "host.system.update", + attemptCount: 1, + displayName: "System update", + eventCount: 3, + finishedAtMs: timestampMs + 1000, + firstStartedAtMs: timestampMs + 100, + id: "019fe000-0000-7000-8000-000000000003", + lastAttemptStartedAtMs: timestampMs + 100, + state: "succeeded", + stateVersion: 3, + updatedAtMs: timestampMs + 1000, +} satisfies JobRunSummary); + +const actionStatus = Object.freeze({ + actions: [ + { + availability: "available", + id: "openclaw-cleanup", + }, + { + availability: "unavailable", + id: "openclaw-update", + }, + { + activeRun: runningRun, + availability: "available", + id: "system-restart", + }, + { + availability: "available", + id: "system-update", + latestRun: succeededRun, + }, + ], + observedAtMs: timestampMs + 2000, +} satisfies GetServiceActionsStatusResult); + +const allAvailableStatus = Object.freeze({ + actions: actionStatus.actions.map((action) => ({ + availability: "available" as const, + id: action.id, + })), + observedAtMs: timestampMs + 3000, +} satisfies GetServiceActionsStatusResult); + +const queuedResult = Object.freeze({ + actionId: "openclaw-cleanup", + jobRunId: queuedRun.id, + queued: true, +} satisfies RequestServiceActionResult); + +interface TransportCall { + readonly input: unknown; + readonly path: string; + readonly signal: AbortSignal | undefined; +} + +type QueryOutput = + | Error + | GetServiceActionsStatusResult + | Promise; +type MutationOutput = Error | RequestServiceActionResult; + +function outputAt(outputs: readonly unknown[], index: number): Promise { + const output = outputs[Math.min(index, outputs.length - 1)]; + if (output === undefined) return Promise.reject(new TypeError("Missing output")); + return output instanceof Error ? Promise.reject(output) : Promise.resolve(output); +} + +class ServiceActionsTransport implements DashboardTrpcTransport { + readonly mutationCalls: TransportCall[] = []; + readonly queryCalls: TransportCall[] = []; + readonly #mutationOutputs: readonly MutationOutput[]; + readonly #queryOutputs: readonly QueryOutput[]; + + constructor( + queryOutputs: readonly QueryOutput[], + mutationOutputs: readonly MutationOutput[] = [] + ) { + this.#queryOutputs = queryOutputs; + this.#mutationOutputs = mutationOutputs; + } + + mutation( + path: string, + input?: unknown, + options?: { readonly signal?: AbortSignal } + ): Promise { + const index = this.mutationCalls.length; + this.mutationCalls.push({ input, path, signal: options?.signal }); + if (path !== "serviceActions.request") { + return Promise.reject(new TypeError(`Unexpected mutation: ${path}`)); + } + return outputAt(this.#mutationOutputs, index); + } + + query( + path: string, + input?: unknown, + options?: { readonly signal?: AbortSignal } + ): Promise { + const index = this.queryCalls.length; + this.queryCalls.push({ input, path, signal: options?.signal }); + if (path !== "serviceActions.getStatus") { + return Promise.reject(new TypeError(`Unexpected query: ${path}`)); + } + return outputAt(this.#queryOutputs, index); + } +} + +interface SectionHarness { + readonly queryClient: ReturnType; + readonly transport: ServiceActionsTransport; + readonly view: ReturnType; +} + +const harnesses: SectionHarness[] = []; + +afterEach(() => { + for (const { queryClient, view } of harnesses.splice(0)) { + view.unmount(); + queryClient.clear(); + } + globalThis.sessionStorage.clear(); +}); + +function renderSection( + queryOutputs: readonly QueryOutput[], + mutationOutputs: readonly MutationOutput[] = [] +): SectionHarness { + const queryClient = createDashboardQueryClient(); + queryClient.setDefaultOptions({ + ...queryClient.getDefaultOptions(), + queries: { + ...queryClient.getDefaultOptions().queries, + retry: false, + }, + }); + queryClient.setQueryData(authStatusQueryKey, authenticatedStatus); + const transport = new ServiceActionsTransport(queryOutputs, mutationOutputs); + const trpcClient = createDashboardTrpcClient(transport); + const rootRoute = createRootRoute(); + const overviewRoute = createRoute({ + component: OverviewServiceActionsSection, + getParentRoute: () => rootRoute, + path: "/", + }); + const jobsRoute = createRoute({ + component: () => null, + getParentRoute: () => rootRoute, + path: "/jobs", + }); + const router = createRouter({ + history: createMemoryHistory({ initialEntries: ["/"] }), + routeTree: rootRoute.addChildren([overviewRoute, jobsRoute]), + }); + const view = render( + + + + + + ); + const harness = { queryClient, transport, view }; + harnesses.push(harness); + return harness; +} + +function operationOutcomeUnknownError(): Error { + return Object.assign(new Error("private lost acknowledgement"), { + data: { + code: "SERVICE_UNAVAILABLE", + reason: "operation_outcome_unknown", + }, + }); +} + +describe("OverviewServiceActionsSection", () => { + test("renders loading, exact fixed inventory, unavailable, active, and latest states", async () => { + const pending = Promise.withResolvers(); + const harness = renderSection([pending.promise]); + + expect(await screen.findByLabelText("Loading service actions…")).toBeTruthy(); + pending.resolve(actionStatus); + expect( + await screen.findByRole("heading", { level: 2, name: "Service actions" }) + ).toBeTruthy(); + expect(harness.transport.queryCalls[0]?.input).toEqual({}); + expect(screen.getByRole("heading", { name: "OpenClaw cleanup" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "OpenClaw update" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "System restart" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "System update" })).toBeTruthy(); + expect(screen.queryByText(/Gateway restart/iu)).toBeNull(); + expect(screen.queryByText(/system cleanup/iu)).toBeNull(); + expect(screen.queryByText(/terminal|command to run/iu)).toBeNull(); + expect( + screen.getByRole("button", { name: "Queue OpenClaw update" }) + ).toBeDisabled(); + expect( + screen.getByRole("button", { name: "Queue system restart" }) + ).toBeDisabled(); + expect( + screen.getByText("No fresh worker currently advertises this fixed operation.") + ).toBeTruthy(); + expect(screen.getByText("Active job")).toBeTruthy(); + expect(screen.getByText("succeeded")).toBeTruthy(); + expect(screen.getByText(succeededRun.id, { exact: false })).toBeTruthy(); + expect(screen.getByRole("link", { name: "View Dashboard jobs" })).toHaveAttribute( + "href", + "/jobs" + ); + }); + + test("recovers initial errors and retains validated status after a refresh failure", async () => { + const failure = new TypeError("private service-actions provider detail"); + const harness = renderSection([failure, actionStatus, failure]); + + expect( + await screen.findByRole("heading", { + level: 2, + name: "Service actions unavailable", + }) + ).toBeTruthy(); + expect(screen.queryByText(failure.message)).toBeNull(); + await userEvent.setup().click(screen.getByRole("button", { name: "Try again" })); + expect( + await screen.findByRole("heading", { level: 2, name: "Service actions" }) + ).toBeTruthy(); + + await harness.queryClient.refetchQueries({ + exact: true, + queryKey: ["service-actions", "status"], + type: "active", + }); + expect( + await screen.findByText("The request could not be completed. Try again.") + ).toBeTruthy(); + expect(screen.getByRole("heading", { name: "OpenClaw cleanup" })).toBeTruthy(); + expect(screen.queryByText(failure.message)).toBeNull(); + }); + + test("presents action-specific interruption, duration, and cleanup warnings", async () => { + renderSection([allAvailableStatus]); + const user = userEvent.setup(); + await screen.findByRole("heading", { name: "Service actions" }); + + await user.click(screen.getByRole("button", { name: "Queue system restart" })); + expect( + screen.getByText(/interrupts Dashboard, OpenClaw, and other host services/iu) + ).toBeTruthy(); + expect( + screen.getByText( + /accepted for durable processing, not that the host restarted/iu + ) + ).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + + await user.click(screen.getByRole("button", { name: "Queue system update" })); + expect(screen.getByText(/System updates can take a long time/iu)).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + + await user.click(screen.getByRole("button", { name: "Queue OpenClaw update" })); + expect(screen.getByText(/OpenClaw updates can take time/iu)).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + + await user.click(screen.getByRole("button", { name: "Queue OpenClaw cleanup" })); + expect( + screen.getByText(/source-owned OpenClaw session and artifact maintenance/iu) + ).toBeTruthy(); + }); + + test("retains one recovery key after unknown outcome and reuses it after remount", async () => { + const first = renderSection( + [allAvailableStatus], + [operationOutcomeUnknownError()] + ); + const user = userEvent.setup(); + await screen.findByRole("heading", { name: "Service actions" }); + await user.click(screen.getByRole("button", { name: "Queue OpenClaw cleanup" })); + expect( + screen.getByText(/source-owned OpenClaw session and artifact maintenance/iu) + ).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Queue cleanup" })); + const unknownOutcomeMessages = await screen.findAllByText( + /could not confirm whether the service action request was queued/iu + ); + expect(unknownOutcomeMessages.length).toBeGreaterThan(0); + expect(first.transport.mutationCalls).toHaveLength(1); + const firstInput = first.transport.mutationCalls[0]?.input as { + readonly idempotencyKey: string; + }; + expect(firstInput.idempotencyKey).toMatch(/^[0-9a-f]{32}$/u); + expect(globalThis.sessionStorage.length).toBe(1); + + first.view.unmount(); + first.queryClient.clear(); + const firstIndex = harnesses.indexOf(first); + if (firstIndex !== -1) harnesses.splice(firstIndex, 1); + const second = renderSection( + [allAvailableStatus, allAvailableStatus], + [queuedResult] + ); + expect( + await screen.findByRole("button", { + name: "Retry openclaw cleanup request", + }) + ).toBeTruthy(); + await user.click( + screen.getByRole("button", { name: "Retry openclaw cleanup request" }) + ); + expect( + screen.getByText(/retry uses the retained request identity/iu) + ).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Retry request" })); + + expect( + await screen.findByText( + `${"OpenClaw cleanup"} request queued. Dashboard job run: ${queuedRun.id}.` + ) + ).toBeTruthy(); + const secondInput = second.transport.mutationCalls[0]?.input as { + readonly idempotencyKey: string; + }; + expect(secondInput.idempotencyKey).toBe(firstInput.idempotencyKey); + expect(globalThis.sessionStorage.length).toBe(0); + await waitFor(() => + expect(second.transport.queryCalls.length).toBeGreaterThan(1) + ); + expect(screen.getByRole("link", { name: "View Dashboard jobs" })).toHaveAttribute( + "href", + "/jobs" + ); + }); + + test.each([ + ["step_up_required", "Verify your identity again before continuing."], + [ + "mfa_enrollment_required", + "Multi-factor authentication must be enrolled before this action.", + ], + ] as const)("renders fixed recent-MFA feedback for %s", async (reason, message) => { + const rawFailure = Object.assign(new Error("private authorization detail"), { + data: { code: "UNAUTHORIZED", reason }, + }); + const harness = renderSection([allAvailableStatus], [rawFailure]); + const user = userEvent.setup(); + await screen.findByRole("heading", { name: "Service actions" }); + await user.click(screen.getByRole("button", { name: "Queue system update" })); + await user.click(screen.getByRole("button", { name: "Queue update" })); + const authorizationMessages = await screen.findAllByText(message); + expect(authorizationMessages.length).toBeGreaterThan(0); + expect(screen.queryByText(rawFailure.message)).toBeNull(); + expect(harness.transport.mutationCalls).toHaveLength(1); + expect(globalThis.sessionStorage.length).toBe(1); + }); + + test("fails closed before transport when a retained key is invalid", async () => { + globalThis.sessionStorage.setItem( + `mira-dashboard.service-actions.request.v1:authenticated:${authenticatedStatus.user.id}:${authenticatedStatus.session.id}:openclaw-cleanup`, + "invalid!" + ); + const harness = renderSection([allAvailableStatus], [queuedResult]); + const user = userEvent.setup(); + await screen.findByRole("heading", { name: "Service actions" }); + await user.click( + screen.getByRole("button", { + name: "Retry openclaw cleanup request", + }) + ); + await user.click(screen.getByRole("button", { name: "Retry request" })); + const recoveryMessages = await screen.findAllByText( + /could not persist a safe recovery key.*was not submitted/iu + ); + expect(recoveryMessages.length).toBeGreaterThan(0); + expect(harness.transport.mutationCalls).toHaveLength(0); + }); +}); diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx new file mode 100644 index 000000000..4a7acbe75 --- /dev/null +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx @@ -0,0 +1,177 @@ +import { queryOptions, useMutation, useQuery } from "@tanstack/react-query"; +import { useState } from "react"; + +import type { ServiceActionId } from "../../contracts/serviceActions.ts"; +import type { DashboardProcedureOutput, DashboardTrpcClient } from "../api/trpcClient.ts"; +import { useDashboardTrpcClient } from "../api/trpcContextValue.ts"; +import { + dashboardBrowserFailureMessage, + dashboardUnavailableReadRetryDelay, + isDashboardOperationOutcomeUnknown, + retryDashboardUnavailableRead, +} from "../api/trpcError.ts"; +import { useAuthenticatedMutationBoundary } from "../auth/useAuthenticatedMutationBoundary.ts"; +import { Alert } from "../ui/Alert.tsx"; +import { Card } from "../ui/Card.tsx"; +import { PageState } from "../ui/PageState.tsx"; +import { OverviewServiceActionsCard } from "./OverviewServiceActionsCard.tsx"; +import { + authenticatedServiceActionIdentity, + clearServiceActionRecovery, + readOrCreateServiceActionIdempotencyKey, + ServiceActionRecoveryError, + serviceActionPresentations, + serviceActionRecoveryExists, + serviceActionRequestInput, +} from "./serviceActionsOperations.ts"; + +const serviceActionsStatusQueryKey = ["service-actions", "status"] as const; +const serviceActionMutationKey = ["service-actions", "request"] as const; +const serviceActionUnknownOutcomeMessage = + "Dashboard could not confirm whether the service action request was queued. Retrying that action reuses the same recovery key; review Dashboard jobs before retrying."; + +function serviceActionsStatusQueryOptions(client: DashboardTrpcClient) { + return queryOptions({ + queryFn: ({ signal }) => client.query("serviceActions.getStatus", {}, { signal }), + queryKey: serviceActionsStatusQueryKey, + retry: retryDashboardUnavailableRead, + retryDelay: dashboardUnavailableReadRetryDelay, + staleTime: 0, + }); +} + +/** + * Owns session-bound fixed-action requests and lost-response recovery identities. + * @returns One no-retry mutation plus safe feedback and recovery observations. + */ +function useServiceActionRequest() { + const client = useDashboardTrpcClient(); + const boundary = useAuthenticatedMutationBoundary(); + const [error, setError] = useState(); + const [notice, setNotice] = useState(); + const mutation = useMutation< + DashboardProcedureOutput<"serviceActions.request">, + Error, + ServiceActionId + >({ + mutationFn: (actionId) => + boundary.run((signal) => { + const identity = authenticatedServiceActionIdentity(boundary.queryClient); + if (identity === undefined) throw new ServiceActionRecoveryError(); + const idempotencyKey = readOrCreateServiceActionIdempotencyKey( + identity, + actionId + ); + return client.mutation( + "serviceActions.request", + serviceActionRequestInput(actionId, idempotencyKey), + { signal } + ); + }), + mutationKey: serviceActionMutationKey, + onError: (mutationError) => { + if (!boundary.completionIsCurrent()) return; + if (mutationError instanceof ServiceActionRecoveryError) { + setError( + "Dashboard could not persist a safe recovery key in this browser session. The service action was not submitted." + ); + return; + } + setError( + isDashboardOperationOutcomeUnknown(mutationError) + ? serviceActionUnknownOutcomeMessage + : dashboardBrowserFailureMessage(mutationError) + ); + }, + onMutate: () => { + setError(undefined); + setNotice(undefined); + }, + onSuccess: async (result, actionId) => { + if (!boundary.completionIsCurrent()) return; + const identity = authenticatedServiceActionIdentity(boundary.queryClient); + const recoveryCleared = + identity !== undefined && clearServiceActionRecovery(identity, actionId); + setNotice( + `${serviceActionPresentations[actionId].actionLabel} request queued. Dashboard job run: ${result.jobRunId}.` + ); + if (!recoveryCleared) { + setError( + "The request was confirmed queued, but Dashboard could not clear its browser recovery key. Do not create a new request identity." + ); + } + await boundary.queryClient.invalidateQueries({ + exact: true, + queryKey: serviceActionsStatusQueryKey, + refetchType: "active", + }); + }, + retry: false, + }); + + return { + ...mutation, + clearError: () => setError(undefined), + clearNotice: () => setNotice(undefined), + error, + notice, + recoveryPending: (actionId: ServiceActionId) => + serviceActionRecoveryExists( + authenticatedServiceActionIdentity(boundary.queryClient), + actionId + ), + }; +} + +/** @returns Fixed service-action status, requests, and partial-read handling. */ +export function OverviewServiceActionsSection() { + const client = useDashboardTrpcClient(); + const query = useQuery(serviceActionsStatusQueryOptions(client)); + const request = useServiceActionRequest(); + + if (query.isPending && query.data === undefined) { + return ( + + + + ); + } + if (query.data === undefined) { + return ( + void query.refetch()} + retryBusy={query.isFetching} + status="error" + title="Service actions unavailable" + /> + ); + } + + return ( +
    + {query.error !== null && ( + + )} + + request.mutate(actionId, { onSuccess: onConfirmed }) + } + recoveryPending={request.recoveryPending} + requestActionId={request.variables} + requestBusy={request.isPending} + /> +
    + ); +} diff --git a/greenfield/src/browser/overview/serviceActionsOperations.test.ts b/greenfield/src/browser/overview/serviceActionsOperations.test.ts new file mode 100644 index 000000000..509c1d2ab --- /dev/null +++ b/greenfield/src/browser/overview/serviceActionsOperations.test.ts @@ -0,0 +1,107 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { QueryClient } from "@tanstack/react-query"; + +import type { AuthStatus } from "../../contracts/auth.ts"; +import { authStatusQueryKey } from "../auth/authQueries.ts"; +import { + authenticatedServiceActionIdentity, + clearServiceActionRecovery, + readOrCreateServiceActionIdempotencyKey, + serviceActionRecoveryExists, + serviceActionRequestInput, +} from "./serviceActionsOperations.ts"; + +const timestampMs = 1_800_000_000_000; + +function authenticatedStatus(userId: string, sessionId: string): AuthStatus { + return { + session: { + authenticatedAtMs: timestampMs, + authMethod: "password", + createdAtMs: timestampMs, + expiresAtMs: timestampMs + 86_400_000, + id: sessionId, + isCurrent: true, + lastSeenAtMs: timestampMs, + }, + state: "authenticated", + user: { id: userId, username: "operator" }, + }; +} + +afterEach(() => { + globalThis.sessionStorage.clear(); +}); + +describe("service action browser operations", () => { + test("binds recovery keys to the exact user, session, and action", () => { + const queryClient = new QueryClient(); + queryClient.setQueryData( + authStatusQueryKey, + authenticatedStatus("019fd974-54a2-74dd-a64b-d4186f8d8828", "a".repeat(32)) + ); + const firstIdentity = authenticatedServiceActionIdentity(queryClient); + expect(firstIdentity).toBe( + `authenticated:019fd974-54a2-74dd-a64b-d4186f8d8828:${"a".repeat(32)}` + ); + const cleanupKey = readOrCreateServiceActionIdempotencyKey( + firstIdentity!, + "openclaw-cleanup" + ); + const updateKey = readOrCreateServiceActionIdempotencyKey( + firstIdentity!, + "system-update" + ); + expect(cleanupKey).toMatch(/^[0-9a-f]{32}$/u); + expect(updateKey).toMatch(/^[0-9a-f]{32}$/u); + expect(updateKey).not.toBe(cleanupKey); + + queryClient.setQueryData( + authStatusQueryKey, + authenticatedStatus("019fd974-54a2-74dd-a64b-d4186f8d8828", "b".repeat(32)) + ); + const secondIdentity = authenticatedServiceActionIdentity(queryClient)!; + const secondCleanupKey = readOrCreateServiceActionIdempotencyKey( + secondIdentity, + "openclaw-cleanup" + ); + expect(secondCleanupKey).not.toBe(cleanupKey); + expect(serviceActionRecoveryExists(firstIdentity, "openclaw-cleanup")).toBe(true); + expect(serviceActionRecoveryExists(secondIdentity, "openclaw-cleanup")).toBe( + true + ); + expect(clearServiceActionRecovery(firstIdentity!, "openclaw-cleanup")).toBe(true); + expect(serviceActionRecoveryExists(firstIdentity, "openclaw-cleanup")).toBe( + false + ); + expect(serviceActionRecoveryExists(secondIdentity, "openclaw-cleanup")).toBe( + true + ); + queryClient.clear(); + }); + + test("builds only the four exact confirmation inputs", () => { + const idempotencyKey = "a".repeat(32); + expect(serviceActionRequestInput("openclaw-cleanup", idempotencyKey)).toEqual({ + actionId: "openclaw-cleanup", + confirmation: "cleanup-openclaw", + idempotencyKey, + }); + expect(serviceActionRequestInput("openclaw-update", idempotencyKey)).toEqual({ + actionId: "openclaw-update", + confirmation: "update-openclaw", + idempotencyKey, + }); + expect(serviceActionRequestInput("system-restart", idempotencyKey)).toEqual({ + actionId: "system-restart", + confirmation: "restart-system", + idempotencyKey, + }); + expect(serviceActionRequestInput("system-update", idempotencyKey)).toEqual({ + actionId: "system-update", + confirmation: "update-system", + idempotencyKey, + }); + }); +}); diff --git a/greenfield/src/browser/overview/serviceActionsOperations.ts b/greenfield/src/browser/overview/serviceActionsOperations.ts new file mode 100644 index 000000000..27a776942 --- /dev/null +++ b/greenfield/src/browser/overview/serviceActionsOperations.ts @@ -0,0 +1,204 @@ +import type { QueryClient } from "@tanstack/react-query"; + +import type { AuthStatus } from "../../contracts/auth.ts"; +import type { + RequestServiceActionInput, + ServiceActionId, +} from "../../contracts/serviceActions.ts"; +import { authStatusCacheIdentity, authStatusQueryKey } from "../auth/authQueries.ts"; + +export const serviceActionRecoveryStoragePrefix = + "mira-dashboard.service-actions.request.v1:"; + +const serviceActionIdempotencyKeyPattern = /^[0-9a-f]{32}$/u; + +export interface ServiceActionPresentation { + readonly actionLabel: string; + readonly buttonLabel: string; + readonly confirmationLabel: string; + readonly confirmationTitle: string; + readonly description: string; + readonly warning: string; +} + +export const serviceActionPresentations = Object.freeze({ + "openclaw-cleanup": { + actionLabel: "OpenClaw cleanup", + buttonLabel: "Queue OpenClaw cleanup", + confirmationLabel: "Queue cleanup", + confirmationTitle: "Queue OpenClaw cleanup?", + description: + "Runs source-owned OpenClaw session and artifact maintenance without generic filesystem or Docker cleanup.", + warning: + "This queues OpenClaw's own bounded session and artifact maintenance. Review Dashboard jobs for the durable result.", + }, + "openclaw-update": { + actionLabel: "OpenClaw update", + buttonLabel: "Queue OpenClaw update", + confirmationLabel: "Queue update", + confirmationTitle: "Queue OpenClaw update?", + description: + "Requests the source-owned OpenClaw update workflow through a fixed worker action.", + warning: + "OpenClaw updates can take time and may restart the Gateway. The Dashboard only confirms that the durable request was queued.", + }, + "system-restart": { + actionLabel: "System restart", + buttonLabel: "Queue system restart", + confirmationLabel: "Queue restart", + confirmationTitle: "Queue a system restart?", + description: + "Requests a fixed host restart through the separately provisioned worker boundary.", + warning: + "A system restart request interrupts Dashboard, OpenClaw, and other host services. Success here means the restart request was accepted for durable processing, not that the host restarted.", + }, + "system-update": { + actionLabel: "System update", + buttonLabel: "Queue system update", + confirmationLabel: "Queue update", + confirmationTitle: "Queue a system update?", + description: + "Runs the fixed host package-update workflow through the separately provisioned worker boundary.", + warning: + "System updates can take a long time and may affect running services. Review Dashboard jobs for the durable result.", + }, +} satisfies Readonly>); + +export class ServiceActionRecoveryError extends Error { + constructor() { + super("Service action recovery is unavailable"); + this.name = "ServiceActionRecoveryError"; + } +} + +/** @returns The exact authenticated user/session browser identity, when available. */ +export function authenticatedServiceActionIdentity( + queryClient: QueryClient +): string | undefined { + const status = queryClient.getQueryData(authStatusQueryKey); + return status?.state === "authenticated" + ? authStatusCacheIdentity(status) + : undefined; +} + +function serviceActionRecoveryStorageKey( + identity: string, + actionId: ServiceActionId +): string { + return `${serviceActionRecoveryStoragePrefix}${identity}:${actionId}`; +} + +/** + * @param identity Exact authenticated browser identity, when available. + * @param actionId Fixed action whose recovery state is inspected. + * @returns Whether the exact identity and action retain a recovery key. + */ +export function serviceActionRecoveryExists( + identity: string | undefined, + actionId: ServiceActionId +): boolean { + if (identity === undefined) return false; + try { + return ( + globalThis.sessionStorage.getItem( + serviceActionRecoveryStorageKey(identity, actionId) + ) !== null + ); + } catch { + return false; + } +} + +/** + * Returns an existing identity/action-bound key or durably records a fresh key. + * Storage failure is fail-closed so a privileged request is never sent unrecoverably. + * @param identity Exact authenticated browser identity. + * @param actionId Fixed action whose request identity is retained. + * @returns The retained or newly persisted idempotency key. + */ +export function readOrCreateServiceActionIdempotencyKey( + identity: string, + actionId: ServiceActionId +): string { + const storageKey = serviceActionRecoveryStorageKey(identity, actionId); + let current: string | null; + try { + current = globalThis.sessionStorage.getItem(storageKey); + } catch { + throw new ServiceActionRecoveryError(); + } + if (current !== null) { + if (serviceActionIdempotencyKeyPattern.test(current)) return current; + throw new ServiceActionRecoveryError(); + } + + const created = globalThis.crypto.randomUUID().replaceAll("-", ""); + try { + globalThis.sessionStorage.setItem(storageKey, created); + if (globalThis.sessionStorage.getItem(storageKey) !== created) { + throw new ServiceActionRecoveryError(); + } + } catch { + throw new ServiceActionRecoveryError(); + } + return created; +} + +/** + * @param identity Exact authenticated browser identity. + * @param actionId Fixed action whose confirmed recovery key is removed. + * @returns Whether the exact recovery key was observably removed. + */ +export function clearServiceActionRecovery( + identity: string, + actionId: ServiceActionId +): boolean { + const storageKey = serviceActionRecoveryStorageKey(identity, actionId); + try { + globalThis.sessionStorage.removeItem(storageKey); + return globalThis.sessionStorage.getItem(storageKey) === null; + } catch { + return false; + } +} + +/** + * @param actionId Fixed action selected by the operator. + * @param idempotencyKey Browser-retained request identity. + * @returns The exact contract input for one fixed action. + */ +export function serviceActionRequestInput( + actionId: ServiceActionId, + idempotencyKey: string +): RequestServiceActionInput { + switch (actionId) { + case "openclaw-cleanup": { + return { + actionId, + confirmation: "cleanup-openclaw", + idempotencyKey, + }; + } + case "openclaw-update": { + return { + actionId, + confirmation: "update-openclaw", + idempotencyKey, + }; + } + case "system-restart": { + return { + actionId, + confirmation: "restart-system", + idempotencyKey, + }; + } + case "system-update": { + return { + actionId, + confirmation: "update-system", + idempotencyKey, + }; + } + } +} diff --git a/greenfield/src/contracts/contractRegistry.ts b/greenfield/src/contracts/contractRegistry.ts index be9fac845..7adacb27f 100644 --- a/greenfield/src/contracts/contractRegistry.ts +++ b/greenfield/src/contracts/contractRegistry.ts @@ -44,6 +44,7 @@ import { import { reportProcedureContracts } from "./reports.ts"; import { scheduleProcedureContracts } from "./schedules.ts"; import { securityAuditProcedureContracts } from "./securityAudit.ts"; +import { serviceActionProcedureContracts } from "./serviceActions.ts"; import { systemProcedureContracts, systemRawHttpContracts } from "./system.ts"; import { taskRealtimeEventContract } from "./taskRealtime.ts"; import { taskProcedureContracts } from "./tasks.ts"; @@ -73,6 +74,7 @@ const registeredProcedureContracts = [ ...reportProcedureContracts, ...scheduleProcedureContracts, ...securityAuditProcedureContracts, + ...serviceActionProcedureContracts, ...systemProcedureContracts, ...taskProcedureContracts, ...terminalProcedureContracts, diff --git a/greenfield/src/contracts/security.test.ts b/greenfield/src/contracts/security.test.ts index b20b9346a..f0d9d69c6 100644 --- a/greenfield/src/contracts/security.test.ts +++ b/greenfield/src/contracts/security.test.ts @@ -39,6 +39,8 @@ describe("request authentication contract", () => { "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", diff --git a/greenfield/src/contracts/security.ts b/greenfield/src/contracts/security.ts index d0177e473..ea1f8f98b 100644 --- a/greenfield/src/contracts/security.ts +++ b/greenfield/src/contracts/security.ts @@ -115,6 +115,8 @@ export const applicationCapabilities = [ "openclaw-tasks:write", "reports:read", "reports:write", + "service-actions:read", + "service-actions:write", "tasks:read", "tasks:write", "terminal:read", diff --git a/greenfield/src/contracts/serviceActions.test.ts b/greenfield/src/contracts/serviceActions.test.ts new file mode 100644 index 000000000..054e2bc00 --- /dev/null +++ b/greenfield/src/contracts/serviceActions.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, test } from "bun:test"; + +import * as v from "valibot"; + +import { procedureContracts } from "./contractRegistry.ts"; +import { + getServiceActionsStatusResultSchema, + requestServiceActionInputSchema, + requestServiceActionResultSchema, + serviceActionIds, + serviceActionProcedureContracts, +} from "./serviceActions.ts"; + +const runId = "018f6f50-6a9e-7b88-8000-000000000001"; +const idempotencyKey = "A".repeat(43); + +function queuedRun(actionKey: string) { + return { + actionKey, + attemptCount: 0, + attemptLimit: 1, + availableAtMs: 1000, + cancellationPolicy: "never" as const, + displayName: "Service action", + eventCount: 1, + id: runId, + priority: 0, + queuedAtMs: 1000, + resourceClass: "exclusive" as const, + resourceKeys: ["host.mutation"], + retrySafe: false, + state: "queued" as const, + stateVersion: 1, + timeoutMs: 60_000, + triggerType: "manual" as const, + updatedAtMs: 1000, + }; +} + +describe("service action contracts", () => { + test("registers one session-only status query and one recent-MFA mutation", () => { + expect( + serviceActionProcedureContracts.map( + ({ access, errors, kind, name, transport }) => ({ + access, + errors, + kind, + name, + transport, + }) + ) + ).toEqual([ + { + access: { + capabilities: ["service-actions:read"], + capabilityPolicy: "all", + kind: "authenticated", + principalKinds: ["session"], + }, + errors: ["FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], + kind: "query", + name: "serviceActions.getStatus", + transport: { + batching: "adapter-default", + handler: "default", + requestBody: "default", + }, + }, + { + access: { + capabilities: ["service-actions:write"], + kind: "recent-auth", + principalKinds: ["session"], + whenMfaDisabled: "deny", + whenMfaEnabled: "mfa", + }, + errors: ["CONFLICT", "FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], + kind: "mutation", + name: "serviceActions.request", + transport: { + batching: "forbidden", + handler: "default", + requestBody: "default", + }, + }, + ]); + expect( + procedureContracts + .filter(({ domain }) => domain === "service-actions") + .map(({ name }) => name) + ).toEqual(serviceActionProcedureContracts.map(({ name }) => name)); + }); + + test("accepts only exact fixed action confirmations and idempotency keys", () => { + const valid = [ + ["openclaw-cleanup", "cleanup-openclaw"], + ["openclaw-update", "update-openclaw"], + ["system-restart", "restart-system"], + ["system-update", "update-system"], + ] as const; + + for (const [actionId, confirmation] of valid) { + expect( + v.parse(requestServiceActionInputSchema, { + actionId, + confirmation, + idempotencyKey, + }) + ).toEqual({ actionId, confirmation, idempotencyKey }); + } + + for (const input of [ + { + actionId: "system-restart", + confirmation: "update-system", + idempotencyKey, + }, + { + actionId: "system-cleanup", + confirmation: "cleanup-system", + idempotencyKey, + }, + { + actionId: "system-update", + confirmation: "update-system", + idempotencyKey: "short", + }, + { + actionId: "system-update", + confirmation: "update-system", + idempotencyKey, + command: "apt-get upgrade", + }, + ]) { + expect( + v.safeParse(requestServiceActionInputSchema, input).success + ).toBeFalse(); + } + + expect( + v.parse(requestServiceActionResultSchema, { + actionId: "system-update", + jobRunId: runId, + queued: true, + }) + ).toEqual({ actionId: "system-update", jobRunId: runId, queued: true }); + }); + + test("requires the complete canonical fixed inventory and bounded run projections", () => { + const actions = serviceActionIds.map((id, index) => ({ + ...(index === 0 ? { activeRun: queuedRun("openclaw.sessions.cleanup") } : {}), + availability: index === 3 ? "unavailable" : "available", + id, + })); + expect( + v + .parse(getServiceActionsStatusResultSchema, { + actions, + observedAtMs: 2000, + }) + .actions.map(({ id }) => id) + ).toEqual(serviceActionIds); + + for (const invalidActions of [ + actions.slice(1), + actions.toReversed(), + [...actions.slice(0, -1), actions[0]], + [...actions, actions[0]], + ]) { + expect( + v.safeParse(getServiceActionsStatusResultSchema, { + actions: invalidActions, + observedAtMs: 2000, + }).success + ).toBeFalse(); + } + + expect( + v.safeParse(getServiceActionsStatusResultSchema, { + actions: actions.map((action, index) => + index === 0 + ? { + ...action, + activeRun: { + ...action.activeRun, + payload: { command: "apt-get upgrade" }, + }, + } + : action + ), + observedAtMs: 2000, + }).success + ).toBeFalse(); + }); +}); diff --git a/greenfield/src/contracts/serviceActions.ts b/greenfield/src/contracts/serviceActions.ts new file mode 100644 index 000000000..8aaefa557 --- /dev/null +++ b/greenfield/src/contracts/serviceActions.ts @@ -0,0 +1,185 @@ +import * as v from "valibot"; + +import { compareStrings, hasUniqueArrayItems } from "../shared/validation.ts"; +import { + jobIdempotencyKeySchema, + jobRunIdSchema, + jobRunSummarySchema, + jobTimestampSchema, +} from "./jobModel.ts"; +import type { ProcedureContract } from "./registry.ts"; + +/** Fixed privileged operations accepted by the purpose-built service-actions boundary. */ +export const serviceActionIds = [ + "openclaw-cleanup", + "openclaw-update", + "system-restart", + "system-update", +] as const; + +export const serviceActionIdSchema = v.picklist( + serviceActionIds, + "Service action id is invalid" +); + +export const serviceActionAvailabilitySchema = v.picklist( + ["available", "unavailable"], + "Service action availability is invalid" +); + +const serviceActionStatusSchema = v.strictObject({ + activeRun: v.optional(jobRunSummarySchema), + availability: serviceActionAvailabilitySchema, + id: serviceActionIdSchema, + latestRun: v.optional(jobRunSummarySchema), +}); + +type ServiceActionStatusValue = v.InferOutput; + +/** + * @param actions Fixed status rows to validate. + * @returns Whether the fixed action inventory is complete, unique, and canonical. + */ +export function serviceActionStatusesAreCanonical( + actions: ServiceActionStatusValue[] +): boolean { + return ( + actions.length === serviceActionIds.length && + hasUniqueArrayItems(actions.map(({ id }) => id)) && + actions.every( + ({ id }, index) => + id === serviceActionIds[index] && + (index === 0 || compareStrings(actions[index - 1]?.id ?? "", id) < 0) + ) + ); +} + +export const getServiceActionsStatusInputSchema = v.strictObject({}); + +export const getServiceActionsStatusResultSchema = v.strictObject({ + actions: v.pipe( + v.array(serviceActionStatusSchema, "Service action statuses are invalid"), + v.maxLength( + serviceActionIds.length, + "Service action statuses are outside their budget" + ), + v.check( + serviceActionStatusesAreCanonical, + "Service action statuses are not canonical" + ) + ), + observedAtMs: jobTimestampSchema, +}); + +const serviceActionRequestBase = { + idempotencyKey: jobIdempotencyKeySchema, +}; + +export const requestServiceActionInputSchema = v.variant("actionId", [ + v.strictObject({ + actionId: v.literal("openclaw-cleanup"), + confirmation: v.literal( + "cleanup-openclaw", + "OpenClaw cleanup confirmation is invalid" + ), + ...serviceActionRequestBase, + }), + v.strictObject({ + actionId: v.literal("openclaw-update"), + confirmation: v.literal( + "update-openclaw", + "OpenClaw update confirmation is invalid" + ), + ...serviceActionRequestBase, + }), + v.strictObject({ + actionId: v.literal("system-restart"), + confirmation: v.literal( + "restart-system", + "System restart confirmation is invalid" + ), + ...serviceActionRequestBase, + }), + v.strictObject({ + actionId: v.literal("system-update"), + confirmation: v.literal("update-system", "System update confirmation is invalid"), + ...serviceActionRequestBase, + }), +]); + +export const requestServiceActionResultSchema = v.strictObject({ + actionId: serviceActionIdSchema, + jobRunId: jobRunIdSchema, + queued: v.literal(true, "Service action queue result is invalid"), +}); + +export type ServiceActionId = v.InferOutput; +export type ServiceActionStatus = v.InferOutput; +export type GetServiceActionsStatusResult = v.InferOutput< + typeof getServiceActionsStatusResultSchema +>; +export type RequestServiceActionInput = v.InferOutput< + typeof requestServiceActionInputSchema +>; +export type RequestServiceActionResult = v.InferOutput< + typeof requestServiceActionResultSchema +>; + +const readAccess = { + capabilities: ["service-actions:read"], + capabilityPolicy: "all", + kind: "authenticated", + principalKinds: ["session"], +} as const; +const controlAccess = { + capabilities: ["service-actions:write"], + kind: "recent-auth", + principalKinds: ["session"], + whenMfaDisabled: "deny", + whenMfaEnabled: "mfa", +} as const; + +/** Session-only status and recent-MFA fixed-operation request metadata. */ +export const serviceActionProcedureContracts = [ + { + access: readAccess, + domain: "service-actions", + errors: ["FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], + input: getServiceActionsStatusInputSchema, + inputSchemaId: "serviceActions.getStatus.input", + kind: "query", + name: "serviceActions.getStatus", + output: getServiceActionsStatusResultSchema, + outputSchemaId: "serviceActions.getStatus.output", + summary: + "Returns bounded availability and durable-run observations for fixed privileged service actions.", + transport: { + batching: "adapter-default", + handler: "default", + requestBody: "default", + }, + }, + { + access: controlAccess, + domain: "service-actions", + errorReasons: [ + "mfa_enrollment_required", + "operation_outcome_unknown", + "step_up_required", + ], + errors: ["CONFLICT", "FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], + input: requestServiceActionInputSchema, + inputSchemaId: "serviceActions.request.input", + kind: "mutation", + name: "serviceActions.request", + output: requestServiceActionResultSchema, + outputSchemaId: "serviceActions.request.output", + summary: + "Queues one exact worker-owned service action after recent-MFA authorization and durable audit admission.", + transport: { + batching: "forbidden", + handler: "default", + requestBody: "default", + }, + }, +] as const satisfies readonly ProcedureContract[]; diff --git a/greenfield/src/server/database/migrations/jobsSchema.test.ts b/greenfield/src/server/database/migrations/jobsSchema.test.ts index 33edf6033..b48bf2f76 100644 --- a/greenfield/src/server/database/migrations/jobsSchema.test.ts +++ b/greenfield/src/server/database/migrations/jobsSchema.test.ts @@ -1662,6 +1662,25 @@ describe("jobs baseline schema", () => { insertWorker(database, workerId, 1001); insertWorker(database, otherWorkerId, 1002); insertWorker(database, lifecycleWorkerId, 1003); + expect(() => + database.sqlite.run( + `INSERT INTO worker_instances ( + action_keys_json, capacity, heartbeat_at, id, pid, + release_id, started_at, state + ) VALUES (?, 1, 1000, ?, 1004, ?, 1000, 'online')`, + [ + '["openclaw.sessions.cleanup","host.system.update"]', + uuid(43), + releaseId, + ] + ) + ).toThrow("worker_instances action keys must be canonical"); + expect(() => + database.sqlite.run( + `UPDATE worker_instances SET action_keys_json = '[]' WHERE id = ?`, + [lifecycleWorkerId] + ) + ).toThrow("worker_instances identity is immutable"); insertQueuedRun(database, { id: runId, idempotencyKey: idempotencyKey(43), diff --git a/greenfield/src/server/database/migrations/migrationGraph.test.ts b/greenfield/src/server/database/migrations/migrationGraph.test.ts index 50834f52d..a8a0e7519 100644 --- a/greenfield/src/server/database/migrations/migrationGraph.test.ts +++ b/greenfield/src/server/database/migrations/migrationGraph.test.ts @@ -173,6 +173,7 @@ describe("database migration graph", () => { "worker_instances_reject_active_delete", "worker_instances_reject_identity_update", "worker_instances_reject_replace", + "worker_instances_validate_action_keys_insert", "worker_instances_validate_lifecycle_update", ]) { expect(foundationSql).toContain(`CREATE TRIGGER ${trigger}`); diff --git a/greenfield/src/server/database/schema/automationPrincipalCapabilities.ts b/greenfield/src/server/database/schema/automationPrincipalCapabilities.ts index 8bb02e9e2..b93f66323 100644 --- a/greenfield/src/server/database/schema/automationPrincipalCapabilities.ts +++ b/greenfield/src/server/database/schema/automationPrincipalCapabilities.ts @@ -18,7 +18,7 @@ export const automationPrincipalCapabilities = sqliteTable( (table) => [ check( "automation_principal_capabilities_capability_check", - sql`${table.capability} IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')` + sql`${table.capability} IN ('agents:read', 'agents:write', 'cache:read', 'cache:write', 'chat:read', 'chat:write', 'files:read', 'files:write', 'gateway-sessions:read', 'gateway-sessions:write', 'jobs:read', 'jobs:write', 'logs:read', 'logs:write', 'monitoring:write', 'notifications:read', 'notifications:write', 'openclaw-settings:read', 'openclaw-settings:write', 'openclaw-tasks:read', 'openclaw-tasks:write', 'reports:read', 'reports:write', 'service-actions:read', 'service-actions:write', 'tasks:read', 'tasks:write', 'terminal:read', 'terminal:write')` ), check( "automation_principal_capabilities_granted_at_check", diff --git a/greenfield/src/server/database/schema/workerInstances.ts b/greenfield/src/server/database/schema/workerInstances.ts index bfc069049..3cbca61dd 100644 --- a/greenfield/src/server/database/schema/workerInstances.ts +++ b/greenfield/src/server/database/schema/workerInstances.ts @@ -1,16 +1,19 @@ import { sql } from "drizzle-orm"; import { check, index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { workerActionKeysMaximumBytes } from "../workerActionKeyPolicy.ts"; import { lowercaseHexTextCheck, timestampMillisecondsCheck, uuidV7TextCheck, } from "./checks.ts"; +import { boundedJsonArrayCheck } from "./jobChecks.ts"; /** Durable worker registration and heartbeat state shared across rolling releases. */ export const workerInstances = sqliteTable( "worker_instances", { + actionKeysJson: text("action_keys_json").notNull().default("[]"), capacity: integer("capacity").notNull(), drainingAt: integer("draining_at", { mode: "timestamp_ms" }), heartbeatAt: integer("heartbeat_at", { mode: "timestamp_ms" }).notNull(), @@ -22,6 +25,10 @@ export const workerInstances = sqliteTable( stoppedAt: integer("stopped_at", { mode: "timestamp_ms" }), }, (table) => [ + check( + "worker_instances_action_keys_json_check", + boundedJsonArrayCheck(table.actionKeysJson, workerActionKeysMaximumBytes) + ), check("worker_instances_capacity_check", sql`${table.capacity} BETWEEN 1 AND 16`), check("worker_instances_id_check", uuidV7TextCheck(table.id)), check("worker_instances_pid_check", sql`${table.pid} BETWEEN 1 AND 2147483647`), diff --git a/greenfield/src/server/database/validation/rowSchemas.test.ts b/greenfield/src/server/database/validation/rowSchemas.test.ts index ee2a93057..b1c6e3526 100644 --- a/greenfield/src/server/database/validation/rowSchemas.test.ts +++ b/greenfield/src/server/database/validation/rowSchemas.test.ts @@ -511,6 +511,7 @@ describe("Drizzle-generated Valibot row schemas", () => { expect(v.parse(jobRunEventSelectSchema, jobEvent)).toBeDefined(); const worker = { + actionKeysJson: '["host.system.update"]', capacity: 2, drainingAt: null, heartbeatAt: jobUpdatedAt, @@ -681,6 +682,7 @@ describe("Drizzle-generated Valibot row schemas", () => { ).toThrow(); expect(() => v.parse(workerInstanceSelectSchema, { + actionKeysJson: "[]", capacity: 1, drainingAt: null, heartbeatAt: jobUpdatedAt, @@ -692,6 +694,20 @@ describe("Drizzle-generated Valibot row schemas", () => { stoppedAt: null, }) ).toThrow(); + expect(() => + v.parse(workerInstanceInsertSchema, { + actionKeysJson: '["host.system.update","host.system.restart"]', + capacity: 2, + drainingAt: null, + heartbeatAt: jobUpdatedAt, + id: jobWorkerId, + pid: 1234, + releaseId: "b".repeat(40), + startedAt: jobCreatedAt, + state: "online", + stoppedAt: null, + }) + ).toThrow("Stored worker action keys are invalid"); expect(() => v.parse(resourceLeaseSelectSchema, { acquiredAt: jobCreatedAt, diff --git a/greenfield/src/server/database/validation/workerActionKeys.ts b/greenfield/src/server/database/validation/workerActionKeys.ts new file mode 100644 index 000000000..dac92ce40 --- /dev/null +++ b/greenfield/src/server/database/validation/workerActionKeys.ts @@ -0,0 +1,61 @@ +import * as v from "valibot"; + +import { jobActionKeySchema } from "../../../contracts/jobModel.ts"; +import { utf8ByteLength } from "../../../shared/encoding.ts"; +import { parseJsonText } from "../../../shared/json.ts"; +import { compareStrings, hasUniqueArrayItems } from "../../../shared/validation.ts"; +import { + workerActionKeyMaximum, + workerActionKeysMaximumBytes, +} from "../workerActionKeyPolicy.ts"; + +function actionKeysAreCanonical(keys: string[]): boolean { + return ( + hasUniqueArrayItems(keys) && + keys.every((key, index) => { + const previous = keys[index - 1]; + return previous === undefined || compareStrings(previous, key) < 0; + }) && + utf8ByteLength(JSON.stringify(keys)) <= workerActionKeysMaximumBytes + ); +} + +/** Strict bounded canonical worker action inventory. */ +export const workerActionKeysSchema = v.pipe( + v.array(jobActionKeySchema, "Worker action keys are invalid"), + v.maxLength(workerActionKeyMaximum, "Worker action keys are outside their budget"), + v.check(actionKeysAreCanonical, "Worker action keys are not canonical") +); + +/** + * Canonicalizes one validated release-owned executable-action inventory. + * @param actionKeys Candidate action identities from worker action definitions. + * @returns Frozen sorted unique keys safe to persist as worker identity. + */ +export function canonicalWorkerActionKeys( + actionKeys: readonly string[] +): readonly string[] { + const sorted = actionKeys + .map((actionKey) => v.parse(jobActionKeySchema, actionKey)) + .toSorted(compareStrings); + return Object.freeze([...v.parse(workerActionKeysSchema, sorted)]); +} + +/** + * Serializes one canonical inventory without whitespace or unbounded fields. + * @param actionKeys Candidate action identities. + * @returns Canonical JSON text accepted by the worker persistence boundary. + */ +export function serializeWorkerActionKeys(actionKeys: readonly string[]): string { + return JSON.stringify(canonicalWorkerActionKeys(actionKeys)); +} + +/** + * Parses the immutable action inventory stored on one worker row. + * @param value Stored JSON text. + * @returns Frozen validated canonical action identities. + */ +export function parseWorkerActionKeysJson(value: string): readonly string[] { + const parsed = v.parse(workerActionKeysSchema, parseJsonText(value)); + return Object.freeze([...parsed]); +} diff --git a/greenfield/src/server/database/validation/workerInstances.ts b/greenfield/src/server/database/validation/workerInstances.ts index ea5957eae..8b9f32c49 100644 --- a/greenfield/src/server/database/validation/workerInstances.ts +++ b/greenfield/src/server/database/validation/workerInstances.ts @@ -11,6 +11,7 @@ import { } from "../../../shared/validation.ts"; import { workerInstances } from "../schema/workerInstances.ts"; import { nonnegativeDateSchema, uuidV7TextSchema } from "./scalars.ts"; +import { parseWorkerActionKeysJson } from "./workerActionKeys.ts"; const workerCapacitySchema = v.pipe( positiveSafeIntegerSchema("Stored worker capacity is invalid"), @@ -20,6 +21,17 @@ const workerPidSchema = v.pipe( positiveSafeIntegerSchema("Stored worker pid is invalid"), v.maxValue(2_147_483_647, "Stored worker pid is invalid") ); +const workerActionKeysJsonSchema = v.pipe( + v.string("Stored worker action keys are invalid"), + v.check((value) => { + try { + parseWorkerActionKeysJson(value); + return true; + } catch { + return false; + } + }, "Stored worker action keys are invalid") +); interface StoredWorkerInstance { readonly drainingAt?: Date | null; @@ -49,6 +61,7 @@ function workerLifecycleIsConsistent(worker: StoredWorkerInstance): boolean { } const workerRefinements = { + actionKeysJson: () => workerActionKeysJsonSchema, capacity: () => workerCapacitySchema, drainingAt: nonnegativeDateSchema, heartbeatAt: nonnegativeDateSchema, @@ -81,9 +94,10 @@ const generatedWorkerInstanceInsertSchema = createInsertSchema( workerInstances, workerRefinements ); -const workerInstanceInsertObjectSchema = v.strictObject( - generatedWorkerInstanceInsertSchema.entries -); +const workerInstanceInsertObjectSchema = v.strictObject({ + ...generatedWorkerInstanceInsertSchema.entries, + actionKeysJson: workerActionKeysJsonSchema, +}); /** Validates one initially-online worker registration before insertion. */ export const workerInstanceInsertSchema = v.pipe( diff --git a/greenfield/src/server/database/workerActionKeyPolicy.ts b/greenfield/src/server/database/workerActionKeyPolicy.ts new file mode 100644 index 000000000..f369620f0 --- /dev/null +++ b/greenfield/src/server/database/workerActionKeyPolicy.ts @@ -0,0 +1,4 @@ +/** Maximum executable action identities advertised by one worker process. */ +export const workerActionKeyMaximum = 32; +/** Maximum canonical UTF-8 JSON representation retained in one worker row. */ +export const workerActionKeysMaximumBytes = 4 * 1024; diff --git a/greenfield/src/server/domains/cache/repository.test.ts b/greenfield/src/server/domains/cache/repository.test.ts index 9f8dedc2f..9dd25aa61 100644 --- a/greenfield/src/server/domains/cache/repository.test.ts +++ b/greenfield/src/server/domains/cache/repository.test.ts @@ -63,6 +63,7 @@ async function runningClaim( await jobs.registerWorker({ ...noSideEffects, worker: { + actionKeysJson: "[]", capacity: 1, drainingAt: null, heartbeatAt: new Date(1000), diff --git a/greenfield/src/server/domains/jobs/actionExecutors.test.ts b/greenfield/src/server/domains/jobs/actionExecutors.test.ts index 4a5d0af42..ea186761b 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.test.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.test.ts @@ -2,15 +2,18 @@ import { describe, expect, test } from "bun:test"; import { Effect } from "effect"; +import { OpenClawServiceActionsExecutionError } from "../../../shared/openClawServiceActions.ts"; import { testMoltbookCollector, testMoltbookDashboardSnapshot, } from "../../test/support/moltbook.ts"; import { createJobWorkerActionResolver, + createHostOperationJobExecutor, createLogMaintenanceJobExecutor, createMoltbookDashboardExecutor, createOpenClawGatewayRestartJobExecutor, + createOpenClawServiceActionJobExecutor, createSystemHostExecutor, createWorkspaceFileWriteJobExecutor, createJobWorkerActionRegistry, @@ -19,6 +22,7 @@ import { import { type JobActionExecutionContext, type JobCacheAttemptCommit, + JobActionOutcomeUnknownError, JobActionRetryableError, jobActionDefinitions, } from "./actionRegistry.ts"; @@ -45,6 +49,25 @@ describe("worker-only job executor registry", () => { logMaintenance: { run: () => Promise.resolve(undefined) }, moltbook: testMoltbookCollector, openClawGateway: { restart: () => Promise.resolve() }, + openClawServiceActions: { + cleanupSessions: () => + Promise.resolve({ + artifactsRemoved: 0, + bytesFreed: 0, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + status: "completed", + storesProcessed: 0, + }), + updateInstallation: () => Promise.resolve({ status: "accepted" }), + }, }); expect(findAction("system.worker-smoke")).toBeDefined(); expect(findAction("cache.refresh.system-host")).toBeDefined(); @@ -54,6 +77,8 @@ describe("worker-only job executor registry", () => { resourceClass: "exclusive", retrySafe: false, }); + expect(findAction("openclaw.sessions.cleanup")).toBeDefined(); + expect(findAction("openclaw.installation.update")).toBeDefined(); expect(findAction("system.shell")).toBeUndefined(); }); @@ -77,6 +102,155 @@ describe("worker-only job executor registry", () => { expect(failure).toBeInstanceOf(Error); }); + test("persists only fixed host-operation settlement and rejects mismatched results", async () => { + const calls: unknown[] = []; + const hostOperations = { + availableOperations: () => Promise.resolve([]), + request( + operationId: "system-restart" | "system-update", + signal?: AbortSignal + ) { + calls.push({ operationId, signal }); + return Promise.resolve( + operationId === "system-restart" + ? ({ status: "accepted" } as const) + : ({ status: "completed" } as const) + ); + }, + }; + expect( + await Effect.runPromise( + createHostOperationJobExecutor(hostOperations, "system-restart")( + executionContext([]), + {} + ) + ) + ).toEqual({ completedAtMs: 5000, status: "accepted" }); + expect( + await Effect.runPromise( + createHostOperationJobExecutor(hostOperations, "system-update")( + executionContext([]), + {} + ) + ) + ).toEqual({ completedAtMs: 5000, status: "completed" }); + expect(calls).toMatchObject([ + { operationId: "system-restart", signal: expect.any(AbortSignal) }, + { operationId: "system-update", signal: expect.any(AbortSignal) }, + ]); + + const failure = await Effect.runPromise( + createHostOperationJobExecutor( + { + availableOperations: () => Promise.resolve([]), + request: () => Promise.resolve({ status: "completed" }), + }, + "system-restart" + )(executionContext([]), {}) + ).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + }); + + test("persists only aggregate OpenClaw cleanup and validated update summaries", async () => { + const signals: AbortSignal[] = []; + const serviceActions = { + cleanupSessions(signal?: AbortSignal) { + if (signal !== undefined) signals.push(signal); + return Promise.resolve({ + artifactsRemoved: 2, + bytesFreed: 1024, + diskEntriesRemoved: 1, + diskFilesRemoved: 1, + dmScopesRetired: 3, + entriesAfter: 4, + entriesBefore: 8, + entriesCapped: 0, + entriesPruned: 2, + missingEntriesRemoved: 1, + modelRunsPruned: 1, + status: "completed" as const, + storesProcessed: 2, + }); + }, + updateInstallation(signal?: AbortSignal) { + if (signal !== undefined) signals.push(signal); + return Promise.resolve({ + afterVersion: "2026.8.0", + beforeVersion: "2026.7.2-beta.7", + status: "completed" as const, + }); + }, + }; + expect( + await Effect.runPromise( + createOpenClawServiceActionJobExecutor( + serviceActions, + "openclaw-cleanup" + )(executionContext([]), {}) + ) + ).toEqual({ + artifactsRemoved: 2, + bytesFreed: 1024, + completedAtMs: 5000, + diskEntriesRemoved: 1, + diskFilesRemoved: 1, + dmScopesRetired: 3, + entriesAfter: 4, + entriesBefore: 8, + entriesCapped: 0, + entriesPruned: 2, + missingEntriesRemoved: 1, + modelRunsPruned: 1, + status: "completed", + storesProcessed: 2, + }); + expect( + await Effect.runPromise( + createOpenClawServiceActionJobExecutor(serviceActions, "openclaw-update")( + executionContext([]), + {} + ) + ) + ).toEqual({ + afterVersion: "2026.8.0", + beforeVersion: "2026.7.2-beta.7", + completedAtMs: 5000, + status: "completed", + }); + expect(signals).toEqual([expect.any(AbortSignal), expect.any(AbortSignal)]); + + const failure = await Effect.runPromise( + createOpenClawServiceActionJobExecutor( + { + cleanupSessions: (signal) => serviceActions.cleanupSessions(signal), + updateInstallation: () => + Promise.resolve({ + afterVersion: "../../private", + status: "completed", + }), + }, + "openclaw-update" + )(executionContext([]), {}) + ).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(JSON.stringify(failure)).not.toContain("../../private"); + + const unknownFailure = await Effect.runPromise( + createOpenClawServiceActionJobExecutor( + { + cleanupSessions: () => + Promise.reject( + new OpenClawServiceActionsExecutionError("unknown-outcome") + ), + updateInstallation: () => Promise.resolve({ status: "accepted" }), + }, + "openclaw-cleanup" + )(executionContext([]), {}) + ).catch((error: unknown) => error); + expect(unknownFailure).toBeInstanceOf(JobActionOutcomeUnknownError); + expect(JSON.stringify(unknownFailure)).not.toContain("Gateway"); + }); + test("fails closed for missing, extra, and duplicate executor keys", () => { expect(() => createJobWorkerActionRegistry(jobActionDefinitions, [ diff --git a/greenfield/src/server/domains/jobs/actionExecutors.ts b/greenfield/src/server/domains/jobs/actionExecutors.ts index 95f62f54e..f14ffcdd8 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.ts @@ -7,8 +7,13 @@ import { type LogMaintenanceExecutionSummary, logMaintenancePolicyIdSchema, } from "../../../contracts/logs.ts"; +import type { FixedHostOperationsExecutionPort } from "../../../shared/hostOperations.ts"; import type { JsonObject } from "../../../shared/json.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; +import { + OpenClawServiceActionsExecutionError, + type OpenClawServiceActionsExecutionPort, +} from "../../../shared/openClawServiceActions.ts"; import { collectSystemHostPayload } from "../cache/systemHostProvider.ts"; import { parseWorkspaceFileJobPayload } from "../files/jobPayload.ts"; import type { MoltbookDashboardCollector } from "../moltbook/provider.ts"; @@ -17,12 +22,25 @@ import { type JobExecutableActionDefinition, type JobActionRegistration, type JobActionSuccessfulSettlementHandler, + JobActionOutcomeUnknownError, JobActionRetryableError, + hostSystemRestartJobActionDefinition, + hostSystemRestartJobActionKey, + hostSystemRestartJobResultSchema, + hostSystemUpdateJobActionDefinition, + hostSystemUpdateJobActionKey, + hostSystemUpdateJobResultSchema, jobActionDefinitions, logMaintenanceJobActionKey, openClawGatewayRestartJobActionDefinition, openClawGatewayRestartJobActionKey, openClawGatewayRestartJobResultSchema, + openClawInstallationUpdateJobActionDefinition, + openClawInstallationUpdateJobActionKey, + openClawInstallationUpdateJobResultSchema, + openClawSessionsCleanupJobActionDefinition, + openClawSessionsCleanupJobActionKey, + openClawSessionsCleanupJobResultSchema, validateJobActionRegistration, workspaceFileReplaceJobActionDefinition, workspaceFileReplaceJobActionKey, @@ -291,6 +309,69 @@ export function createOpenClawGatewayRestartJobExecutor( }); } +/** + * Adapts a separately privileged fixed host-operation port without persisting output. + * @param hostOperations Worker-only separately privileged fixed-operation authority. + * @param operationId Exact reviewed host operation. + * @returns A non-retryable empty-payload executor with a constant result surface. + */ +export function createHostOperationJobExecutor( + hostOperations: FixedHostOperationsExecutionPort, + operationId: "system-restart" | "system-update" +): JobActionExecutor { + return (context, payload) => + Effect.tryPromise({ + catch: () => new Error("Fixed host operation failed"), + try: async (signal) => { + v.parse(emptyPayloadSchema, payload); + const result = await hostOperations.request(operationId, signal); + if (operationId === "system-restart") { + return v.parse(hostSystemRestartJobResultSchema, { + completedAtMs: context.nowMs(), + status: result.status, + }); + } + return v.parse(hostSystemUpdateJobResultSchema, { + completedAtMs: context.nowMs(), + status: result.status, + }); + }, + }); +} + +/** + * Adapts one exact worker-only OpenClaw operation to a secret-free job result. + * @returns A non-retryable empty-payload executor for the selected operation. + */ +export function createOpenClawServiceActionJobExecutor( + serviceActions: OpenClawServiceActionsExecutionPort, + operationId: "openclaw-cleanup" | "openclaw-update" +): JobActionExecutor { + return (context, payload) => + Effect.tryPromise({ + catch: (error) => + error instanceof OpenClawServiceActionsExecutionError && + error.reason === "unknown-outcome" + ? new JobActionOutcomeUnknownError() + : new Error("Fixed OpenClaw Service Action failed"), + try: async (signal) => { + v.parse(emptyPayloadSchema, payload); + if (operationId === "openclaw-cleanup") { + const result = await serviceActions.cleanupSessions(signal); + return v.parse(openClawSessionsCleanupJobResultSchema, { + ...result, + completedAtMs: context.nowMs(), + }); + } + const result = await serviceActions.updateInstallation(signal); + return v.parse(openClawInstallationUpdateJobResultSchema, { + ...result, + completedAtMs: context.nowMs(), + }); + }, + }); +} + /** * Adapts one schema-validated spooled command to the worker-only structural writer. * @param writer Worker-owned descriptor writer. @@ -390,8 +471,10 @@ export function createJobWorkerActionRegistry( export interface JobWorkerActionResolverDependencies { readonly actionDefinitions?: readonly JobExecutableActionDefinition[]; readonly logMaintenance: LogMaintenanceExecutionPort; + readonly hostOperations?: FixedHostOperationsExecutionPort; readonly moltbook: MoltbookDashboardCollector; readonly openClawGateway?: OpenClawGatewayLifecycleExecutionPort; + readonly openClawServiceActions?: OpenClawServiceActionsExecutionPort; readonly workspaceFiles?: WorkspaceFileWriteExecutionPort; } @@ -406,6 +489,18 @@ export function createJobWorkerActionResolver( ...(dependencies.openClawGateway === undefined ? [] : [openClawGatewayRestartJobActionDefinition]), + ...(dependencies.openClawServiceActions === undefined + ? [] + : [ + openClawSessionsCleanupJobActionDefinition, + openClawInstallationUpdateJobActionDefinition, + ]), + ...(dependencies.hostOperations === undefined + ? [] + : [ + hostSystemRestartJobActionDefinition, + hostSystemUpdateJobActionDefinition, + ]), ...(workspaceFiles === undefined ? [] : [ @@ -438,6 +533,58 @@ export function createJobWorkerActionResolver( ), }), ]), + ...(dependencies.openClawServiceActions === undefined || + !definitions.some( + ({ actionKey }) => actionKey === openClawSessionsCleanupJobActionKey + ) + ? [] + : [ + Object.freeze({ + actionKey: openClawSessionsCleanupJobActionKey, + execute: createOpenClawServiceActionJobExecutor( + dependencies.openClawServiceActions, + "openclaw-cleanup" + ), + }), + ]), + ...(dependencies.openClawServiceActions === undefined || + !definitions.some( + ({ actionKey }) => actionKey === openClawInstallationUpdateJobActionKey + ) + ? [] + : [ + Object.freeze({ + actionKey: openClawInstallationUpdateJobActionKey, + execute: createOpenClawServiceActionJobExecutor( + dependencies.openClawServiceActions, + "openclaw-update" + ), + }), + ]), + ...(dependencies.hostOperations === undefined || + !definitions.some(({ actionKey }) => actionKey === hostSystemRestartJobActionKey) + ? [] + : [ + Object.freeze({ + actionKey: hostSystemRestartJobActionKey, + execute: createHostOperationJobExecutor( + dependencies.hostOperations, + "system-restart" + ), + }), + ]), + ...(dependencies.hostOperations === undefined || + !definitions.some(({ actionKey }) => actionKey === hostSystemUpdateJobActionKey) + ? [] + : [ + Object.freeze({ + actionKey: hostSystemUpdateJobActionKey, + execute: createHostOperationJobExecutor( + dependencies.hostOperations, + "system-update" + ), + }), + ]), Object.freeze({ actionKey: "system.worker-smoke", execute: workerSmokeExecutor, diff --git a/greenfield/src/server/domains/jobs/actionRegistry.test.ts b/greenfield/src/server/domains/jobs/actionRegistry.test.ts index bcaa8a591..a6c49084c 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.test.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.test.ts @@ -2,8 +2,12 @@ import { describe, expect, test } from "bun:test"; import { findJobActionDefinition, + hostSystemRestartJobActionDefinition, + hostSystemUpdateJobActionDefinition, isRegisteredJobSchedule, openClawGatewayRestartJobActionDefinition, + openClawInstallationUpdateJobActionDefinition, + openClawSessionsCleanupJobActionDefinition, parseJobActionOutputMessage, parseJobActionProgress, validateJobActionRegistration, @@ -142,4 +146,44 @@ describe("durable job action registry", () => { "scheduleId" ); }); + + test("publishes four fixed Service Actions with cross-domain exclusive locks", () => { + for (const definition of [ + openClawSessionsCleanupJobActionDefinition, + hostSystemRestartJobActionDefinition, + hostSystemUpdateJobActionDefinition, + ]) { + expect(definition).toMatchObject({ + attemptLimit: 1, + cancellationPolicy: "never", + manualExposure: "none", + priority: 20, + resourceClass: "exclusive", + retrySafe: false, + }); + expect(definition).not.toHaveProperty("scheduleId"); + } + expect(openClawSessionsCleanupJobActionDefinition.resourceKeys).toEqual([ + "host.mutation", + "openclaw.gateway", + ]); + expect(hostSystemRestartJobActionDefinition.resourceKeys).toEqual([ + "host.mutation", + ]); + expect(hostSystemUpdateJobActionDefinition.resourceKeys).toEqual([ + "host.mutation", + ]); + expect(openClawInstallationUpdateJobActionDefinition).toMatchObject({ + attemptLimit: 1, + cancellationPolicy: "never", + manualExposure: "none", + resourceClass: "exclusive", + resourceKeys: ["host.mutation", "openclaw.gateway"], + retrySafe: false, + }); + expect(hostSystemRestartJobActionDefinition.timeoutMs).toBe(60_000); + expect(hostSystemUpdateJobActionDefinition.timeoutMs).toBe(7_200_000); + expect(openClawSessionsCleanupJobActionDefinition.timeoutMs).toBe(630_000); + expect(openClawInstallationUpdateJobActionDefinition.timeoutMs).toBe(2_130_000); + }); }); diff --git a/greenfield/src/server/domains/jobs/actionRegistry.ts b/greenfield/src/server/domains/jobs/actionRegistry.ts index d2f5140de..2d5c60a01 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.ts @@ -44,6 +44,14 @@ export const workspaceFileWriteJobActionKey = "workspace-files.apply-write"; export const workspaceFileReplaceJobActionKey = "workspace-files.apply-replacement"; /** Fixed non-retryable worker action for one operator-requested Gateway restart. */ export const openClawGatewayRestartJobActionKey = "openclaw.gateway.restart"; +/** Fixed worker-only OpenClaw maintenance identity selected by Service Actions. */ +export const openClawSessionsCleanupJobActionKey = "openclaw.sessions.cleanup"; +/** Fixed worker-only OpenClaw update identity selected by Service Actions. */ +export const openClawInstallationUpdateJobActionKey = "openclaw.installation.update"; +/** Fixed root-brokered host restart identity selected by Service Actions. */ +export const hostSystemRestartJobActionKey = "host.system.restart"; +/** Fixed root-brokered host update identity selected by Service Actions. */ +export const hostSystemUpdateJobActionKey = "host.system.update"; /** Secret-free terminal payload persisted by the fixed Gateway restart executor. */ export const openClawGatewayRestartJobResultSchema = v.strictObject({ @@ -51,6 +59,59 @@ export const openClawGatewayRestartJobResultSchema = v.strictObject({ status: v.literal("restarted", "OpenClaw Gateway restart result is invalid"), }); +/** Redacted accepted-only result for a host restart request. */ +export const hostSystemRestartJobResultSchema = v.strictObject({ + completedAtMs: jobTimestampSchema, + status: v.literal("accepted", "Host restart result is invalid"), +}); + +/** Redacted terminal result for one fixed host update unit. */ +export const hostSystemUpdateJobResultSchema = v.strictObject({ + completedAtMs: jobTimestampSchema, + status: v.literal("completed", "Host update result is invalid"), +}); + +const openClawOperationCountSchema = v.pipe( + v.number("OpenClaw operation count is invalid"), + v.safeInteger("OpenClaw operation count is invalid"), + v.minValue(0, "OpenClaw operation count is invalid") +); +const openClawOperationVersionSchema = v.pipe( + v.string("OpenClaw operation version is invalid"), + v.minLength(1, "OpenClaw operation version is invalid"), + v.maxLength(128, "OpenClaw operation version is invalid"), + v.regex( + /^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u, + "OpenClaw operation version is invalid" + ) +); + +/** Aggregate-only result for source-owned OpenClaw session maintenance. */ +export const openClawSessionsCleanupJobResultSchema = v.strictObject({ + artifactsRemoved: openClawOperationCountSchema, + bytesFreed: openClawOperationCountSchema, + completedAtMs: jobTimestampSchema, + diskEntriesRemoved: openClawOperationCountSchema, + diskFilesRemoved: openClawOperationCountSchema, + dmScopesRetired: openClawOperationCountSchema, + entriesAfter: openClawOperationCountSchema, + entriesBefore: openClawOperationCountSchema, + entriesCapped: openClawOperationCountSchema, + entriesPruned: openClawOperationCountSchema, + missingEntriesRemoved: openClawOperationCountSchema, + modelRunsPruned: openClawOperationCountSchema, + status: v.literal("completed"), + storesProcessed: openClawOperationCountSchema, +}); + +/** Version/status-only result for source-owned OpenClaw installation updates. */ +export const openClawInstallationUpdateJobResultSchema = v.strictObject({ + afterVersion: v.optional(openClawOperationVersionSchema), + beforeVersion: v.optional(openClawOperationVersionSchema), + completedAtMs: jobTimestampSchema, + status: v.picklist(["accepted", "completed"]), +}); + export type JobCacheAttemptCommit = | { readonly durationMs: number; @@ -92,6 +153,14 @@ export class JobActionRetryableError extends Error { } } +/** Explicit action-owned classification for irreversible effects with unknown settlement. */ +export class JobActionOutcomeUnknownError extends Error { + constructor() { + super("The job action outcome is unknown"); + this.name = "JobActionOutcomeUnknownError"; + } +} + const jobActionOutputMessageSchema = v.pipe( boundedControlSafeTextSchema(4096, "Job action output is invalid"), v.check((message) => utf8ByteLength(message) <= 4096, "Job action output is invalid") @@ -399,6 +468,61 @@ export const openClawGatewayRestartJobActionDefinition = timeoutMs: 60_000, }); +function serviceActionDefinition(input: { + readonly actionKey: string; + readonly description: string; + readonly displayName: string; + readonly resourceKeys?: readonly string[]; + readonly timeoutMs: number; +}): JobUnscheduledActionDefinition { + return validateJobUnscheduledActionDefinition({ + ...input, + attemptLimit: 1, + cancellationPolicy: "never", + manualExposure: "none", + priority: 20, + resourceClass: "exclusive", + resourceKeys: Object.freeze(input.resourceKeys ?? ["host.mutation"]), + retrySafe: false, + }); +} + +/** Non-retryable source-owned OpenClaw session/artifact maintenance. */ +export const openClawSessionsCleanupJobActionDefinition = serviceActionDefinition({ + actionKey: openClawSessionsCleanupJobActionKey, + description: + "Runs source-owned OpenClaw session and artifact maintenance with fixed enforcement policy.", + displayName: "Clean up OpenClaw sessions", + resourceKeys: Object.freeze(["host.mutation", "openclaw.gateway"]), + timeoutMs: 10 * 60_000 + 30_000, +}); + +/** Non-retryable source-owned OpenClaw installation update and handoff. */ +export const openClawInstallationUpdateJobActionDefinition = serviceActionDefinition({ + actionKey: openClawInstallationUpdateJobActionKey, + description: + "Runs the fixed OpenClaw installation update and managed restart handoff.", + displayName: "Update OpenClaw", + resourceKeys: Object.freeze(["host.mutation", "openclaw.gateway"]), + timeoutMs: 35 * 60_000 + 30_000, +}); + +/** Accepted-only host restart request reserved for a separately privileged adapter. */ +export const hostSystemRestartJobActionDefinition = serviceActionDefinition({ + actionKey: hostSystemRestartJobActionKey, + description: "Requests a host restart through a separately privileged fixed adapter.", + displayName: "Restart host system", + timeoutMs: 60_000, +}); + +/** Non-retryable host package update reserved for a separately privileged adapter. */ +export const hostSystemUpdateJobActionDefinition = serviceActionDefinition({ + actionKey: hostSystemUpdateJobActionKey, + description: "Runs a fixed host update through a separately privileged adapter.", + displayName: "Update host system", + timeoutMs: 2 * 60 * 60_000, +}); + /** Complete reviewed pure-definition registry for this slice. */ export const jobActionDefinitions = Object.freeze([ systemHostCacheDefinition, diff --git a/greenfield/src/server/domains/jobs/coordinator.test.ts b/greenfield/src/server/domains/jobs/coordinator.test.ts index 2b41098d9..6e68920df 100644 --- a/greenfield/src/server/domains/jobs/coordinator.test.ts +++ b/greenfield/src/server/domains/jobs/coordinator.test.ts @@ -9,6 +9,7 @@ import { createJobWorkerActionResolver } from "./actionExecutors.ts"; import { type JobActionRegistration, type JobActionExecutionContext, + JobActionOutcomeUnknownError, JobActionRetryableError, jobActionDefinitions, openClawGatewayRestartJobActionDefinition, @@ -80,6 +81,7 @@ function workerRecord( heartbeatAt = at ) { return { + actionKeysJson: '["system.worker-smoke"]', capacity: 1, drainingAt: state === "online" ? null : heartbeatAt, heartbeatAt, @@ -267,6 +269,7 @@ function repositoryFixture(options: RepositoryFixtureOptions = {}) { const expiryEligibility: boolean[] = []; const expiryNextRuns: Date[] = []; const recoverySideEffects: JobMutationSideEffects[] = []; + const registrations: Array[0]> = []; const reconciliationInputs: Array< Parameters[0] > = []; @@ -383,6 +386,7 @@ function repositoryFixture(options: RepositoryFixtureOptions = {}) { }, registerWorker(input) { events.push("register"); + registrations.push(input); return options.registrationFailure === undefined ? Promise.resolve(workerRecord(input.worker.id, "online")) : Promise.reject(options.registrationFailure); @@ -430,6 +434,7 @@ function repositoryFixture(options: RepositoryFixtureOptions = {}) { expiryNextRuns, lifecycleSideEffects, reconciliationInputs, + registrations, recoverySideEffects, repository, settlements, @@ -494,6 +499,9 @@ describe("durable job worker coordinator", () => { expect(fixture.events.indexOf("register")).toBeGreaterThan( fixture.events.indexOf("reconcile:1") ); + expect(fixture.registrations[0]?.worker.actionKeysJson).toBe( + '["system.worker-smoke"]' + ); expect(fixture.events.indexOf("drain")).toBeLessThan( fixture.events.indexOf("stop") ); @@ -1840,6 +1848,35 @@ describe("durable job worker coordinator", () => { }); }); + test("persists an explicit non-retryable outcome-unknown settlement", async () => { + const workerId = Bun.randomUUIDv7(); + const run = claimedRun(workerId, "test.outcome-unknown"); + const fixture = repositoryFixture({ claim: { kind: "claimed", run } }); + const baseRegistration = jobActionDefinitions.at(0); + if (baseRegistration === undefined) throw new Error("Missing smoke action"); + const registration: JobActionRegistration = { + ...baseRegistration, + actionKey: run.actionKey, + execute: () => Effect.fail(new JobActionOutcomeUnknownError()), + }; + const coordinator = createJobWorkerCoordinator({ + ...coordinatorOptions(fixture.repository, workerId), + findAction: (actionKey) => + actionKey === registration.actionKey ? registration : undefined, + }); + + await coordinator.initialize(); + await waitUntil(() => fixture.settlements.length === 1); + await coordinator.dispose(); + + expect(fixture.settlements[0]?.outcome).toEqual({ + kind: "failed", + terminalCode: "operation-outcome-unknown", + terminalMessage: + "The action may have taken effect; inspect current state before retrying.", + }); + }); + test("settles a synchronous executor-construction defect without failing the worker", async () => { const workerId = Bun.randomUUIDv7(); const run = claimedRun(workerId, "test.synchronous-failure"); diff --git a/greenfield/src/server/domains/jobs/coordinator.ts b/greenfield/src/server/domains/jobs/coordinator.ts index 743058d51..06b1f51cc 100644 --- a/greenfield/src/server/domains/jobs/coordinator.ts +++ b/greenfield/src/server/domains/jobs/coordinator.ts @@ -10,6 +10,7 @@ import { } from "../../../contracts/jobModel.ts"; import type { JsonObject } from "../../../shared/json.ts"; import { parseJsonText } from "../../../shared/json.ts"; +import { serializeWorkerActionKeys } from "../../database/validation/workerActionKeys.ts"; import { sha256Hex } from "../../shared/crypto.ts"; import { type JobActionDefinition, @@ -18,6 +19,7 @@ import { type JobCacheAttemptCommit, type JobCacheAttemptWriteResult, type JobExecutableActionDefinition, + JobActionOutcomeUnknownError, JobActionRetryableError, jobActionDefinitions, logMaintenanceJobActionKey, @@ -466,6 +468,14 @@ function executionOutcome( terminalMessage: "The job action was cancelled.", }; } + if (actionFailure instanceof JobActionOutcomeUnknownError) { + return { + kind: "failed", + terminalCode: "operation-outcome-unknown", + terminalMessage: + "The action may have taken effect; inspect current state before retrying.", + }; + } const shutdown = abortReason instanceof JobCoordinatorShutdownError; return actionFailureOutcome( run, @@ -772,6 +782,9 @@ export function createJobWorkerCoordinator( const generateId = options.generateId ?? (() => Bun.randomUUIDv7()); const findAction = options.findAction ?? findNoAction; const actionDefinitions = options.actionDefinitions ?? jobActionDefinitions; + const actionKeysJson = serializeWorkerActionKeys( + actionDefinitions.map(({ actionKey }) => actionKey) + ); const abortController = new AbortController(); let activeExecution: Promise | undefined; let initializePromise: Promise | undefined; @@ -1096,6 +1109,7 @@ export function createJobWorkerCoordinator( }), }); const worker: WorkerInstanceInsert = { + actionKeysJson, capacity: jobWorkerCapacity, drainingAt: null, heartbeatAt: at, diff --git a/greenfield/src/server/domains/jobs/repository.test.ts b/greenfield/src/server/domains/jobs/repository.test.ts index c84af9c63..40ad34086 100644 --- a/greenfield/src/server/domains/jobs/repository.test.ts +++ b/greenfield/src/server/domains/jobs/repository.test.ts @@ -122,8 +122,13 @@ function queuedEvent(run: JobRunInsert): JobRunEventInsert { }; } -function worker(id: string, capacity = 1): WorkerInstanceInsert { +function worker( + id: string, + capacity = 1, + actionKeysJson = '["system.worker-smoke"]' +): WorkerInstanceInsert { return { + actionKeysJson, capacity, drainingAt: null, heartbeatAt: new Date(2000), @@ -2366,6 +2371,169 @@ describe("durable jobs repository", () => { } }); + test("reads only fresh online exact-release worker actions and preserves identity", async () => { + const database = await openFreshMigratedDatabase(); + const repository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const minimumHeartbeatAt = new Date(10_000); + const expectedReleaseId = "b".repeat(40); + const exactWorkerId = uuid(710); + try { + database.orm + .insert(workerInstances) + .values([ + { + ...worker(exactWorkerId), + actionKeysJson: + '["host.system.update","openclaw.sessions.cleanup"]', + heartbeatAt: minimumHeartbeatAt, + releaseId: expectedReleaseId, + startedAt: new Date(9000), + }, + { + ...worker(uuid(711)), + actionKeysJson: '["openclaw.installation.update"]', + heartbeatAt: new Date(minimumHeartbeatAt.getTime() - 1), + releaseId: expectedReleaseId, + startedAt: new Date(9000), + }, + { + ...worker(uuid(712)), + actionKeysJson: '["host.system.restart"]', + heartbeatAt: minimumHeartbeatAt, + releaseId: "c".repeat(40), + startedAt: new Date(9000), + }, + { + ...worker(uuid(713)), + actionKeysJson: '["host.system.restart"]', + drainingAt: minimumHeartbeatAt, + heartbeatAt: minimumHeartbeatAt, + releaseId: expectedReleaseId, + startedAt: new Date(9000), + state: "draining" as const, + }, + ]) + .run(); + + expect( + repository.readWorkerActionAvailability({ + actionKeys: [ + "openclaw.sessions.cleanup", + "host.system.update", + "host.system.restart", + "openclaw.installation.update", + ], + expectedReleaseId, + minimumHeartbeatAt, + }) + ).toEqual(["host.system.update", "openclaw.sessions.cleanup"]); + + const heartbeat = await repository.heartbeatWorker({ + at: new Date(11_000), + workerId: exactWorkerId, + }); + expect(heartbeat?.actionKeysJson).toBe( + '["host.system.update","openclaw.sessions.cleanup"]' + ); + const draining = await repository.beginWorkerDrain({ + at: new Date(12_000), + sideEffectsForWorker: () => noSideEffects, + workerId: exactWorkerId, + }); + expect(draining).toMatchObject({ + kind: "updated", + worker: { + actionKeysJson: '["host.system.update","openclaw.sessions.cleanup"]', + }, + }); + const stopped = await repository.stopWorker({ + at: new Date(13_000), + sideEffectsForWorker: () => noSideEffects, + workerId: exactWorkerId, + }); + expect(stopped).toMatchObject({ + kind: "updated", + worker: { + actionKeysJson: '["host.system.update","openclaw.sessions.cleanup"]', + }, + }); + } finally { + database.sqlite.close(true); + } + }); + + test("claims only actions advertised by each heterogeneous worker", async () => { + const database = await openFreshMigratedDatabase(); + const repository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const installationRun = queuedRun(714, { + actionKey: "openclaw.installation.update", + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + const hostRun = queuedRun(715, { + actionKey: "host.system.update", + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + try { + for (const run of [installationRun, hostRun]) { + await repository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(run), + run, + }); + } + await repository.registerWorker({ + ...noSideEffects, + worker: worker(workerOneId, 1, '["host.system.update"]'), + }); + await repository.registerWorker({ + ...noSideEffects, + worker: worker(workerTwoId, 1, '["openclaw.installation.update"]'), + }); + + expect( + await repository.claimNextRun({ + at: new Date(5000), + leaseExpiresAt: new Date(35_000), + leaseToken: uuid(716), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerOneId, + }) + ).toMatchObject({ + kind: "claimed", + run: { actionKey: "host.system.update", id: hostRun.id }, + }); + expect( + await repository.claimNextRun({ + at: new Date(5001), + leaseExpiresAt: new Date(35_001), + leaseToken: uuid(717), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toMatchObject({ + kind: "claimed", + run: { + actionKey: "openclaw.installation.update", + id: installationRun.id, + }, + }); + } finally { + database.sqlite.close(true); + } + }); + test("reserves the terminal event when payload consumes the byte budget", async () => { const database = await openFreshMigratedDatabase(); const repository = createJobRepository( @@ -3067,7 +3235,7 @@ describe("durable jobs repository", () => { }); await repository.registerWorker({ ...noSideEffects, - worker: worker(workerOneId), + worker: worker(workerOneId, 1, '["maintenance.rotate-logs"]'), }); const activeReal = await enqueue(83, realPayload); expect( diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index bfa476200..44724f1ff 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -31,6 +31,7 @@ import { jobResourceClasses, jobResourceKeysSchema, jobRunStates, + jobTimestampSchema, jobWorkerSummaryMaximum, type JobResourceClass, type JobRunState, @@ -52,6 +53,7 @@ import { logMaintenanceJobActionKey, logMaintenanceJobPayloadIndexMaximumBytes, } from "../../../shared/logMaintenanceUnits.ts"; +import { fullCommitShaSchema } from "../../../shared/validation.ts"; import type { ImmediateDatabaseWriteAdmission } from "../../database/immediateWriteAdmission.ts"; import { auditEvents } from "../../database/schema/auditEvents.ts"; import { jobDisableIntents } from "../../database/schema/jobDisableIntents.ts"; @@ -89,6 +91,11 @@ import { scheduledJobInsertSchema, scheduledJobSelectSchema, } from "../../database/validation/scheduledJobs.ts"; +import { parseWorkerActionKeysJson } from "../../database/validation/workerActionKeys.ts"; +import { + canonicalWorkerActionKeys, + workerActionKeysSchema, +} from "../../database/validation/workerActionKeys.ts"; import { workerInstanceInsertSchema, workerInstanceSelectSchema, @@ -160,6 +167,19 @@ export interface ReadJobHealthStateInput { readonly minimumHeartbeatAt: Date; } +export interface ReadWorkerActionAvailabilityInput { + readonly actionKeys: readonly string[]; + readonly expectedReleaseId: string; + readonly minimumHeartbeatAt: Date; +} + +/** Narrow exact-release executable-action inventory reader. */ +export interface WorkerActionAvailabilityReader { + readWorkerActionAvailability( + input: ReadWorkerActionAvailabilityInput + ): readonly string[]; +} + /** Narrow aggregate reader kept separate from the ordinary job-service repository port. */ export interface JobHealthStateReader { readHealthState(input: ReadJobHealthStateInput): JobHealthState; @@ -1159,6 +1179,47 @@ class DrizzleJobReader implements JobRepositoryReader { }; } + public readWorkerActionAvailability( + input: ReadWorkerActionAvailabilityInput + ): readonly string[] { + const actionKeys = canonicalWorkerActionKeys(input.actionKeys); + if (actionKeys.length === 0) return Object.freeze([]); + const expectedReleaseId = v.parse( + fullCommitShaSchema("Expected worker release id is invalid"), + input.expectedReleaseId + ); + const minimumHeartbeatAtMs = v.parse( + jobTimestampSchema, + getTime(input.minimumHeartbeatAt) + ); + const rows = this.database.all(sql` + SELECT DISTINCT CAST(action.value AS TEXT) AS actionKey + FROM ${workerInstances} AS worker + JOIN json_each(worker.action_keys_json) AS action + WHERE worker.state = 'online' + AND worker.release_id = ${expectedReleaseId} + AND worker.heartbeat_at >= ${minimumHeartbeatAtMs} + AND CAST(action.value AS TEXT) IN (${sql.join( + actionKeys.map((actionKey) => sql`${actionKey}`), + sql`, ` + )}) + ORDER BY actionKey ASC + LIMIT ${actionKeys.length} + `); + const parsed = v + .parse( + v.array( + v.strictObject({ + actionKey: v.string("Worker action key is invalid"), + }), + "Worker action availability is invalid" + ), + rows + ) + .map(({ actionKey }) => actionKey); + return Object.freeze([...v.parse(workerActionKeysSchema, parsed)]); + } + public readWorkerControl(): JobWorkerControlRecord { const row = this.database .select() @@ -2084,6 +2145,8 @@ class DrizzleJobWriter extends DrizzleJobReader { "worker active count" ).value; if (activeCount >= worker.capacity) return { kind: "worker-unavailable" }; + const workerActionKeys = parseWorkerActionKeysJson(worker.actionKeysJson); + if (workerActionKeys.length === 0) return { kind: "empty" }; const availableThrough = input.cursor?.availableThrough ?? input.at; const candidates: JobRunRecord[] = []; @@ -2098,6 +2161,7 @@ class DrizzleJobWriter extends DrizzleJobReader { and( eq(jobRuns.state, "queued"), lte(jobRuns.availableAt, availableThrough), + inArray(jobRuns.actionKey, workerActionKeys), range ) ) @@ -2700,7 +2764,7 @@ class DrizzleJobWriter extends DrizzleJobReader { export function createJobRepository( database: SQLiteBunDatabase, writeAdmission: ImmediateDatabaseWriteAdmission -): JobRepository & JobHealthStateReader { +): JobRepository & JobHealthStateReader & WorkerActionAvailabilityReader { // Drizzle exposes the synchronous transaction overload through a conditional // return type that cannot preserve our generic callback without this narrowing. const runTransaction = database.transaction.bind(database) as unknown as ( @@ -2783,6 +2847,8 @@ export function createJobRepository( readQueueState: (input: ReadQueueStateInput) => read((reader) => reader.readQueueState(input)), readWorkerControl: () => read((reader) => reader.readWorkerControl()), + readWorkerActionAvailability: (input: ReadWorkerActionAvailabilityInput) => + read((reader) => reader.readWorkerActionAvailability(input)), reconcileSchedules: (input: ReconcileSchedulesInput) => write((writer) => writer.reconcileSchedules(input)), recoverExpiredClaims: (input: RecoverExpiredClaimsInput) => diff --git a/greenfield/src/server/domains/jobs/service.test.ts b/greenfield/src/server/domains/jobs/service.test.ts index 511740f51..dcba23a99 100644 --- a/greenfield/src/server/domains/jobs/service.test.ts +++ b/greenfield/src/server/domains/jobs/service.test.ts @@ -277,6 +277,7 @@ describe("durable jobs service", () => { await repository.registerWorker({ ...noSideEffects, worker: { + actionKeysJson: '["system.worker-smoke"]', capacity: 1, drainingAt: null, heartbeatAt: transitionAt, @@ -1204,6 +1205,7 @@ describe("durable jobs service", () => { id: string, heartbeatAt: Date ): WorkerInstanceRecord => ({ + actionKeysJson: "[]", capacity: 1, drainingAt: null, heartbeatAt, diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts new file mode 100644 index 000000000..331c085ea --- /dev/null +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, test } from "bun:test"; + +import { + type ServiceActionId, + serviceActionIds, +} from "../../../contracts/serviceActions.ts"; +import type { JobUnscheduledActionDefinition } from "./actionRegistry.ts"; +import type { JobRunRecord } from "./records.ts"; +import type { EnqueueManualRunInput, EnqueueManualRunResult } from "./repository.ts"; +import { + createServiceActionQueue, + serviceActionJobActionKeys, + ServiceActionQueueError, + type ServiceActionQueueDependencies, + type ServiceActionQueueRequest, +} from "./serviceActionQueue.ts"; + +const actor = Object.freeze({ + authenticatorId: "019fdf50-0000-7000-8000-000000000010", + id: "019fdf50-0000-7000-8000-000000000011", + kind: "user" as const, +}); +const idempotencyKey = "019fdf50-0000-4000-8000-000000000012"; + +function definition(actionId: ServiceActionId): JobUnscheduledActionDefinition { + return Object.freeze({ + actionKey: serviceActionJobActionKeys[actionId], + attemptLimit: 1, + cancellationPolicy: "never", + description: `Runs ${actionId}.`, + displayName: actionId, + manualExposure: "none", + priority: 20, + resourceClass: "exclusive", + resourceKeys: Object.freeze(["host.mutation", actionId]), + retrySafe: false, + timeoutMs: 60_000, + }); +} + +const definitions = Object.freeze( + Object.fromEntries( + serviceActionIds.map((actionId) => [actionId, definition(actionId)]) + ) as Record +); + +function repositoryFixture() { + const enqueues: EnqueueManualRunInput[] = []; + const idempotencyReads: [JobRunRecord["requestedByKind"], string, string][] = []; + let stored: JobRunRecord | undefined; + const repository: ServiceActionQueueDependencies["repository"] = { + enqueueManualRun(input): Promise { + enqueues.push(input); + stored = { + ...input.run, + attemptCount: 0, + eventBytes: 0, + eventCount: 1, + payloadEventCount: 0, + stateVersion: 1, + }; + return Promise.resolve({ kind: "inserted", run: stored }); + }, + findRunByIdempotency(requestedByKind, requestedById, observedKey) { + idempotencyReads.push([requestedByKind, requestedById, observedKey]); + return stored?.requestedByKind === requestedByKind && + stored.requestedById === requestedById && + stored.idempotencyKey === observedKey + ? stored + : undefined; + }, + }; + return { + enqueues, + idempotencyReads, + repository, + run: () => stored, + setRun(run: JobRunRecord | undefined) { + stored = run; + }, + }; +} + +function request( + actionId: ServiceActionId, + overrides: Partial = {} +): ServiceActionQueueRequest { + return { + actionId, + actor, + authorizeDispatch: () => Promise.resolve(), + idempotencyKey, + requestId: "request-1", + ...overrides, + }; +} + +function queueFixture( + fixture = repositoryFixture(), + overrides: Partial = {} +) { + const ids = [ + "019fdf50-0000-7000-8000-000000000020", + "019fdf50-0000-7000-8000-000000000021", + "019fdf50-0000-7000-8000-000000000022", + "019fdf50-0000-7000-8000-000000000023", + ]; + return { + fixture, + queue: createServiceActionQueue({ + definitions, + generateId: () => ids.shift()!, + nowMs: () => 1000, + repository: fixture.repository, + ...overrides, + }), + }; +} + +describe("Service Action durable queue", () => { + for (const actionId of serviceActionIds) { + test(`queues exact empty payload and action mapping for ${actionId}`, async () => { + const wakeCalls: string[] = []; + let authorizationChecks = 0; + const { fixture, queue } = queueFixture(repositoryFixture(), { + wakeEventPump: () => { + wakeCalls.push(actionId); + }, + }); + + const result = await queue.enqueue( + request(actionId, { + authorizeDispatch: () => { + authorizationChecks += 1; + expect(fixture.enqueues).toHaveLength(0); + return Promise.resolve(); + }, + }) + ); + + expect(result).toEqual({ + actionId, + jobRunId: "019fdf50-0000-7000-8000-000000000020", + queued: true, + }); + expect(authorizationChecks).toBe(1); + expect(wakeCalls).toEqual([actionId]); + expect(fixture.enqueues).toHaveLength(1); + expect(fixture.enqueues[0]?.run).toMatchObject({ + actionKey: serviceActionJobActionKeys[actionId], + attemptLimit: 1, + cancellationPolicy: "never", + idempotencyKey, + payloadJson: "{}", + requestedById: actor.id, + requestedByKind: "user", + resourceClass: "exclusive", + retrySafe: false, + triggerType: "manual", + }); + expect(fixture.enqueues[0]?.auditEvents).toMatchObject([ + { + action: "service-actions.request.enqueue", + actorId: actor.id, + authenticatorId: actor.authenticatorId, + metadataJson: JSON.stringify({ actionId }), + outcome: "accepted", + requestId: "request-1", + }, + ]); + }); + } + + test("returns a matching replay without reauthorization or re-enqueue", async () => { + const { fixture, queue } = queueFixture(); + const first = await queue.enqueue(request("system-update")); + let authorizationChecks = 0; + + const replay = await queue.enqueue( + request("system-update", { + authorizeDispatch: () => { + authorizationChecks += 1; + return Promise.resolve(); + }, + }) + ); + + expect(replay).toEqual(first); + expect(authorizationChecks).toBe(0); + expect(fixture.enqueues).toHaveLength(1); + }); + + test("binds one idempotency key to the exact action and authenticator session", async () => { + const { fixture, queue } = queueFixture(); + await queue.enqueue(request("system-update")); + + for (const conflicting of [ + request("system-restart"), + request("system-update", { + actor: { + ...actor, + authenticatorId: "019fdf50-0000-7000-8000-000000000099", + }, + }), + ]) { + const failure = await queue + .enqueue(conflicting) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ServiceActionQueueError); + expect(failure).toMatchObject({ reason: "conflict" }); + } + expect(fixture.enqueues).toHaveLength(1); + }); + + test("reconciles a matching actor/action/payload run after enqueue throws", async () => { + const fixture = repositoryFixture(); + const originalEnqueue = fixture.repository.enqueueManualRun; + fixture.repository.enqueueManualRun = async (input) => { + await originalEnqueue(input); + throw new Error("private commit acknowledgement failure"); + }; + const wakes: string[] = []; + const { queue } = queueFixture(fixture, { + wakeEventPump: () => { + wakes.push("wake"); + }, + }); + + const result = await queue.enqueue(request("openclaw-cleanup")); + + expect(result).toEqual({ + actionId: "openclaw-cleanup", + jobRunId: "019fdf50-0000-7000-8000-000000000020", + queued: true, + }); + expect(fixture.idempotencyReads).toHaveLength(2); + expect(wakes).toEqual(["wake"]); + }); + + test("maps missing or failed enqueue readback to unknown outcome", async () => { + for (const readback of ["missing", "throws"] as const) { + const fixture = repositoryFixture(); + fixture.repository.enqueueManualRun = () => + Promise.reject(new Error("private enqueue failure")); + if (readback === "throws") { + let reads = 0; + fixture.repository.findRunByIdempotency = () => { + reads += 1; + if (reads > 1) throw new Error("private read failure"); + }; + } + const { queue } = queueFixture(fixture); + + const failure = await queue + .enqueue(request("openclaw-update")) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(ServiceActionQueueError); + expect(failure).toMatchObject({ reason: "unknown-outcome" }); + expect(String(failure)).not.toContain("private"); + } + }); + + test("maps a mismatched enqueue readback to conflict", async () => { + const fixture = repositoryFixture(); + const originalEnqueue = fixture.repository.enqueueManualRun; + fixture.repository.enqueueManualRun = async (input) => { + await originalEnqueue(input); + const committed = fixture.run(); + if (committed === undefined) throw new Error("Expected committed run"); + fixture.setRun({ ...committed, payloadJson: '{"unexpected":true}' }); + throw new Error("private enqueue failure"); + }; + const { queue } = queueFixture(fixture); + + const failure = await queue + .enqueue(request("system-restart")) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(ServiceActionQueueError); + expect(failure).toMatchObject({ reason: "conflict" }); + }); + + test("never calls the repository when dispatch authorization rejects", async () => { + const { fixture, queue } = queueFixture(); + const authorizationFailure = new Error("authorization expired"); + + const failure = await queue + .enqueue( + request("system-restart", { + authorizeDispatch: () => Promise.reject(authorizationFailure), + }) + ) + .catch((error: unknown) => error); + + expect(failure).toBe(authorizationFailure); + expect(fixture.enqueues).toEqual([]); + expect(fixture.run()).toBeUndefined(); + }); + + test("rejects unsafe injected action mappings at composition", () => { + expect(() => + createServiceActionQueue({ + definitions: { + ...definitions, + "system-update": { + ...definitions["system-update"], + actionKey: "host.system.unreviewed", + }, + }, + repository: repositoryFixture().repository, + }) + ).toThrow("Service Action definition is invalid"); + }); +}); diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.ts new file mode 100644 index 000000000..4714e14f2 --- /dev/null +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.ts @@ -0,0 +1,313 @@ +import * as v from "valibot"; + +import { + type RequestServiceActionResult, + type ServiceActionId, + serviceActionIds, +} from "../../../contracts/serviceActions.ts"; +import { parseJsonText } from "../../../shared/json.ts"; +import { sha256Hex } from "../../shared/crypto.ts"; +import { + hostSystemRestartJobActionKey, + hostSystemUpdateJobActionKey, + openClawInstallationUpdateJobActionKey, + openClawSessionsCleanupJobActionKey, + type JobUnscheduledActionDefinition, + validateJobUnscheduledActionDefinition, +} from "./actionRegistry.ts"; +import { preflightManualEnqueue } from "./manualEnqueue.ts"; +import type { JobRunRecord } from "./records.ts"; +import type { JobRepository } from "./repository.ts"; +import { createJobMutationSideEffects } from "./sideEffects.ts"; + +const emptyPayload = Object.freeze({}); +const emptyPayloadJson = JSON.stringify(emptyPayload); +const emptyPayloadSchema = v.strictObject({}); + +/** Exact worker action selected by each browser-visible Service Action. */ +export const serviceActionJobActionKeys = Object.freeze({ + "openclaw-cleanup": openClawSessionsCleanupJobActionKey, + "openclaw-update": openClawInstallationUpdateJobActionKey, + "system-restart": hostSystemRestartJobActionKey, + "system-update": hostSystemUpdateJobActionKey, +} as const satisfies Readonly>); + +export type ServiceActionQueueErrorReason = + | "conflict" + | "unavailable" + | "unknown-outcome"; + +/** Sanitized durable enqueue failure; job diagnostics remain inside Jobs. */ +export class ServiceActionQueueError extends Error { + public readonly reason: ServiceActionQueueErrorReason; + + public constructor(reason: ServiceActionQueueErrorReason) { + super("Service Action queue failed"); + this.name = "ServiceActionQueueError"; + this.reason = reason; + } +} + +export interface ServiceActionQueueActor { + readonly authenticatorId: string; + readonly id: string; + readonly kind: "user"; +} + +export interface ServiceActionQueueRequest { + readonly actionId: ServiceActionId; + readonly actor: ServiceActionQueueActor; + readonly authorizeDispatch: () => Promise; + readonly idempotencyKey: string; + readonly requestId: string; + readonly signal?: AbortSignal; +} + +export interface ServiceActionQueue { + readonly enqueue: ( + request: ServiceActionQueueRequest + ) => Promise; +} + +export interface ServiceActionQueueDependencies { + readonly definitions: Readonly< + Record + >; + readonly generateId?: () => string; + readonly nowMs?: () => number; + readonly repository: Pick; + readonly wakeEventPump?: () => Promise | void; +} + +interface PreparedServiceAction { + readonly definition: JobUnscheduledActionDefinition; + readonly enqueueSha256: string; +} + +function enqueueDigest(actionKey: string, authenticatorId: string): string { + return sha256Hex( + JSON.stringify({ + actionKey, + authenticatorId, + payload: emptyPayload, + version: 1, + }) + ); +} + +function prepareDefinitions( + definitions: ServiceActionQueueDependencies["definitions"] +): Readonly> { + return Object.freeze( + Object.fromEntries( + serviceActionIds.map((actionId) => { + const definition = validateJobUnscheduledActionDefinition( + definitions[actionId] + ); + if ( + definition.actionKey !== serviceActionJobActionKeys[actionId] || + definition.manualExposure !== "none" + ) { + throw new TypeError("Service Action definition is invalid"); + } + return [actionId, definition]; + }) + ) as Record + ); +} + +function matchingRun( + run: JobRunRecord | undefined, + request: ServiceActionQueueRequest, + prepared: PreparedServiceAction +): JobRunRecord | undefined { + if ( + run === undefined || + run.actionKey !== prepared.definition.actionKey || + run.enqueueSha256 !== prepared.enqueueSha256 || + run.idempotencyKey !== request.idempotencyKey || + run.requestedById !== request.actor.id || + run.requestedByKind !== request.actor.kind + ) { + return undefined; + } + try { + const payload = v.safeParse(emptyPayloadSchema, parseJsonText(run.payloadJson)); + return payload.success ? run : undefined; + } catch { + return undefined; + } +} + +function result( + actionId: ServiceActionId, + run: JobRunRecord +): RequestServiceActionResult { + return Object.freeze({ actionId, jobRunId: run.id, queued: true }); +} + +/** + * Creates the actor- and authenticator-bound durable queue for four exact Service Actions. + * The queue returns after durable admission and never waits for worker settlement. + * @returns The purpose-built fixed-action enqueue boundary. + */ +export function createServiceActionQueue( + dependencies: ServiceActionQueueDependencies +): ServiceActionQueue { + const definitions = prepareDefinitions(dependencies.definitions); + const generateId = dependencies.generateId ?? (() => Bun.randomUUIDv7()); + const nowMs = dependencies.nowMs ?? Date.now; + + async function wakeQueuedRun(run: JobRunRecord): Promise { + if (run.state !== "queued") return; + try { + await dependencies.wakeEventPump?.(); + } catch { + // The durable run remains authoritative for worker polling. + } + } + + return Object.freeze({ + async enqueue( + request: ServiceActionQueueRequest + ): Promise { + request.signal?.throwIfAborted(); + const definition = definitions[request.actionId]; + const prepared = Object.freeze({ + definition, + enqueueSha256: enqueueDigest( + definition.actionKey, + request.actor.authenticatorId + ), + }); + let replay: ReturnType; + try { + replay = preflightManualEnqueue(dependencies.repository, { + enqueueSha256: prepared.enqueueSha256, + idempotencyKey: request.idempotencyKey, + requestedById: request.actor.id, + requestedByKind: request.actor.kind, + }); + } catch { + throw new ServiceActionQueueError("unavailable"); + } + if (replay.kind === "idempotency-mismatch") { + throw new ServiceActionQueueError("conflict"); + } + if (replay.kind === "replayed") { + const run = matchingRun(replay.run, request, prepared); + if (run === undefined) throw new ServiceActionQueueError("conflict"); + return result(request.actionId, run); + } + + const atMs = nowMs(); + if (!Number.isSafeInteger(atMs) || atMs < 0) { + throw new ServiceActionQueueError("unavailable"); + } + const at = new Date(atMs); + const runId = generateId(); + const sideEffects = createJobMutationSideEffects({ + action: "service-actions.request.enqueue", + actor: request.actor, + auditId: generateId(), + metadata: { actionId: request.actionId }, + occurredAt: at, + outcome: "accepted", + realtime: { id: runId, kind: "run", operation: "created" }, + requestId: request.requestId, + targetId: runId, + targetType: "job-run", + }); + const enqueueInput = Object.freeze({ + ...sideEffects, + queuedEvent: { + attempt: 0, + jobRunId: runId, + kind: "queued" as const, + message: null, + occurredAt: at, + progressJson: null, + sequence: 1, + workerInstanceId: null, + }, + rejectWhenActionActive: true, + run: { + actionKey: definition.actionKey, + attemptLimit: definition.attemptLimit, + availableAt: at, + cancellationPolicy: definition.cancellationPolicy, + cancelRequestedAt: null, + cancelRequestedById: null, + cancelRequestedByKind: null, + displayName: definition.displayName, + enqueueSha256: prepared.enqueueSha256, + finishedAt: null, + firstStartedAt: null, + heartbeatAt: null, + id: runId, + idempotencyKey: request.idempotencyKey, + lastAttemptStartedAt: null, + leaseExpiresAt: null, + leaseOwnerId: null, + leaseToken: null, + payloadJson: emptyPayloadJson, + priority: definition.priority, + queuedAt: at, + requestedById: request.actor.id, + requestedByKind: request.actor.kind, + resourceClass: definition.resourceClass, + resourceKeysJson: JSON.stringify(definition.resourceKeys), + resultJson: null, + retrySafe: definition.retrySafe, + scheduledForAt: null, + scheduledJobId: null, + scheduledJobVersion: null, + state: "queued" as const, + terminalCode: null, + terminalMessage: null, + timeoutMs: definition.timeoutMs, + triggerType: "manual" as const, + updatedAt: at, + }, + }); + + request.signal?.throwIfAborted(); + await request.authorizeDispatch(); + request.signal?.throwIfAborted(); + + let enqueued: Awaited>; + try { + enqueued = await dependencies.repository.enqueueManualRun(enqueueInput); + } catch { + let recovered: JobRunRecord | undefined; + try { + recovered = dependencies.repository.findRunByIdempotency( + request.actor.kind, + request.actor.id, + request.idempotencyKey + ); + } catch { + throw new ServiceActionQueueError("unknown-outcome"); + } + if (recovered === undefined) { + throw new ServiceActionQueueError("unknown-outcome"); + } + const run = matchingRun(recovered, request, prepared); + if (run === undefined) throw new ServiceActionQueueError("conflict"); + await wakeQueuedRun(run); + return result(request.actionId, run); + } + + if (enqueued.kind === "idempotency-mismatch" || enqueued.kind === "active") { + throw new ServiceActionQueueError("conflict"); + } + if (enqueued.kind === "action-unavailable") { + throw new ServiceActionQueueError("unavailable"); + } + const run = matchingRun(enqueued.run, request, prepared); + if (run === undefined) throw new ServiceActionQueueError("unknown-outcome"); + if (enqueued.kind === "inserted") await wakeQueuedRun(run); + return result(request.actionId, run); + }, + }); +} diff --git a/greenfield/src/server/domains/jobs/workerRuntime.test.ts b/greenfield/src/server/domains/jobs/workerRuntime.test.ts index deda425c2..5a8de46a4 100644 --- a/greenfield/src/server/domains/jobs/workerRuntime.test.ts +++ b/greenfield/src/server/domains/jobs/workerRuntime.test.ts @@ -36,6 +36,25 @@ const baseRuntimeOptions = { openClawGateway: Object.freeze({ restart: () => Promise.resolve(), }), + openClawServiceActions: Object.freeze({ + cleanupSessions: () => + Promise.resolve({ + artifactsRemoved: 0, + bytesFreed: 0, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + status: "completed" as const, + storesProcessed: 0, + }), + updateInstallation: () => Promise.resolve({ status: "accepted" as const }), + }), pid: 123, releaseId: "a".repeat(40), sideEffects: { @@ -100,6 +119,8 @@ function runtimeFixture(initializationFailure?: Error) { }); let dieNotificationLoop: ((error: unknown) => void) | undefined; const persistentGatewayTransport = Object.freeze({ + requestOpenClawServiceAction: () => + Promise.reject(new Error("OpenClaw operations are unavailable in fixture")), start() { events.push("gateway-start"); }, @@ -152,6 +173,8 @@ function runtimeFixture(initializationFailure?: Error) { resourceClass: "exclusive", retrySafe: false, }); + expect(options.findAction?.("openclaw.sessions.cleanup")).toBeDefined(); + expect(options.findAction?.("openclaw.installation.update")).toBeDefined(); return coordinator; }, createDatabaseRuntime() { @@ -333,6 +356,44 @@ describe("Dashboard worker runtime", () => { ); }); + test("registers only fixed host operations reported available at worker startup", async () => { + const fixture = runtimeFixture(); + let requests = 0; + const options: DashboardWorkerRuntimeOptions = { + ...fixture.options, + hostOperations: { + availableOperations: () => Promise.resolve(["system-restart"]), + request: () => { + requests += 1; + return Promise.resolve({ status: "accepted" }); + }, + }, + }; + const dependencies: DashboardWorkerRuntimeDependencies = { + ...fixture.dependencies, + createCoordinator(coordinatorOptions) { + expect( + coordinatorOptions.findAction?.("host.system.restart") + ).toBeDefined(); + expect( + coordinatorOptions.findAction?.("host.system.update") + ).toBeUndefined(); + expect( + coordinatorOptions.actionDefinitions?.map( + ({ actionKey }) => actionKey + ) + ).toContain("host.system.restart"); + return fixture.dependencies.createCoordinator(coordinatorOptions); + }, + }; + const runtime = createDashboardWorkerRuntime(options, dependencies); + + await runtime.initialize(); + await runtime.dispose(); + + expect(requests).toBe(0); + }); + test("forces teardown when durable notification release stalls", async () => { const fixture = runtimeFixture(); const retryStarted = deferred(); diff --git a/greenfield/src/server/domains/jobs/workerRuntime.ts b/greenfield/src/server/domains/jobs/workerRuntime.ts index c8080065e..0f84b8659 100644 --- a/greenfield/src/server/domains/jobs/workerRuntime.ts +++ b/greenfield/src/server/domains/jobs/workerRuntime.ts @@ -1,6 +1,11 @@ import { Cause, Effect, Exit, Fiber, ManagedRuntime } from "effect"; +import { + hostOperationIds, + type FixedHostOperationsExecutionPort, +} from "../../../shared/hostOperations.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; +import type { OpenClawServiceActionsExecutionPort } from "../../../shared/openClawServiceActions.ts"; import type { TaskNotificationChatSender, TaskNotificationQueue, @@ -24,7 +29,11 @@ import { } from "./actionExecutors.ts"; import { jobActionDefinitions, + hostSystemRestartJobActionDefinition, + hostSystemUpdateJobActionDefinition, openClawGatewayRestartJobActionDefinition, + openClawInstallationUpdateJobActionDefinition, + openClawSessionsCleanupJobActionDefinition, workspaceFileReplaceJobActionDefinition, workspaceFileWriteJobActionDefinition, } from "./actionRegistry.ts"; @@ -43,8 +52,10 @@ import { export interface DashboardWorkerRuntimeOptions { readonly database: DatabaseRuntimeLayerOptions; readonly logMaintenance: LogMaintenanceExecutionPort; + readonly hostOperations?: FixedHostOperationsExecutionPort; readonly moltbook: MoltbookDashboardCollector; - readonly openClawGateway: OpenClawGatewayLifecycleExecutionPort; + readonly openClawGateway?: OpenClawGatewayLifecycleExecutionPort; + readonly openClawServiceActions?: OpenClawServiceActionsExecutionPort; readonly workspaceFiles?: WorkspaceFileWriteExecutionPort & { readonly dispose: () => Promise | void; }; @@ -404,9 +415,36 @@ export function createDashboardWorkerRuntime( database.database, database.writeAdmission ); + const availableHostOperations = + (await options.hostOperations?.availableOperations()) ?? []; + if ( + availableHostOperations.length > hostOperationIds.length || + new Set(availableHostOperations).size !== + availableHostOperations.length || + availableHostOperations.some( + (operationId) => !hostOperationIds.includes(operationId) + ) + ) { + throw new Error("Fixed host operation availability is invalid"); + } + const availableHostOperationSet = new Set(availableHostOperations); const actionDefinitions = Object.freeze([ ...jobActionDefinitions, - openClawGatewayRestartJobActionDefinition, + ...(options.openClawGateway === undefined + ? [] + : [openClawGatewayRestartJobActionDefinition]), + ...(options.openClawServiceActions === undefined + ? [] + : [ + openClawSessionsCleanupJobActionDefinition, + openClawInstallationUpdateJobActionDefinition, + ]), + ...(availableHostOperationSet.has("system-restart") + ? [hostSystemRestartJobActionDefinition] + : []), + ...(availableHostOperationSet.has("system-update") + ? [hostSystemUpdateJobActionDefinition] + : []), ...(options.workspaceFiles === undefined ? [] : [ @@ -416,9 +454,18 @@ export function createDashboardWorkerRuntime( ]); const findAction = createJobWorkerActionResolver({ actionDefinitions, + ...(availableHostOperations.length === 0 || + options.hostOperations === undefined + ? {} + : { hostOperations: options.hostOperations }), logMaintenance: options.logMaintenance, moltbook: options.moltbook, - openClawGateway: options.openClawGateway, + ...(options.openClawGateway === undefined + ? {} + : { openClawGateway: options.openClawGateway }), + ...(options.openClawServiceActions === undefined + ? {} + : { openClawServiceActions: options.openClawServiceActions }), ...(options.workspaceFiles === undefined ? {} : { workspaceFiles: options.workspaceFiles }), diff --git a/greenfield/src/server/domains/serviceActions/operationAudit.test.ts b/greenfield/src/server/domains/serviceActions/operationAudit.test.ts new file mode 100644 index 000000000..0be3d4f8f --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/operationAudit.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test"; + +import { asc } from "drizzle-orm"; + +import { auditEvents } from "../../database/schema/auditEvents.ts"; +import { testImmediateDatabaseWriteAdmission } from "../../test/support/databaseWriteAdmission.ts"; +import { openFreshMigratedDatabase } from "../../test/support/freshDatabase.ts"; +import { createSqliteServiceActionAuditWriter } from "./operationAudit.ts"; + +describe("service action operation audit", () => { + test("persists only fixed action, run identity, and classified settlement", async () => { + const database = await openFreshMigratedDatabase(); + const ids = [ + "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a1", + "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a2", + ]; + const writer = createSqliteServiceActionAuditWriter({ + clock: () => new Date(1000), + database: database.orm, + generateId: () => { + const id = ids.shift(); + if (id === undefined) throw new Error("Audit id budget exhausted"); + return id; + }, + writeAdmission: testImmediateDatabaseWriteAdmission, + }); + const context = { + actionId: "system-update", + actor: { + authenticatorId: "a".repeat(32), + id: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b4", + kind: "user", + }, + requestId: "request-1", + } as const; + + try { + await writer.record({ ...context, settlement: "attempted" }); + await writer.record({ + ...context, + jobRunId: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b5", + settlement: "succeeded", + }); + const rows = database.orm + .select() + .from(auditEvents) + .orderBy(asc(auditEvents.id)) + .all(); + + expect(rows).toMatchObject([ + { + action: "service-actions.system-update.request", + metadataJson: '{"settlement":"attempted"}', + outcome: "attempted", + requestId: "request-1", + targetId: "system-update", + targetType: "service-action", + }, + { + action: "service-actions.system-update.request", + metadataJson: '{"settlement":"succeeded"}', + outcome: "succeeded", + targetId: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b5", + targetType: "job-run", + }, + ]); + expect(JSON.stringify(rows)).not.toContain("apt-get"); + } finally { + database.sqlite.close(true); + } + }); +}); diff --git a/greenfield/src/server/domains/serviceActions/operationAudit.ts b/greenfield/src/server/domains/serviceActions/operationAudit.ts new file mode 100644 index 000000000..eba23fc60 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/operationAudit.ts @@ -0,0 +1,86 @@ +import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"; + +import type { ServiceActionId } from "../../../contracts/serviceActions.ts"; +import type { ImmediateDatabaseWriteAdmission } from "../../database/immediateWriteAdmission.ts"; +import { createSecurityAuditEvent } from "../security/audit.ts"; +import { DrizzleSecurityAuditStore } from "../security/securityAuditStore.ts"; + +export type ServiceActionAuditSettlement = + | "attempted" + | "failed" + | "partial" + | "succeeded"; + +export interface ServiceActionAuditContext { + readonly actor: { + readonly authenticatorId: string; + readonly id: string; + readonly kind: "user"; + }; + readonly requestId: string; +} + +export interface ServiceActionAuditEvent extends ServiceActionAuditContext { + readonly actionId: ServiceActionId; + readonly jobRunId?: string; + readonly settlement: ServiceActionAuditSettlement; +} + +/** Durable audit append port. Commands, provider results, and host details are absent. */ +export interface ServiceActionAuditWriter { + readonly record: (event: ServiceActionAuditEvent) => Promise; +} + +export interface SqliteServiceActionAuditWriterOptions { + readonly clock?: () => Date; + readonly database: SQLiteBunDatabase; + readonly generateId?: () => string; + readonly writeAdmission: ImmediateDatabaseWriteAdmission; +} + +function auditOutcome( + settlement: ServiceActionAuditSettlement +): "attempted" | "failed" | "succeeded" { + if (settlement === "attempted") return "attempted"; + if (settlement === "succeeded") return "succeeded"; + return "failed"; +} + +/** + * Creates a fail-closed admitted audit writer for fixed privileged service actions. + * @param options Database, admission, clock, and identity dependencies. + * @returns A sanitized append-only audit writer. + */ +export function createSqliteServiceActionAuditWriter({ + clock = () => new Date(), + database, + generateId = () => Bun.randomUUIDv7(), + writeAdmission, +}: SqliteServiceActionAuditWriterOptions): ServiceActionAuditWriter { + return Object.freeze({ + record(input: ServiceActionAuditEvent) { + const event = createSecurityAuditEvent({ + action: `service-actions.${input.actionId}.request`, + actor: input.actor, + id: generateId(), + metadata: { settlement: input.settlement }, + occurredAt: clock(), + outcome: auditOutcome(input.settlement), + requestId: input.requestId, + targetId: input.jobRunId ?? input.actionId, + targetType: input.jobRunId === undefined ? "service-action" : "job-run", + }); + return writeAdmission.run((markTransactionStarted) => + database.transaction( + (transaction) => { + markTransactionStarted(); + new DrizzleSecurityAuditStore(transaction).insertAuditEvent( + event + ); + }, + { behavior: "immediate" } + ) + ); + }, + }); +} diff --git a/greenfield/src/server/domains/serviceActions/procedures.test.ts b/greenfield/src/server/domains/serviceActions/procedures.test.ts new file mode 100644 index 000000000..0a8d354f8 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/procedures.test.ts @@ -0,0 +1,288 @@ +import { describe, expect, test } from "bun:test"; + +import { TRPCError } from "@trpc/server"; + +import type { + GetServiceActionsStatusResult, + RequestServiceActionInput, + RequestServiceActionResult, +} from "../../../contracts/serviceActions.ts"; +import { captureFailure } from "../../test/support/promise.ts"; +import { + createTestApplicationRuntime, + createTestAuthenticationLifecycleService, + createTestAutomationAuthentication, + createTestRequestContext, + createTestSessionAuthentication, + testSecurityUserId, + testSessionSelector, +} from "../../test/support/requestContext.ts"; +import { appRouter } from "../../trpc/appRouter.ts"; +import { + ServiceActionsServiceError, + type ServiceActionControlContext, + type ServiceActionsService, +} from "./service.ts"; + +const idempotencyKey = "A".repeat(43); +const jobRunId = "018f6f50-6a9e-7b88-8000-000000000001"; + +const statusResult = Object.freeze({ + actions: [ + { availability: "available" as const, id: "openclaw-cleanup" as const }, + { availability: "available" as const, id: "openclaw-update" as const }, + { availability: "available" as const, id: "system-restart" as const }, + { availability: "unavailable" as const, id: "system-update" as const }, + ], + observedAtMs: 1_800_000_000_000, +}) satisfies GetServiceActionsStatusResult; + +const requestInput = Object.freeze({ + actionId: "system-update" as const, + confirmation: "update-system" as const, + idempotencyKey, +}); + +const requestResult = Object.freeze({ + actionId: "system-update" as const, + jobRunId, + queued: true as const, +}) satisfies RequestServiceActionResult; + +function testService( + calls: string[], + contexts: ServiceActionControlContext[] = [] +): ServiceActionsService { + return Object.freeze({ + getStatus: () => { + calls.push("get-status"); + return Promise.resolve(statusResult); + }, + request: ( + input: RequestServiceActionInput, + context: ServiceActionControlContext + ) => { + context.reauthorize(); + contexts.push(context); + calls.push(`request:${input.actionId}:${input.idempotencyKey}`); + return Promise.resolve(requestResult); + }, + }); +} + +async function expectCode( + operation: () => Promise, + code: TRPCError["code"] +): Promise { + const failure = await captureFailure(operation); + expect(failure).toBeInstanceOf(TRPCError); + expect((failure as TRPCError).code).toBe(code); + return failure as TRPCError; +} + +describe("Service Actions procedures", () => { + test("serves bounded status and reauthorizes at the queue handoff", async () => { + const calls: string[] = []; + const contexts: ServiceActionControlContext[] = []; + let authorizationChecks = 0; + const caller = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication([ + "service-actions:read", + "service-actions:write", + ]), + createTestApplicationRuntime(), + { + authenticationLifecycle: createTestAuthenticationLifecycleService({ + authorizeRecentMfa: () => { + authorizationChecks += 1; + return "authorized"; + }, + }), + requestId: "service-actions-request-1", + serviceActionsService: testService(calls, contexts), + } + ) + ).serviceActions; + + expect(await caller.getStatus({})).toEqual(statusResult); + expect(await caller.request(requestInput)).toEqual(requestResult); + expect(calls).toEqual(["get-status", `request:system-update:${idempotencyKey}`]); + expect(authorizationChecks).toBe(2); + expect(contexts).toEqual([ + expect.objectContaining({ + actor: { + authenticatorId: testSessionSelector, + id: testSecurityUserId, + kind: "user", + }, + requestId: "service-actions-request-1", + }), + ]); + }); + + test("rejects anonymous, automation, and capability-crossed callers", async () => { + const calls: string[] = []; + const serviceActionsService = testService(calls); + const anonymous = appRouter.createCaller( + await createTestRequestContext(undefined, createTestApplicationRuntime(), { + serviceActionsService, + }) + ).serviceActions; + await expectCode(() => anonymous.getStatus({}), "UNAUTHORIZED"); + + const automation = appRouter.createCaller( + await createTestRequestContext( + createTestAutomationAuthentication([ + "service-actions:read", + "service-actions:write", + ]), + createTestApplicationRuntime(), + { serviceActionsService } + ) + ).serviceActions; + await expectCode(() => automation.getStatus({}), "FORBIDDEN"); + await expectCode(() => automation.request(requestInput), "FORBIDDEN"); + + const readOnly = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:read"]), + createTestApplicationRuntime(), + { serviceActionsService } + ) + ).serviceActions; + expect(await readOnly.getStatus({})).toEqual(statusResult); + await expectCode(() => readOnly.request(requestInput), "FORBIDDEN"); + + const writeOnly = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:write"]), + createTestApplicationRuntime(), + { serviceActionsService } + ) + ).serviceActions; + await expectCode(() => writeOnly.getStatus({}), "FORBIDDEN"); + expect(calls).toEqual(["get-status"]); + }); + + test("enforces recent MFA before service dispatch and clears changed sessions", async () => { + for (const [status, code] of [ + ["mfa-enrollment-required", "FORBIDDEN"], + ["step-up-required", "FORBIDDEN"], + ["session-changed", "UNAUTHORIZED"], + ] as const) { + const calls: string[] = []; + const responseHeaders = new Headers(); + const caller = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:write"]), + createTestApplicationRuntime(), + { + authenticationLifecycle: createTestAuthenticationLifecycleService( + { + authorizeRecentMfa: () => status, + } + ), + responseHeaders, + serviceActionsService: testService(calls), + } + ) + ).serviceActions; + + await expectCode(() => caller.request(requestInput), code); + expect(calls).toEqual([]); + expect(responseHeaders.get("set-cookie") ?? "").toContain( + status === "session-changed" ? "Max-Age=0" : "" + ); + } + }); + + test("clears a session that changes at the post-preflight handoff", async () => { + const calls: string[] = []; + const responseHeaders = new Headers(); + let authorizationChecks = 0; + const caller = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:write"]), + createTestApplicationRuntime(), + { + authenticationLifecycle: createTestAuthenticationLifecycleService({ + authorizeRecentMfa: () => + authorizationChecks++ === 0 + ? "authorized" + : "session-changed", + }), + responseHeaders, + serviceActionsService: testService(calls), + } + ) + ).serviceActions; + + await expectCode(() => caller.request(requestInput), "UNAUTHORIZED"); + expect(authorizationChecks).toBe(2); + expect(calls).toEqual([]); + expect(responseHeaders.get("set-cookie")).toContain("Max-Age=0"); + }); + + test("maps only fixed conflict, unavailable, audit, and unknown-outcome errors", async () => { + for (const [reason, code, expectedMessage] of [ + [ + "conflict", + "CONFLICT", + "Service action request conflicts with an existing intent", + ], + [ + "audit-unavailable", + "SERVICE_UNAVAILABLE", + "Service actions are temporarily unavailable", + ], + [ + "unavailable", + "SERVICE_UNAVAILABLE", + "Service actions are temporarily unavailable", + ], + [ + "unknown-outcome", + "SERVICE_UNAVAILABLE", + "Service action queue outcome could not be confirmed", + ], + ] as const) { + const caller = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:write"]), + createTestApplicationRuntime(), + { + serviceActionsService: Object.freeze({ + getStatus: () => Promise.resolve(statusResult), + request: () => + Promise.reject( + new ServiceActionsServiceError(reason, { + cause: new Error("private host detail"), + }) + ), + }), + } + ) + ).serviceActions; + + const failure = await expectCode(() => caller.request(requestInput), code); + expect(failure.message).toBe(expectedMessage); + expect(failure.message).not.toContain("private host detail"); + } + + const statusCaller = appRouter.createCaller( + await createTestRequestContext( + createTestSessionAuthentication(["service-actions:read"]), + createTestApplicationRuntime(), + { + serviceActionsService: Object.freeze({ + getStatus: () => + Promise.reject(new ServiceActionsServiceError("unavailable")), + request: () => Promise.resolve(requestResult), + }), + } + ) + ).serviceActions; + await expectCode(() => statusCaller.getStatus({}), "SERVICE_UNAVAILABLE"); + }); +}); diff --git a/greenfield/src/server/domains/serviceActions/procedures.ts b/greenfield/src/server/domains/serviceActions/procedures.ts new file mode 100644 index 000000000..e81336a44 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/procedures.ts @@ -0,0 +1,10 @@ +import { router } from "../../trpc/trpc.ts"; +import { serviceActionsRoutes } from "./routes.ts"; + +/** Leaf procedure names owned by the fixed Service Actions router. */ +export const serviceActionsProcedureNames = Object.freeze( + Object.keys(serviceActionsRoutes) +); + +/** Session-only status and recent-MFA fixed-operation queue controls. */ +export const serviceActionsRouter = router(serviceActionsRoutes); diff --git a/greenfield/src/server/domains/serviceActions/routes.ts b/greenfield/src/server/domains/serviceActions/routes.ts new file mode 100644 index 000000000..7e560af78 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/routes.ts @@ -0,0 +1,125 @@ +import { TRPCError } from "@trpc/server"; + +import { + getServiceActionsStatusInputSchema, + getServiceActionsStatusResultSchema, + requestServiceActionInputSchema, + requestServiceActionResultSchema, +} from "../../../contracts/serviceActions.ts"; +import { appendClearedDashboardSessionCookie } from "../../rawHttp/sessionCookie.ts"; +import type { RequestContext } from "../../trpc/context.ts"; +import { + authenticationPolicyError, + operationOutcomeUnknownError, + sessionCapabilityProcedure, +} from "../../trpc/trpc.ts"; +import type { AuthenticatedBrowserIdentity } from "../security/authenticationSession.ts"; +import { sessionActor } from "../security/authenticationSession.ts"; +import { + type ServiceActionControlContext, + ServiceActionsServiceError, +} from "./service.ts"; + +function authorizeControl( + context: RequestContext & { + readonly sessionIdentity: AuthenticatedBrowserIdentity; + } +): void { + const status = context.authenticationLifecycle.authorizeRecentMfa( + context.sessionIdentity + ); + switch (status) { + case "authorized": { + return; + } + case "mfa-enrollment-required": { + throw authenticationPolicyError( + "mfa_enrollment_required", + "Multi-factor authentication enrollment is required" + ); + } + case "step-up-required": { + throw authenticationPolicyError( + "step_up_required", + "Recent multi-factor authentication is required" + ); + } + case "session-changed": { + appendClearedDashboardSessionCookie(context.responseHeaders); + throw new TRPCError({ + code: "UNAUTHORIZED", + message: "Authentication state changed; sign in again", + }); + } + } +} + +function controlContext( + context: RequestContext & { + readonly sessionIdentity: AuthenticatedBrowserIdentity; + } +): ServiceActionControlContext { + return { + actor: sessionActor(context.sessionIdentity), + reauthorize: () => authorizeControl(context), + requestId: context.requestId, + }; +} + +function throwServiceFailure(error: unknown): never { + if (!(error instanceof ServiceActionsServiceError)) throw error; + switch (error.reason) { + case "conflict": { + throw new TRPCError({ + cause: error, + code: "CONFLICT", + message: "Service action request conflicts with an existing intent", + }); + } + case "unknown-outcome": { + throw operationOutcomeUnknownError( + "Service action queue outcome could not be confirmed" + ); + } + case "audit-unavailable": + case "unavailable": { + throw new TRPCError({ + cause: error, + code: "SERVICE_UNAVAILABLE", + message: "Service actions are temporarily unavailable", + }); + } + } +} + +const readProcedure = sessionCapabilityProcedure("service-actions:read"); +const controlProcedure = sessionCapabilityProcedure("service-actions:write"); + +/** Session-only fixed service-action status and recent-MFA queue controls. */ +export const serviceActionsRoutes = { + getStatus: readProcedure + .input(getServiceActionsStatusInputSchema) + .output(getServiceActionsStatusResultSchema) + .query(async ({ ctx, signal }) => { + try { + return await ctx.serviceActionsService.getStatus(signal); + } catch (error) { + return throwServiceFailure(error); + } + }), + request: controlProcedure + .input(requestServiceActionInputSchema) + .output(requestServiceActionResultSchema) + .mutation(async ({ ctx, input, signal }) => { + authorizeControl(ctx); + try { + return await ctx.serviceActionsService.request( + input, + controlContext(ctx), + signal + ); + } catch (error) { + return throwServiceFailure(error); + } + }), +}; diff --git a/greenfield/src/server/domains/serviceActions/service.test.ts b/greenfield/src/server/domains/serviceActions/service.test.ts new file mode 100644 index 000000000..2aeef3c43 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/service.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, test } from "bun:test"; + +import { ServiceActionQueueError } from "../jobs/serviceActionQueue.ts"; +import type { ServiceActionAuditEvent } from "./operationAudit.ts"; +import { createServiceActionsService, ServiceActionsServiceError } from "./service.ts"; + +const actor = Object.freeze({ + authenticatorId: "a".repeat(32), + id: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b4", + kind: "user" as const, +}); +const jobRunId = "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b5"; +const input = Object.freeze({ + actionId: "system-update" as const, + confirmation: "update-system" as const, + idempotencyKey: "A".repeat(43), +}); +const queuedResult = Object.freeze({ + actionId: input.actionId, + jobRunId, + queued: true as const, +}); + +function queuedRun(id: string) { + return { + actionKey: "host.system.update", + attemptCount: 0, + attemptLimit: 1, + availableAtMs: 1000, + cancellationPolicy: "never" as const, + displayName: "Update host system", + eventCount: 1, + id, + priority: 20, + queuedAtMs: 1000, + resourceClass: "exclusive" as const, + resourceKeys: ["host.mutation"], + retrySafe: false, + state: "queued" as const, + stateVersion: 1, + timeoutMs: 7_200_000, + triggerType: "manual" as const, + updatedAtMs: 1000, + }; +} + +function fixture( + options: { + readonly auditFailure?: "attempted" | "failed" | "partial" | "succeeded"; + readonly queue?: Parameters[0]["queue"]; + readonly statuses?: Parameters< + typeof createServiceActionsService + >[0]["statusReader"]; + } = {} +) { + const auditEvents: ServiceActionAuditEvent[] = []; + const settlementFailures: string[] = []; + let reauthorizations = 0; + const service = createServiceActionsService({ + auditWriter: { + record: (event) => { + if (event.settlement === options.auditFailure) { + return Promise.reject(new Error("private audit failure")); + } + auditEvents.push(event); + return Promise.resolve(); + }, + }, + nowMs: () => 2000, + onAuditSettlementFailure: ({ settlement }) => { + settlementFailures.push(settlement); + }, + queue: options.queue ?? { + enqueue: async (request) => { + await request.authorizeDispatch(); + return queuedResult; + }, + }, + statusReader: options.statuses ?? { + read: () => + Promise.resolve([ + { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-update" }, + { availability: "unavailable", id: "system-restart" }, + { + activeRun: queuedRun(jobRunId), + availability: "available", + id: "system-update", + }, + ]), + }, + }); + return { + auditEvents, + context: { + actor, + reauthorize: () => { + reauthorizations += 1; + }, + requestId: "request-1", + }, + reauthorizations: () => reauthorizations, + service, + settlementFailures, + }; +} + +async function captureFailure(work: () => Promise): Promise { + try { + await work(); + } catch (error) { + return error; + } + throw new Error("Expected work to fail"); +} + +describe("service actions service", () => { + test("projects the exact bounded status inventory", async () => { + const result = await fixture().service.getStatus(); + expect(result).toMatchObject({ + actions: [ + { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-update" }, + { availability: "unavailable", id: "system-restart" }, + { + activeRun: { id: jobRunId, state: "queued" }, + availability: "available", + id: "system-update", + }, + ], + observedAtMs: 2000, + }); + }); + + test("records attempted before reauthorization/enqueue and links the queued run", async () => { + const order: string[] = []; + const auditEvents: ServiceActionAuditEvent[] = []; + const service = createServiceActionsService({ + auditWriter: { + record: (event) => { + order.push(`audit:${event.settlement}`); + auditEvents.push(event); + return Promise.resolve(); + }, + }, + queue: { + enqueue: async (request) => { + order.push("queue:preflight"); + await request.authorizeDispatch(); + order.push("queue:enqueue"); + return queuedResult; + }, + }, + statusReader: { + read: () => + Promise.resolve([ + { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-update" }, + { availability: "available", id: "system-restart" }, + { availability: "available", id: "system-update" }, + ]), + }, + }); + const result = await service.request(input, { + actor, + reauthorize: () => order.push("authorize"), + requestId: "request-1", + }); + + expect(result).toEqual({ + actionId: "system-update", + jobRunId, + queued: true, + }); + expect(order).toEqual([ + "audit:attempted", + "queue:preflight", + "authorize", + "queue:enqueue", + "audit:succeeded", + ]); + expect(auditEvents[1]).toMatchObject({ jobRunId, settlement: "succeeded" }); + }); + + test("fails closed before queue work when attempted audit fails", async () => { + let queueCalled = false; + const state = fixture({ + auditFailure: "attempted", + queue: { + enqueue: () => { + queueCalled = true; + return Promise.resolve(queuedResult); + }, + }, + }); + expect( + await captureFailure(() => state.service.request(input, state.context)) + ).toMatchObject({ reason: "audit-unavailable" }); + expect(queueCalled).toBeFalse(); + }); + + test("rejects unavailable actions before the final authorization handoff", async () => { + let durableEnqueue = false; + const state = fixture({ + queue: { + enqueue: async (request) => { + await request.authorizeDispatch(); + durableEnqueue = true; + return queuedResult; + }, + }, + statuses: { + read: () => + Promise.resolve([ + { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-update" }, + { availability: "available", id: "system-restart" }, + { availability: "unavailable", id: "system-update" }, + ]), + }, + }); + + expect( + await captureFailure(() => state.service.request(input, state.context)) + ).toMatchObject({ reason: "unavailable" }); + expect(durableEnqueue).toBeFalse(); + expect(state.reauthorizations()).toBe(0); + expect(state.auditEvents.map(({ settlement }) => settlement)).toEqual([ + "attempted", + "failed", + ]); + }); + + test("preserves a reauthorization rejection even when the queue swallows it", async () => { + const authorizationError = new Error("authorization changed"); + const state = fixture({ + queue: { + enqueue: async (request) => { + await request.authorizeDispatch().catch(() => {}); + return queuedResult; + }, + }, + }); + const failure = await captureFailure(() => + state.service.request(input, { + ...state.context, + reauthorize: () => { + throw authorizationError; + }, + }) + ); + expect(failure).toBe(authorizationError); + expect(state.auditEvents.map(({ settlement }) => settlement)).toEqual([ + "attempted", + "failed", + ]); + }); + + test("classifies unknown queue outcome as partial without leaking its cause", async () => { + const state = fixture({ + queue: { + enqueue: () => + Promise.reject(new ServiceActionQueueError("unknown-outcome")), + }, + }); + const failure = await captureFailure(() => + state.service.request(input, state.context) + ); + expect(failure).toBeInstanceOf(ServiceActionsServiceError); + expect(failure).toMatchObject({ reason: "unknown-outcome" }); + expect(state.auditEvents.map(({ settlement }) => settlement)).toEqual([ + "attempted", + "partial", + ]); + expect(JSON.stringify(failure)).not.toContain("systemctl"); + }); + + test("does not replace a confirmed queued result when settlement audit fails", async () => { + const state = fixture({ auditFailure: "succeeded" }); + expect(await state.service.request(input, state.context)).toMatchObject({ + jobRunId, + queued: true, + }); + expect(state.settlementFailures).toEqual(["succeeded"]); + expect(state.reauthorizations()).toBe(1); + }); +}); diff --git a/greenfield/src/server/domains/serviceActions/service.ts b/greenfield/src/server/domains/serviceActions/service.ts new file mode 100644 index 000000000..1e46942ec --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/service.ts @@ -0,0 +1,225 @@ +import * as v from "valibot"; + +import { + type GetServiceActionsStatusResult, + type RequestServiceActionInput, + type RequestServiceActionResult, + type ServiceActionStatus, + getServiceActionsStatusResultSchema, + requestServiceActionInputSchema, + requestServiceActionResultSchema, +} from "../../../contracts/serviceActions.ts"; +import { + ServiceActionQueueError, + type ServiceActionQueue, +} from "../jobs/serviceActionQueue.ts"; +import { + type ServiceActionAuditContext, + type ServiceActionAuditSettlement, + type ServiceActionAuditWriter, +} from "./operationAudit.ts"; + +export type ServiceActionsServiceErrorReason = + | "audit-unavailable" + | "conflict" + | "unavailable" + | "unknown-outcome"; + +/** Sanitized domain failure without commands, provider results, or host diagnostics. */ +export class ServiceActionsServiceError extends Error { + readonly reason: ServiceActionsServiceErrorReason; + + constructor(reason: ServiceActionsServiceErrorReason, options?: ErrorOptions) { + super("Service action operation failed", options); + this.name = "ServiceActionsServiceError"; + this.reason = reason; + } +} + +export interface ServiceActionControlContext extends ServiceActionAuditContext { + /** Re-checks the current session and recent MFA at durable enqueue handoff. */ + readonly reauthorize: () => void; +} + +export interface ServiceActionStatusReader { + readonly read: (signal?: AbortSignal) => Promise; +} + +export interface ServiceActionsService { + readonly getStatus: (signal?: AbortSignal) => Promise; + readonly request: ( + input: RequestServiceActionInput, + context: ServiceActionControlContext, + signal?: AbortSignal + ) => Promise; +} + +export interface ServiceActionsServiceOptions { + readonly auditWriter: ServiceActionAuditWriter; + readonly nowMs?: () => number; + readonly onAuditSettlementFailure?: (failure: { + readonly actionId: RequestServiceActionInput["actionId"]; + readonly cause: unknown; + readonly settlement: Exclude; + }) => void; + readonly queue: ServiceActionQueue; + readonly statusReader: ServiceActionStatusReader; +} + +function queueFailure(error: ServiceActionQueueError): ServiceActionsServiceError { + return new ServiceActionsServiceError(error.reason, { cause: error }); +} + +function validNowMs(nowMs: () => number): number { + const value = nowMs(); + if (!Number.isSafeInteger(value) || value < 0) { + throw new ServiceActionsServiceError("unavailable"); + } + return value; +} + +/** + * Creates status reads and fail-closed audited enqueue controls for fixed service actions. + * @param options Queue, status, audit, clock, and settlement-observation boundaries. + * @returns A sanitized Service Actions domain service. + */ +export function createServiceActionsService( + options: ServiceActionsServiceOptions +): ServiceActionsService { + const nowMs = options.nowMs ?? Date.now; + + async function recordAttempt( + input: RequestServiceActionInput, + context: ServiceActionAuditContext + ): Promise { + try { + await options.auditWriter.record({ + ...context, + actionId: input.actionId, + settlement: "attempted", + }); + } catch (error) { + throw new ServiceActionsServiceError("audit-unavailable", { cause: error }); + } + } + + async function settleAudit( + input: RequestServiceActionInput, + context: ServiceActionAuditContext, + settlement: Exclude, + jobRunId?: string + ): Promise { + try { + await options.auditWriter.record({ + ...context, + actionId: input.actionId, + ...(jobRunId === undefined ? {} : { jobRunId }), + settlement, + }); + } catch (cause) { + try { + options.onAuditSettlementFailure?.({ + actionId: input.actionId, + cause, + settlement, + }); + } catch { + // Operational observation cannot replace an already-known queue result. + } + } + } + + async function getStatus( + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted(); + try { + const actions = await options.statusReader.read(signal); + signal?.throwIfAborted(); + return v.parse(getServiceActionsStatusResultSchema, { + actions, + observedAtMs: validNowMs(nowMs), + }); + } catch (error) { + if (signal?.aborted) throw error; + if (error instanceof ServiceActionsServiceError) throw error; + throw new ServiceActionsServiceError("unavailable", { cause: error }); + } + } + + async function request( + input: RequestServiceActionInput, + context: ServiceActionControlContext, + signal?: AbortSignal + ): Promise { + const parsed = v.parse(requestServiceActionInputSchema, input); + signal?.throwIfAborted(); + await recordAttempt(parsed, context); + let authorizationFailed = false; + let authorizationFailure: unknown; + try { + const result = await options.queue.enqueue({ + actionId: parsed.actionId, + actor: context.actor, + authorizeDispatch: async () => { + signal?.throwIfAborted(); + const statuses = await options.statusReader.read(signal); + if ( + statuses.find(({ id }) => id === parsed.actionId) + ?.availability !== "available" + ) { + throw new ServiceActionsServiceError("unavailable"); + } + signal?.throwIfAborted(); + try { + context.reauthorize(); + signal?.throwIfAborted(); + } catch (error) { + authorizationFailed = true; + authorizationFailure = error; + throw error; + } + }, + idempotencyKey: parsed.idempotencyKey, + requestId: context.requestId, + ...(signal === undefined ? {} : { signal }), + }); + if (authorizationFailed) throw authorizationFailure; + const output = v.parse(requestServiceActionResultSchema, { + actionId: parsed.actionId, + jobRunId: result.jobRunId, + queued: true, + }); + await settleAudit(parsed, context, "succeeded", output.jobRunId); + return output; + } catch (error) { + if (authorizationFailed && error === authorizationFailure) { + await settleAudit(parsed, context, "failed"); + throw error; + } + const mapped = + error instanceof ServiceActionQueueError + ? queueFailure(error) + : error instanceof v.ValiError + ? new ServiceActionsServiceError("unknown-outcome", { + cause: error, + }) + : signal?.aborted + ? error + : new ServiceActionsServiceError("unavailable", { + cause: error, + }); + await settleAudit( + parsed, + context, + mapped instanceof ServiceActionsServiceError && + mapped.reason === "unknown-outcome" + ? "partial" + : "failed" + ); + throw mapped; + } + } + + return Object.freeze({ getStatus, request }); +} diff --git a/greenfield/src/server/domains/serviceActions/statusReader.test.ts b/greenfield/src/server/domains/serviceActions/statusReader.test.ts new file mode 100644 index 000000000..5f533e566 --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/statusReader.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; + +import type { JobRunRecord } from "../jobs/records.ts"; +import { serviceActionJobActionKeys } from "../jobs/serviceActionQueue.ts"; +import { createSqliteServiceActionStatusReader } from "./statusReader.ts"; + +const expectedReleaseId = "a".repeat(40); +const actorId = "019ff451-7d0d-7880-9fed-67b776ed6631"; + +function run(id: string, actionKey: string, state: "failed" | "queued"): JobRunRecord { + const queuedAt = new Date(1000); + const finishedAt = state === "failed" ? new Date(2000) : null; + return { + actionKey, + attemptCount: state === "failed" ? 1 : 0, + attemptLimit: 1, + availableAt: queuedAt, + cancellationPolicy: "never", + cancelRequestedAt: null, + cancelRequestedById: null, + cancelRequestedByKind: null, + displayName: "Fixed Service Action", + enqueueSha256: "b".repeat(64), + eventBytes: 0, + eventCount: state === "failed" ? 2 : 1, + finishedAt, + firstStartedAt: state === "failed" ? new Date(1500) : null, + heartbeatAt: null, + id, + idempotencyKey: "A".repeat(43), + lastAttemptStartedAt: state === "failed" ? new Date(1500) : null, + leaseExpiresAt: null, + leaseOwnerId: null, + leaseToken: null, + payloadEventCount: 0, + payloadJson: "{}", + priority: 20, + queuedAt, + requestedById: actorId, + requestedByKind: "user", + resourceClass: "exclusive", + resourceKeysJson: '["host.mutation"]', + resultJson: null, + retrySafe: false, + scheduledForAt: null, + scheduledJobId: null, + scheduledJobVersion: null, + state, + stateVersion: state === "failed" ? 3 : 1, + terminalCode: state === "failed" ? "failed/provider" : null, + terminalMessage: state === "failed" ? "Service Action failed." : null, + timeoutMs: 60_000, + triggerType: "manual", + updatedAt: finishedAt ?? queuedAt, + }; +} + +describe("Service Action status reader", () => { + test("requires fresh exact-release worker advertisements and projects bounded runs", async () => { + const availabilityInputs: unknown[] = []; + const active = run( + "019ff451-7d0d-7880-9fed-67b776ed6632", + serviceActionJobActionKeys["system-update"], + "queued" + ); + const latest = run( + "019ff451-7d0d-7880-9fed-67b776ed6633", + serviceActionJobActionKeys["system-update"], + "failed" + ); + const reader = createSqliteServiceActionStatusReader({ + expectedReleaseId, + nowMs: () => 40_000, + repository: { + readActionPayloadRunSnapshots: ({ actionKey, payloadJsons }) => [ + { + ...(actionKey === serviceActionJobActionKeys["system-update"] + ? { activeRun: active, lastRun: latest } + : {}), + payloadJson: payloadJsons[0] ?? "", + }, + ], + readWorkerActionAvailability: (input) => { + availabilityInputs.push(input); + return Object.freeze([ + serviceActionJobActionKeys["openclaw-cleanup"], + serviceActionJobActionKeys["system-update"], + ]); + }, + }, + }); + + expect(await reader.read()).toEqual([ + { availability: "available", id: "openclaw-cleanup" }, + { availability: "unavailable", id: "openclaw-update" }, + { availability: "unavailable", id: "system-restart" }, + { + activeRun: expect.objectContaining({ id: active.id, state: "queued" }), + availability: "available", + id: "system-update", + latestRun: expect.objectContaining({ id: latest.id, state: "failed" }), + }, + ]); + expect(availabilityInputs).toEqual([ + { + actionKeys: [ + "openclaw.sessions.cleanup", + "openclaw.installation.update", + "host.system.restart", + "host.system.update", + ], + expectedReleaseId, + minimumHeartbeatAt: new Date(10_000), + }, + ]); + }); + + test("rejects an already-aborted read before persistence", async () => { + const controller = new AbortController(); + controller.abort(new Error("request closed")); + let called = false; + const reader = createSqliteServiceActionStatusReader({ + expectedReleaseId, + repository: { + readActionPayloadRunSnapshots: () => [], + readWorkerActionAvailability: () => { + called = true; + return []; + }, + }, + }); + + let failure: unknown; + try { + await reader.read(controller.signal); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toBe("request closed"); + expect(called).toBeFalse(); + }); +}); diff --git a/greenfield/src/server/domains/serviceActions/statusReader.ts b/greenfield/src/server/domains/serviceActions/statusReader.ts new file mode 100644 index 000000000..c014f847a --- /dev/null +++ b/greenfield/src/server/domains/serviceActions/statusReader.ts @@ -0,0 +1,92 @@ +import { subMilliseconds } from "date-fns"; +import * as v from "valibot"; + +import { jobTimestampSchema, jobWorkerFreshnessMs } from "../../../contracts/jobModel.ts"; +import { + type ServiceActionStatus, + serviceActionIds, +} from "../../../contracts/serviceActions.ts"; +import { fullCommitShaSchema } from "../../../shared/validation.ts"; +import { toJobRunSummary } from "../jobs/records.ts"; +import type { + JobRepositoryReader, + WorkerActionAvailabilityReader, +} from "../jobs/repository.ts"; +import { serviceActionJobActionKeys } from "../jobs/serviceActionQueue.ts"; +import type { ServiceActionStatusReader } from "./service.ts"; + +type ServiceActionStatusRepository = Pick< + JobRepositoryReader, + "readActionPayloadRunSnapshots" +> & + WorkerActionAvailabilityReader; + +export interface SqliteServiceActionStatusReaderOptions { + readonly expectedReleaseId: string; + readonly nowMs?: () => number; + readonly repository: ServiceActionStatusRepository; +} + +/** + * Creates the exact-release, fresh-worker availability projection for fixed Service Actions. + * @param options Release identity, jobs repository, and observation clock. + * @returns A bounded status reader with sanitized durable-run summaries. + */ +export function createSqliteServiceActionStatusReader( + options: SqliteServiceActionStatusReaderOptions +): ServiceActionStatusReader { + const expectedReleaseId = v.parse( + fullCommitShaSchema("Expected worker release id is invalid"), + options.expectedReleaseId + ); + const nowMs = options.nowMs ?? Date.now; + const actionKeys = Object.freeze( + serviceActionIds.map((actionId) => serviceActionJobActionKeys[actionId]) + ); + + return Object.freeze({ + async read(signal?: AbortSignal): Promise { + await Promise.resolve(); + signal?.throwIfAborted(); + const observedAtMs = v.parse(jobTimestampSchema, nowMs()); + const minimumHeartbeatAt = subMilliseconds( + new Date(observedAtMs), + Math.min(observedAtMs, jobWorkerFreshnessMs) + ); + const availableActionKeys = new Set( + options.repository.readWorkerActionAvailability({ + actionKeys, + expectedReleaseId, + minimumHeartbeatAt, + }) + ); + signal?.throwIfAborted(); + + const statuses = serviceActionIds.map((id): ServiceActionStatus => { + signal?.throwIfAborted(); + const actionKey = serviceActionJobActionKeys[id]; + const snapshot = options.repository.readActionPayloadRunSnapshots({ + actionKey, + payloadJsons: ["{}"], + })[0]; + if (snapshot === undefined || snapshot.payloadJson !== "{}") { + throw new Error("Service Action run status is unavailable"); + } + return Object.freeze({ + ...(snapshot.activeRun === undefined + ? {} + : { activeRun: toJobRunSummary(snapshot.activeRun) }), + availability: availableActionKeys.has(actionKey) + ? "available" + : "unavailable", + id, + ...(snapshot.lastRun === undefined + ? {} + : { latestRun: toJobRunSummary(snapshot.lastRun) }), + }); + }); + signal?.throwIfAborted(); + return Object.freeze(statuses); + }, + }); +} diff --git a/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts new file mode 100644 index 000000000..21086888f --- /dev/null +++ b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, test } from "bun:test"; + +import { captureFailure } from "../../test/support/promise.ts"; +import { + createPersistentGatewayOpenClawServiceActionsProvider, + OpenClawServiceActionsProviderError, +} from "./persistentGatewayOpenClawServiceActionsProvider.ts"; +import type { PersistentGatewayTaskNotificationTransport } from "./persistentGatewayTransport.ts"; +import { PersistentGatewayUnknownOutcomeError } from "./persistentGatewayTransport.ts"; + +type Request = PersistentGatewayTaskNotificationTransport["requestOpenClawServiceAction"]; + +describe("persistent Gateway OpenClaw Service Actions provider", () => { + test("uses the exact cleanup request and returns only bounded aggregate counts", async () => { + const calls: Parameters[] = []; + const signal = new AbortController().signal; + const provider = createPersistentGatewayOpenClawServiceActionsProvider({ + requestOpenClawServiceAction: (...input: Parameters) => { + calls.push(input); + return Promise.resolve({ + method: "sessions.cleanup", + stores: [ + { + artifactsRemoved: 2, + bytesFreed: 300, + diskEntriesRemoved: 3, + diskFilesRemoved: 4, + dmScopesRetired: 5, + entriesAfter: 6, + entriesBefore: 7, + entriesCapped: 8, + entriesPruned: 9, + missingEntriesRemoved: 10, + modelRunsPruned: 11, + }, + { + artifactsRemoved: 1, + bytesFreed: 20, + diskEntriesRemoved: 1, + diskFilesRemoved: 1, + dmScopesRetired: 1, + entriesAfter: 1, + entriesBefore: 2, + entriesCapped: 1, + entriesPruned: 1, + missingEntriesRemoved: 1, + modelRunsPruned: 1, + }, + ], + }); + }, + }); + + expect(await provider.cleanupSessions(signal)).toEqual({ + artifactsRemoved: 3, + bytesFreed: 320, + diskEntriesRemoved: 4, + diskFilesRemoved: 5, + dmScopesRetired: 6, + entriesAfter: 7, + entriesBefore: 9, + entriesCapped: 9, + entriesPruned: 10, + missingEntriesRemoved: 11, + modelRunsPruned: 12, + status: "completed", + storesProcessed: 2, + }); + expect(calls).toEqual([ + [ + "sessions.cleanup", + { allAgents: true, enforce: true }, + { signal, timeoutMs: 600_000 }, + ], + ]); + }); + + test("projects completed and accepted updates without raw operational fields", async () => { + const responses = [ + { + afterVersion: "2026.8.0", + beforeVersion: "2026.7.2-beta.7", + method: "update.run" as const, + status: "completed" as const, + }, + { + beforeVersion: "2026.7.2-beta.7", + method: "update.run" as const, + status: "accepted" as const, + }, + ]; + const calls: Parameters[] = []; + const provider = createPersistentGatewayOpenClawServiceActionsProvider({ + requestOpenClawServiceAction: (...input: Parameters) => { + calls.push(input); + const response = responses.shift(); + if (response === undefined) throw new Error("missing fixture response"); + return Promise.resolve(response); + }, + }); + + expect(await provider.updateInstallation()).toEqual({ + afterVersion: "2026.8.0", + beforeVersion: "2026.7.2-beta.7", + status: "completed", + }); + expect(await provider.updateInstallation()).toEqual({ + beforeVersion: "2026.7.2-beta.7", + status: "accepted", + }); + expect( + calls.map(([method, parameters, options]) => ({ + method, + parameters, + timeoutMs: options?.timeoutMs, + })) + ).toEqual([ + { + method: "update.run", + parameters: { timeoutMs: 1_200_000 }, + timeoutMs: 2_100_000, + }, + { + method: "update.run", + parameters: { timeoutMs: 1_200_000 }, + timeoutMs: 2_100_000, + }, + ]); + }); + + test("fails operational errors and preserves unknown outcome without replay", async () => { + let attempts = 0; + const operational = createPersistentGatewayOpenClawServiceActionsProvider({ + requestOpenClawServiceAction: () => { + attempts += 1; + return Promise.resolve({ method: "update.run", status: "failed" }); + }, + }); + const operationFailure = await captureFailure(() => + operational.updateInstallation() + ); + expect(operationFailure).toBeInstanceOf(OpenClawServiceActionsProviderError); + expect(operationFailure).toMatchObject({ reason: "operation-failed" }); + + const uncertain = createPersistentGatewayOpenClawServiceActionsProvider({ + requestOpenClawServiceAction: () => { + attempts += 1; + return Promise.reject(new PersistentGatewayUnknownOutcomeError()); + }, + }); + const unknownFailure = await captureFailure(() => uncertain.cleanupSessions()); + expect(unknownFailure).toBeInstanceOf(OpenClawServiceActionsProviderError); + expect(unknownFailure).toMatchObject({ + message: "OpenClaw Service Action failed", + reason: "unknown-outcome", + }); + expect(JSON.stringify(unknownFailure)).not.toContain("systemctl"); + expect(attempts).toBe(2); + }); + + test("fails closed when aggregate cleanup counts exceed safe integers", async () => { + const provider = createPersistentGatewayOpenClawServiceActionsProvider({ + requestOpenClawServiceAction: () => + Promise.resolve({ + method: "sessions.cleanup", + stores: [ + { + artifactsRemoved: Number.MAX_SAFE_INTEGER, + bytesFreed: 0, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + }, + { + artifactsRemoved: 1, + bytesFreed: 0, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + }, + ], + }), + }); + + expect(await captureFailure(() => provider.cleanupSessions())).toMatchObject({ + message: "OpenClaw Service Action failed", + reason: "unavailable", + }); + }); +}); diff --git a/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts new file mode 100644 index 000000000..4f7e81983 --- /dev/null +++ b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.ts @@ -0,0 +1,159 @@ +import { + type OpenClawServiceActionsExecutionPort, + OpenClawServiceActionsExecutionError, + type OpenClawServiceActionsExecutionErrorReason, + type OpenClawSessionsCleanupSummary, +} from "../../../shared/openClawServiceActions.ts"; +import { + type PersistentGatewayOpenClawCleanupStoreProjection, + persistentGatewayOpenClawServiceActionRequestTimeoutMs, + persistentGatewayOpenClawUpdateTimeoutMs, +} from "./persistentGatewayProtocol.ts"; +import { + PersistentGatewayUnknownOutcomeError, + type PersistentGatewayTaskNotificationTransport, +} from "./persistentGatewayTransport.ts"; + +/** Constant worker-facing failure with no upstream message, path, or output. */ +export class OpenClawServiceActionsProviderError extends OpenClawServiceActionsExecutionError { + public constructor(reason: OpenClawServiceActionsExecutionErrorReason) { + super(reason); + this.name = "OpenClawServiceActionsProviderError"; + } +} + +type OpenClawServiceActionTransport = Pick< + PersistentGatewayTaskNotificationTransport, + "requestOpenClawServiceAction" +>; + +function addCount(left: number, right: number): number { + const total = left + right; + if (!Number.isSafeInteger(total) || total < 0) { + throw new OpenClawServiceActionsProviderError("unavailable"); + } + return total; +} + +function aggregateCleanup( + stores: readonly PersistentGatewayOpenClawCleanupStoreProjection[] +): OpenClawSessionsCleanupSummary { + let artifactsRemoved = 0; + let bytesFreed = 0; + let diskEntriesRemoved = 0; + let diskFilesRemoved = 0; + let dmScopesRetired = 0; + let entriesAfter = 0; + let entriesBefore = 0; + let entriesCapped = 0; + let entriesPruned = 0; + let missingEntriesRemoved = 0; + let modelRunsPruned = 0; + let storesProcessed = 0; + for (const store of stores) { + artifactsRemoved = addCount(artifactsRemoved, store.artifactsRemoved); + bytesFreed = addCount(bytesFreed, store.bytesFreed); + diskEntriesRemoved = addCount(diskEntriesRemoved, store.diskEntriesRemoved); + diskFilesRemoved = addCount(diskFilesRemoved, store.diskFilesRemoved); + dmScopesRetired = addCount(dmScopesRetired, store.dmScopesRetired); + entriesAfter = addCount(entriesAfter, store.entriesAfter); + entriesBefore = addCount(entriesBefore, store.entriesBefore); + entriesCapped = addCount(entriesCapped, store.entriesCapped); + entriesPruned = addCount(entriesPruned, store.entriesPruned); + missingEntriesRemoved = addCount( + missingEntriesRemoved, + store.missingEntriesRemoved + ); + modelRunsPruned = addCount(modelRunsPruned, store.modelRunsPruned); + storesProcessed = addCount(storesProcessed, 1); + } + return Object.freeze({ + artifactsRemoved, + bytesFreed, + diskEntriesRemoved, + diskFilesRemoved, + dmScopesRetired, + entriesAfter, + entriesBefore, + entriesCapped, + entriesPruned, + missingEntriesRemoved, + modelRunsPruned, + status: "completed", + storesProcessed, + }); +} + +function mapProviderFailure(error: unknown): never { + if (error instanceof OpenClawServiceActionsProviderError) throw error; + throw new OpenClawServiceActionsProviderError( + error instanceof PersistentGatewayUnknownOutcomeError + ? "unknown-outcome" + : "unavailable" + ); +} + +/** + * Creates the worker-only fixed OpenClaw operations adapter. The transport has + * already stripped raw paths, commands, process metadata, and command output. + * @returns A worker-only fixed-operation execution port. + */ +export function createPersistentGatewayOpenClawServiceActionsProvider( + transport: OpenClawServiceActionTransport +): OpenClawServiceActionsExecutionPort { + return Object.freeze({ + async cleanupSessions(signal?: AbortSignal) { + try { + const response = await transport.requestOpenClawServiceAction( + "sessions.cleanup", + { allAgents: true, enforce: true }, + { + signal, + timeoutMs: + persistentGatewayOpenClawServiceActionRequestTimeoutMs[ + "sessions.cleanup" + ], + } + ); + if (response.method !== "sessions.cleanup") { + throw new OpenClawServiceActionsProviderError("unavailable"); + } + return aggregateCleanup(response.stores); + } catch (error) { + mapProviderFailure(error); + } + }, + async updateInstallation(signal?: AbortSignal) { + try { + const response = await transport.requestOpenClawServiceAction( + "update.run", + { timeoutMs: persistentGatewayOpenClawUpdateTimeoutMs }, + { + signal, + timeoutMs: + persistentGatewayOpenClawServiceActionRequestTimeoutMs[ + "update.run" + ], + } + ); + if (response.method !== "update.run") { + throw new OpenClawServiceActionsProviderError("unavailable"); + } + if (response.status === "failed") { + throw new OpenClawServiceActionsProviderError("operation-failed"); + } + return Object.freeze({ + ...(response.afterVersion === undefined + ? {} + : { afterVersion: response.afterVersion }), + ...(response.beforeVersion === undefined + ? {} + : { beforeVersion: response.beforeVersion }), + status: response.status, + }); + } catch (error) { + mapProviderFailure(error); + } + }, + }); +} diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts index 9e164aadf..774ca866b 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts @@ -9,6 +9,7 @@ import { assertPersistentGatewayChatWriteParameters, assertPersistentGatewayOpenClawSettingsReadParameters, assertPersistentGatewayOpenClawSettingsWriteParameters, + assertPersistentGatewayOpenClawServiceActionParameters, assertPersistentGatewayReadWriteParameters, assertPersistentGatewayTaskReadParameters, assertPersistentGatewayTaskWriteParameters, @@ -16,6 +17,7 @@ import { isPersistentGatewayAdminMethod, isPersistentGatewayOpenClawSettingsReadMethod, isPersistentGatewayOpenClawSettingsWriteMethod, + isPersistentGatewayOpenClawServiceActionMethod, isPersistentGatewayReadWriteMethod, parsePersistentGatewayChallenge, parsePersistentGatewayChatSendAcknowledgement, @@ -23,6 +25,7 @@ import { parsePersistentGatewayEventEnvelope, parsePersistentGatewayHello, parsePersistentGatewayPrivateChatEvent, + parsePersistentGatewayOpenClawServiceActionResponse, parsePersistentGatewayResponse, parsePersistentGatewaySessionMessagesSubscriptionAcknowledgement, parsePersistentGatewaySessionsSubscriptionAcknowledgement, @@ -39,6 +42,9 @@ import { persistentGatewayOpenClawSettingsReadMethods, persistentGatewayOpenClawSettingsPatchMaximumBytes, persistentGatewayOpenClawSettingsWriteMethods, + persistentGatewayOpenClawCleanupStoreMaximum, + persistentGatewayOpenClawServiceActionMethods, + persistentGatewayOpenClawServiceActionResponseMaximumBytes, persistentGatewaySessionScopedEventsCapability, persistentGatewayTaskNotificationMethod, persistentGatewayTaskReadMethods, @@ -203,6 +209,10 @@ describe("persistent Gateway protocol-v4 boundary", () => { "config.patch", "skills.update", ]); + expect(persistentGatewayOpenClawServiceActionMethods).toEqual([ + "sessions.cleanup", + "update.run", + ]); expect(isPersistentGatewayReadWriteMethod("sessions.list")).toBe(true); expect(isPersistentGatewayReadWriteMethod("chat.send")).toBe(false); expect(isPersistentGatewayReadWriteMethod("config.patch")).toBe(false); @@ -213,6 +223,13 @@ describe("persistent Gateway protocol-v4 boundary", () => { expect(isPersistentGatewayOpenClawSettingsWriteMethod("skills.update")).toBe( true ); + expect(isPersistentGatewayOpenClawServiceActionMethod("sessions.cleanup")).toBe( + true + ); + expect(isPersistentGatewayOpenClawServiceActionMethod("update.run")).toBe(true); + expect(isPersistentGatewayOpenClawServiceActionMethod("config.patch")).toBe( + false + ); }); test("keeps persistent web reads object-bound and all controls admin-only", () => { @@ -451,6 +468,191 @@ describe("persistent Gateway protocol-v4 boundary", () => { ).toThrow(TypeError); }); + test("strictly binds and sanitizes worker-only OpenClaw Service Actions", () => { + expect(() => + assertPersistentGatewayOpenClawServiceActionParameters("sessions.cleanup", { + allAgents: true, + enforce: true, + }) + ).not.toThrow(); + expect(() => + assertPersistentGatewayOpenClawServiceActionParameters("update.run", { + timeoutMs: 1_200_000, + }) + ).not.toThrow(); + for (const parameters of [ + {}, + { allAgents: false, enforce: true }, + { allAgents: true, enforce: true, fixMissing: true }, + ]) { + expect(() => + assertPersistentGatewayOpenClawServiceActionParameters( + "sessions.cleanup", + parameters + ) + ).toThrow(TypeError); + } + for (const parameters of [ + {}, + { timeoutMs: 120_000 }, + { note: "unsafe", timeoutMs: 1_200_000 }, + ]) { + expect(() => + assertPersistentGatewayOpenClawServiceActionParameters( + "update.run", + parameters + ) + ).toThrow(TypeError); + } + + expect( + parsePersistentGatewayOpenClawServiceActionResponse("sessions.cleanup", { + allAgents: true, + dryRun: false, + mode: "enforce", + stores: [ + { + afterCount: 5, + agentId: "main", + applied: true, + appliedCount: 5, + beforeCount: 8, + capped: 1, + diskBudget: { + freedBytes: 200, + highWaterBytes: 800, + maxBytes: 1000, + overBudget: false, + removedEntries: 2, + removedFiles: 1, + totalBytesAfter: 500, + totalBytesBefore: 700, + }, + dmScopeRetired: 0, + dryRun: false, + missing: 1, + mode: "enforce", + modelRunPruned: 0, + pruned: 2, + storePath: "/private/openclaw/agents/main/sessions.db", + unreferencedArtifacts: { + freedBytes: 100, + olderThanMs: 86_400_000, + removedFiles: 3, + scannedFiles: 10, + }, + wouldMutate: true, + }, + ], + }) + ).toEqual({ + method: "sessions.cleanup", + stores: [ + { + artifactsRemoved: 3, + bytesFreed: 300, + diskEntriesRemoved: 2, + diskFilesRemoved: 1, + dmScopesRetired: 0, + entriesAfter: 5, + entriesBefore: 8, + entriesCapped: 1, + entriesPruned: 2, + missingEntriesRemoved: 1, + modelRunsPruned: 0, + }, + ], + }); + expect( + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + handoff: { + command: "private command", + pid: 42, + status: "started", + }, + ok: true, + restart: { pid: 43 }, + result: { + before: { version: "2026.7.2-beta.7" }, + cwd: "/private", + root: "/private/openclaw", + status: "skipped", + steps: [{ stderrTail: "secret" }], + }, + sentinel: { payload: { root: "/private/openclaw" } }, + }) + ).toEqual({ + beforeVersion: "2026.7.2-beta.7", + method: "update.run", + status: "accepted", + }); + expect( + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + handoff: { status: "started" }, + ok: true, + restart: { pid: 43 }, + result: { status: "error" }, + sentinel: {}, + }) + ).toEqual({ method: "update.run", status: "failed" }); + expect(() => + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + ok: true, + restart: null, + result: { + before: { version: "../../private" }, + status: "ok", + }, + sentinel: {}, + }) + ).toThrow(TypeError); + expect(() => + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + ok: false, + restart: null, + result: { status: "error" }, + sentinel: "x".repeat( + persistentGatewayOpenClawServiceActionResponseMaximumBytes + ), + }) + ).toThrow(TypeError); + + const cleanupStore = { + afterCount: 0, + agentId: "main", + applied: true, + appliedCount: 0, + beforeCount: 0, + capped: 0, + diskBudget: null, + dmScopeRetired: 0, + dryRun: false, + missing: 0, + mode: "enforce", + modelRunPruned: 0, + pruned: 0, + storePath: "/private/openclaw/sessions.db", + unreferencedArtifacts: { + freedBytes: 0, + olderThanMs: 86_400_000, + removedFiles: 0, + scannedFiles: 0, + }, + wouldMutate: false, + } as const; + expect(() => + parsePersistentGatewayOpenClawServiceActionResponse("sessions.cleanup", { + allAgents: true, + dryRun: false, + mode: "enforce", + stores: Array.from( + { length: persistentGatewayOpenClawCleanupStoreMaximum + 1 }, + (_, index) => ({ ...cleanupStore, agentId: `agent-${index}` }) + ), + }) + ).toThrow(TypeError); + }); + test("admits every exact Settings leaf delta and rejects adjacent shapes", () => { const baseHash = "a".repeat(64); const parameters = ( diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts index 083cd3958..707eeb71e 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts @@ -29,6 +29,17 @@ export const persistentGatewayChatOutboundFrameMaximumBytes = 24 * 1024 * 1024; export const persistentGatewayChatHistoryMaximumChars = 500_000; /** Exact serialized raw-patch ceiling shared by the Settings provider and protocol. */ export const persistentGatewayOpenClawSettingsPatchMaximumBytes = 64 * 1024; +/** Fixed audited per-step timeout sent to OpenClaw's update runner. */ +export const persistentGatewayOpenClawUpdateTimeoutMs = 20 * 60_000; +/** Exact outer deadlines admitted only for the two worker-owned OpenClaw operations. */ +export const persistentGatewayOpenClawServiceActionRequestTimeoutMs = Object.freeze({ + "sessions.cleanup": 10 * 60_000, + "update.run": 35 * 60_000, +} as const); +/** Dashboard-owned bound applied before parsing privileged operation results. */ +export const persistentGatewayOpenClawServiceActionResponseMaximumBytes = 2 * 1024 * 1024; +/** Dashboard-owned maximum number of cleanup stores aggregated into one result. */ +export const persistentGatewayOpenClawCleanupStoreMaximum = 256; export const persistentGatewayWebReadScopes = Object.freeze(["operator.read"] as const); export const persistentGatewayTaskNotificationScopes = Object.freeze([ @@ -114,6 +125,14 @@ export type PersistentGatewayOpenClawSettingsReadMethod = export type PersistentGatewayOpenClawSettingsWriteMethod = (typeof persistentGatewayOpenClawSettingsWriteMethods)[number]; +/** Worker-only OpenClaw operations admitted only to fresh operator.admin sockets. */ +export const persistentGatewayOpenClawServiceActionMethods = Object.freeze([ + "sessions.cleanup", + "update.run", +] as const); +export type PersistentGatewayOpenClawServiceActionMethod = + (typeof persistentGatewayOpenClawServiceActionMethods)[number]; + /** Installed protocol-v4 top-level request error discriminants. */ export const persistentGatewayErrorCodes = Object.freeze([ "AGENT_TIMEOUT", @@ -174,6 +193,9 @@ const openClawSettingsReadMethodSet = new Set( const openClawSettingsWriteMethodSet = new Set( persistentGatewayOpenClawSettingsWriteMethods ); +const openClawServiceActionMethodSet = new Set( + persistentGatewayOpenClawServiceActionMethods +); const eventNameSet = new Set(persistentGatewayEventNames); const boundedIdentifierSchema = v.pipe( @@ -639,6 +661,92 @@ const gatewayOpenClawSkillUpdateParamsSchema = v.strictObject({ enabled: v.boolean("OpenClaw skill enabled state is invalid"), skillKey: openClawSkillKeySchema, }); +const gatewayOpenClawSessionsCleanupParamsSchema = v.strictObject({ + allAgents: v.literal(true), + enforce: v.literal(true), +}); +const gatewayOpenClawInstallationUpdateParamsSchema = v.strictObject({ + timeoutMs: v.literal(persistentGatewayOpenClawUpdateTimeoutMs), +}); +const gatewayOpenClawOperationSensitiveTextSchema = v.pipe( + v.string("OpenClaw operation response text is invalid"), + v.maxLength(32 * 1024, "OpenClaw operation response text is invalid") +); +const gatewayOpenClawCleanupArtifactsSchema = v.strictObject({ + freedBytes: nonnegativeSafeIntegerSchema, + olderThanMs: nonnegativeSafeIntegerSchema, + removedFiles: nonnegativeSafeIntegerSchema, + scannedFiles: nonnegativeSafeIntegerSchema, +}); +const gatewayOpenClawCleanupDiskBudgetSchema = v.nullable( + v.strictObject({ + freedBytes: nonnegativeSafeIntegerSchema, + highWaterBytes: nonnegativeSafeIntegerSchema, + maxBytes: nonnegativeSafeIntegerSchema, + overBudget: v.boolean(), + removedEntries: nonnegativeSafeIntegerSchema, + removedFiles: nonnegativeSafeIntegerSchema, + totalBytesAfter: nonnegativeSafeIntegerSchema, + totalBytesBefore: nonnegativeSafeIntegerSchema, + }) +); +const gatewayOpenClawCleanupStoreSchema = v.strictObject({ + afterCount: nonnegativeSafeIntegerSchema, + agentId: v.pipe(v.string(), v.minLength(1), v.maxLength(256)), + applied: v.literal(true), + appliedCount: nonnegativeSafeIntegerSchema, + beforeCount: nonnegativeSafeIntegerSchema, + capped: nonnegativeSafeIntegerSchema, + diskBudget: gatewayOpenClawCleanupDiskBudgetSchema, + dmScopeRetired: nonnegativeSafeIntegerSchema, + dryRun: v.literal(false), + missing: nonnegativeSafeIntegerSchema, + mode: v.literal("enforce"), + modelRunPruned: nonnegativeSafeIntegerSchema, + pruned: nonnegativeSafeIntegerSchema, + storePath: gatewayOpenClawOperationSensitiveTextSchema, + unreferencedArtifacts: gatewayOpenClawCleanupArtifactsSchema, + wouldMutate: v.boolean(), +}); +const gatewayOpenClawCleanupResponseSchema = v.union([ + gatewayOpenClawCleanupStoreSchema, + v.strictObject({ + allAgents: v.literal(true), + dryRun: v.literal(false), + mode: v.literal("enforce"), + stores: v.pipe( + v.array(gatewayOpenClawCleanupStoreSchema), + v.maxLength(persistentGatewayOpenClawCleanupStoreMaximum) + ), + }), +]); +const gatewayOpenClawVersionSchema = v.pipe( + v.string("OpenClaw update version is invalid"), + v.minLength(1, "OpenClaw update version is invalid"), + v.maxLength(128, "OpenClaw update version is invalid"), + v.regex( + /^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u, + "OpenClaw update version is invalid" + ) +); +const gatewayOpenClawUpdateVersionProjectionSchema = v.object({ + version: gatewayOpenClawVersionSchema, +}); +const gatewayOpenClawUpdateResultSchema = v.object({ + after: v.optional(gatewayOpenClawUpdateVersionProjectionSchema), + before: v.optional(gatewayOpenClawUpdateVersionProjectionSchema), + status: v.picklist(["error", "ok", "skipped"]), +}); +const gatewayOpenClawUpdateHandoffSchema = v.object({ + status: v.picklist(["already-running", "started", "unavailable"]), +}); +const gatewayOpenClawUpdateResponseSchema = v.strictObject({ + handoff: v.optional(gatewayOpenClawUpdateHandoffSchema), + ok: v.boolean(), + restart: v.unknown(), + result: gatewayOpenClawUpdateResultSchema, + sentinel: v.unknown(), +}); const gatewayOpenClawSettingsNullableTextSchema = (maximum: number) => v.nullable( v.pipe( @@ -1232,6 +1340,12 @@ export function isPersistentGatewayOpenClawSettingsWriteMethod( return openClawSettingsWriteMethodSet.has(method); } +export function isPersistentGatewayOpenClawServiceActionMethod( + method: string +): method is PersistentGatewayOpenClawServiceActionMethod { + return openClawServiceActionMethodSet.has(method); +} + /** * Enforces the installed Gateway's dynamic least-privilege rules before a * request reaches the long-lived read/write socket. @@ -1283,6 +1397,131 @@ export function assertPersistentGatewayOpenClawSettingsWriteParameters( } } +/** Locks worker-owned Service Actions to their source-audited fixed arguments. */ +export function assertPersistentGatewayOpenClawServiceActionParameters( + method: PersistentGatewayOpenClawServiceActionMethod, + parameters: unknown +): asserts parameters is Readonly> { + const schema = + method === "sessions.cleanup" + ? gatewayOpenClawSessionsCleanupParamsSchema + : gatewayOpenClawInstallationUpdateParamsSchema; + if (!v.safeParse(schema, parameters).success) { + throw new TypeError( + "Persistent Gateway OpenClaw operation parameters are invalid" + ); + } +} + +export interface PersistentGatewayOpenClawCleanupStoreProjection { + readonly artifactsRemoved: number; + readonly bytesFreed: number; + readonly diskEntriesRemoved: number; + readonly diskFilesRemoved: number; + readonly dmScopesRetired: number; + readonly entriesAfter: number; + readonly entriesBefore: number; + readonly entriesCapped: number; + readonly entriesPruned: number; + readonly missingEntriesRemoved: number; + readonly modelRunsPruned: number; +} + +export type PersistentGatewayOpenClawServiceActionResponse = + | { + readonly method: "sessions.cleanup"; + readonly stores: readonly PersistentGatewayOpenClawCleanupStoreProjection[]; + } + | { + readonly afterVersion?: string; + readonly beforeVersion?: string; + readonly method: "update.run"; + readonly status: "accepted" | "completed" | "failed"; + }; + +function addGatewayOperationCounts(left: number, right: number): number { + const total = left + right; + if (!Number.isSafeInteger(total) || total < 0) { + throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); + } + return total; +} + +/** + * Removes paths, commands, process metadata, and output before the worker provider + * can observe a privileged OpenClaw response. + * @returns A bounded path-free operation projection. + */ +export function parsePersistentGatewayOpenClawServiceActionResponse( + method: PersistentGatewayOpenClawServiceActionMethod, + payload: unknown +): PersistentGatewayOpenClawServiceActionResponse { + if ( + !jsonValueFitsByteBudget( + payload, + persistentGatewayOpenClawServiceActionResponseMaximumBytes + ) + ) { + throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); + } + if (method === "sessions.cleanup") { + const parsed = v.safeParse(gatewayOpenClawCleanupResponseSchema, payload); + if (!parsed.success) { + throw new TypeError( + "Persistent Gateway OpenClaw operation response is invalid" + ); + } + const stores = "stores" in parsed.output ? parsed.output.stores : [parsed.output]; + return Object.freeze({ + method, + stores: Object.freeze( + stores.map((store) => + Object.freeze({ + artifactsRemoved: store.unreferencedArtifacts.removedFiles, + bytesFreed: addGatewayOperationCounts( + store.unreferencedArtifacts.freedBytes, + store.diskBudget?.freedBytes ?? 0 + ), + diskEntriesRemoved: store.diskBudget?.removedEntries ?? 0, + diskFilesRemoved: store.diskBudget?.removedFiles ?? 0, + dmScopesRetired: store.dmScopeRetired, + entriesAfter: store.afterCount, + entriesBefore: store.beforeCount, + entriesCapped: store.capped, + entriesPruned: store.pruned, + missingEntriesRemoved: store.missing, + modelRunsPruned: store.modelRunPruned, + }) + ) + ), + }); + } + const parsed = v.safeParse(gatewayOpenClawUpdateResponseSchema, payload); + if (!parsed.success) { + throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); + } + let status: "accepted" | "completed" | "failed" = "failed"; + if (parsed.output.ok && parsed.output.result.status === "ok") { + status = "completed"; + } else if ( + parsed.output.ok && + parsed.output.result.status === "skipped" && + parsed.output.handoff?.status === "started" + ) { + status = "accepted"; + } + return Object.freeze({ + ...(parsed.output.result.after === undefined + ? {} + : { afterVersion: parsed.output.result.after.version }), + ...(parsed.output.result.before === undefined + ? {} + : { beforeVersion: parsed.output.result.before.version }), + method, + status, + }); +} + export function assertPersistentGatewayChatReadParameters( method: PersistentGatewayChatReadMethod, parameters: unknown diff --git a/greenfield/src/server/platform/gateway/persistentGatewayTransport.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayTransport.test.ts index 095d64c2c..78580657b 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayTransport.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayTransport.test.ts @@ -4,11 +4,13 @@ import { Effect, Layer, Redacted } from "effect"; import { chatAttachmentLimits } from "../../../contracts/chatMedia.ts"; import { captureFailure } from "../../test/support/promise.ts"; +import { createPersistentGatewayOpenClawServiceActionsProvider } from "./persistentGatewayOpenClawServiceActionsProvider.ts"; import { persistentGatewayAuthenticatedFrameMaximumBytes, persistentGatewayBufferedAmountMaximumBytes, persistentGatewayBufferedAmountPolicyMaximumBytes, persistentGatewayChatOutboundFrameMaximumBytes, + persistentGatewayOpenClawServiceActionRequestTimeoutMs, type PersistentGatewayAdminMethod, type PersistentGatewayReadWriteMethod, } from "./persistentGatewayProtocol.ts"; @@ -1056,6 +1058,336 @@ describe("persistent native Gateway transport", () => { await stopping; }); + test("runs fixed worker OpenClaw operations only on fresh admin sockets", async () => { + const scheduler = new ManualScheduler(); + const harness = new SocketHarness(); + const transport = createFixtureTaskNotificationTransport(harness, scheduler); + transport.start(); + const persistentSocket = harness.sockets[0]; + if (persistentSocket === undefined) throw new Error("Expected worker socket"); + completeHandshake(persistentSocket, { + lane: "task-notification-worker", + methods: ["chat.send"], + }); + + const invalid = transport.requestOpenClawServiceAction("sessions.cleanup", { + allAgents: true, + enforce: true, + fixMissing: true, + }); + expect(await captureFailure(() => invalid)).toBeInstanceOf( + PersistentGatewayUnavailableError + ); + expect(harness.sockets).toHaveLength(1); + + const cleanup = transport.requestOpenClawServiceAction("sessions.cleanup", { + allAgents: true, + enforce: true, + }); + const adminSocket = harness.sockets[1]; + if (adminSocket === undefined) throw new Error("Expected fresh admin socket"); + const connect = completeHandshake(adminSocket, { + lane: "admin", + methods: ["sessions.cleanup"], + }); + expect(connect.params).toMatchObject({ scopes: ["operator.admin"] }); + const request = sentFrame(adminSocket, 1); + expect(request).toMatchObject({ + method: "sessions.cleanup", + params: { allAgents: true, enforce: true }, + }); + adminSocket.receive({ + id: request.id, + ok: true, + payload: { + afterCount: 4, + agentId: "main", + applied: true, + appliedCount: 4, + beforeCount: 5, + capped: 0, + diskBudget: null, + dmScopeRetired: 0, + dryRun: false, + missing: 0, + mode: "enforce", + modelRunPruned: 0, + pruned: 1, + storePath: "/private/openclaw/sessions.db", + unreferencedArtifacts: { + freedBytes: 200, + olderThanMs: 86_400_000, + removedFiles: 1, + scannedFiles: 10, + }, + wouldMutate: true, + }, + type: "res", + }); + await flushMicrotasks(); + adminSocket.finishClose(); + expect(await cleanup).toEqual({ + method: "sessions.cleanup", + stores: [ + { + artifactsRemoved: 1, + bytesFreed: 200, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 4, + entriesBefore: 5, + entriesCapped: 0, + entriesPruned: 1, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + }, + ], + }); + expect(persistentSocket.closeCalls).toHaveLength(0); + const stopping = transport.stop(); + persistentSocket.finishClose(); + await stopping; + }); + + test("admits the exact production OpenClaw Service Action provider deadlines", async () => { + const scheduler = new ManualScheduler(); + const harness = new SocketHarness(); + const transport = createFixtureTaskNotificationTransport(harness, scheduler); + const provider = createPersistentGatewayOpenClawServiceActionsProvider(transport); + + const cleanup = provider.cleanupSessions(); + const cleanupSocket = harness.sockets[0]; + if (cleanupSocket === undefined) throw new Error("Expected cleanup admin socket"); + completeHandshake(cleanupSocket, { + lane: "admin", + methods: ["sessions.cleanup"], + }); + const cleanupRequest = sentFrame(cleanupSocket, 1); + expect(cleanupRequest).toMatchObject({ + method: "sessions.cleanup", + params: { allAgents: true, enforce: true }, + }); + cleanupSocket.receive({ + id: cleanupRequest.id, + ok: true, + payload: { + afterCount: 0, + agentId: "main", + applied: true, + appliedCount: 0, + beforeCount: 0, + capped: 0, + diskBudget: null, + dmScopeRetired: 0, + dryRun: false, + missing: 0, + mode: "enforce", + modelRunPruned: 0, + pruned: 0, + storePath: "/private/openclaw/sessions.db", + unreferencedArtifacts: { + freedBytes: 0, + olderThanMs: 86_400_000, + removedFiles: 0, + scannedFiles: 0, + }, + wouldMutate: false, + }, + type: "res", + }); + await flushMicrotasks(); + cleanupSocket.finishClose(); + expect(await cleanup).toEqual({ + artifactsRemoved: 0, + bytesFreed: 0, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + status: "completed", + storesProcessed: 1, + }); + + const update = provider.updateInstallation(); + const updateSocket = harness.sockets[1]; + if (updateSocket === undefined) throw new Error("Expected update admin socket"); + completeHandshake(updateSocket, { lane: "admin", methods: ["update.run"] }); + const updateRequest = sentFrame(updateSocket, 1); + expect(updateRequest).toMatchObject({ + method: "update.run", + params: { timeoutMs: 1_200_000 }, + }); + updateSocket.receive({ + id: updateRequest.id, + ok: true, + payload: { + ok: true, + restart: null, + result: { + after: { version: "2026.8.0" }, + before: { version: "2026.7.2-beta.7" }, + status: "ok", + }, + sentinel: {}, + }, + type: "res", + }); + await flushMicrotasks(); + updateSocket.finishClose(); + expect(await update).toEqual({ + afterVersion: "2026.8.0", + beforeVersion: "2026.7.2-beta.7", + status: "completed", + }); + expect(harness.sockets).toHaveLength(2); + await transport.stop(); + }); + + test("settles Service Action deadlines above each exact method ceiling", async () => { + for (const method of ["sessions.cleanup", "update.run"] as const) { + const scheduler = new ManualScheduler(); + const harness = new SocketHarness(); + const transport = createFixtureTaskNotificationTransport(harness, scheduler); + const request = transport.requestOpenClawServiceAction( + method, + method === "sessions.cleanup" + ? { allAgents: true, enforce: true } + : { timeoutMs: 1_200_000 }, + { + timeoutMs: + persistentGatewayOpenClawServiceActionRequestTimeoutMs[method] + + 1, + } + ); + const socket = harness.sockets[0]; + if (socket === undefined) throw new Error("Expected operation admin socket"); + completeHandshake(socket, { lane: "admin", methods: [method] }); + await flushMicrotasks(); + + expect(socket.sent).toHaveLength(1); + expect(socket.closeCalls).toEqual([ + { code: 1000, reason: "gateway lane complete" }, + ]); + socket.finishClose(); + expect(await captureFailure(() => request)).toEqual( + new TypeError("Persistent Gateway request timeout is invalid") + ); + await transport.stop(); + } + }); + + test("retains the five-minute ceiling for ordinary one-shot admin methods", async () => { + const scheduler = new ManualScheduler(); + const harness = new SocketHarness(); + const transport = createFixtureTransport(harness, scheduler); + const request = transport.requestAdmin( + "cron.run", + { id: "cron-job-1" }, + { timeoutMs: 5 * 60_000 + 1 } + ); + const socket = harness.sockets[0]; + if (socket === undefined) throw new Error("Expected ordinary admin socket"); + completeHandshake(socket, { lane: "admin", methods: ["cron.run"] }); + await flushMicrotasks(); + + expect(socket.sent).toHaveLength(1); + expect(socket.closeCalls).toEqual([ + { code: 1000, reason: "gateway lane complete" }, + ]); + socket.finishClose(); + expect(await captureFailure(() => request)).toEqual( + new TypeError("Persistent Gateway request timeout is invalid") + ); + await transport.stop(); + }); + + test("classifies invalid operation acknowledgements as unknown without replay", async () => { + const scheduler = new ManualScheduler(); + const harness = new SocketHarness(); + const transport = createFixtureTaskNotificationTransport(harness, scheduler); + const update = transport.requestOpenClawServiceAction("update.run", { + timeoutMs: 1_200_000, + }); + const socket = harness.sockets[0]; + if (socket === undefined) throw new Error("Expected fresh admin socket"); + completeHandshake(socket, { lane: "admin", methods: ["update.run"] }); + const request = sentFrame(socket, 1); + socket.receive({ + id: request.id, + ok: true, + payload: { + ok: true, + restart: null, + result: { status: "unexpected" }, + sentinel: {}, + }, + type: "res", + }); + await flushMicrotasks(); + socket.finishClose(); + expect(await captureFailure(() => update)).toBeInstanceOf( + PersistentGatewayUnknownOutcomeError + ); + expect(harness.sockets).toHaveLength(1); + await transport.stop(); + }); + + test("keeps post-dispatch OpenClaw operation close and timeout outcome-unknown", async () => { + const closeScheduler = new ManualScheduler(); + const closeHarness = new SocketHarness(); + const closeTransport = createFixtureTaskNotificationTransport( + closeHarness, + closeScheduler + ); + const lostAcknowledgement = closeTransport.requestOpenClawServiceAction( + "sessions.cleanup", + { allAgents: true, enforce: true } + ); + const closeSocket = closeHarness.sockets[0]; + if (closeSocket === undefined) throw new Error("Expected fresh admin socket"); + completeHandshake(closeSocket, { + lane: "admin", + methods: ["sessions.cleanup"], + }); + expect(sentFrame(closeSocket, 1).method).toBe("sessions.cleanup"); + closeSocket.finishClose(); + expect(await captureFailure(() => lostAcknowledgement)).toBeInstanceOf( + PersistentGatewayUnknownOutcomeError + ); + expect(closeHarness.sockets).toHaveLength(1); + await closeTransport.stop(); + + const timeoutScheduler = new ManualScheduler(); + const timeoutHarness = new SocketHarness(); + const timeoutTransport = createFixtureTaskNotificationTransport( + timeoutHarness, + timeoutScheduler + ); + const timedOut = timeoutTransport.requestOpenClawServiceAction( + "update.run", + { timeoutMs: 1_200_000 }, + { timeoutMs: 5 } + ); + const timeoutSocket = timeoutHarness.sockets[0]; + if (timeoutSocket === undefined) throw new Error("Expected fresh admin socket"); + completeHandshake(timeoutSocket, { lane: "admin", methods: ["update.run"] }); + expect(sentFrame(timeoutSocket, 1).method).toBe("update.run"); + timeoutScheduler.advance(5); + await flushMicrotasks(); + timeoutSocket.finishClose(); + expect(await captureFailure(() => timedOut)).toBeInstanceOf( + PersistentGatewayUnknownOutcomeError + ); + expect(timeoutHarness.sockets).toHaveLength(1); + await timeoutTransport.stop(); + }); + test("settles an idempotent task-notification retry after a lost chat-send acknowledgement", async () => { const scheduler = new ManualScheduler(); const harness = new SocketHarness(); @@ -3005,6 +3337,11 @@ describe("persistent native Gateway transport", () => { parameters: Readonly>, options?: PersistentGatewayRequestOptions ) => Promise; + const runtimeServiceAction = ( + transport as unknown as { + requestOpenClawServiceAction: RuntimeRequest; + } + ).requestOpenClawServiceAction.bind(transport); for (const method of [ "config.get", @@ -3026,6 +3363,11 @@ describe("persistent native Gateway transport", () => { PersistentGatewayUnavailableError ); } + for (const method of ["sessions.cleanup", "update.run"]) { + expect( + await captureFailure(() => runtimeServiceAction(method, {})) + ).toBeInstanceOf(PersistentGatewayUnavailableError); + } expect(harness.sockets).toHaveLength(0); await transport.stop(); }); diff --git a/greenfield/src/server/platform/gateway/persistentGatewayTransport.ts b/greenfield/src/server/platform/gateway/persistentGatewayTransport.ts index f7b73f547..aeebd6376 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayTransport.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayTransport.ts @@ -16,6 +16,7 @@ import { assertPersistentGatewayChatWriteParameters, assertPersistentGatewayOpenClawSettingsReadParameters, assertPersistentGatewayOpenClawSettingsWriteParameters, + assertPersistentGatewayOpenClawServiceActionParameters, assertPersistentGatewayReadWriteParameters, assertPersistentGatewayTaskReadParameters, assertPersistentGatewayTaskWriteParameters, @@ -26,6 +27,7 @@ import { isPersistentGatewayChatWriteMethod, isPersistentGatewayOpenClawSettingsReadMethod, isPersistentGatewayOpenClawSettingsWriteMethod, + isPersistentGatewayOpenClawServiceActionMethod, isPersistentGatewayReadWriteMethod, isPersistentGatewayTaskReadMethod, isPersistentGatewayTaskWriteMethod, @@ -46,6 +48,9 @@ import { type PersistentGatewayPrivateChatEvent, type PersistentGatewayOpenClawSettingsReadMethod, type PersistentGatewayOpenClawSettingsWriteMethod, + type PersistentGatewayOpenClawServiceActionMethod, + type PersistentGatewayOpenClawServiceActionResponse, + persistentGatewayOpenClawServiceActionRequestTimeoutMs, persistentGatewayOutboundFrameMaximumBytes, type PersistentGatewayReadWriteMethod, type PersistentGatewayTaskReadMethod, @@ -56,6 +61,7 @@ import { parsePersistentGatewayEventEnvelope, parsePersistentGatewayHello, parsePersistentGatewayPrivateChatEvent, + parsePersistentGatewayOpenClawServiceActionResponse, parsePersistentGatewayResponse, parsePersistentGatewaySessionMessagesSubscriptionAcknowledgement, parsePersistentGatewaySessionsSubscriptionAcknowledgement, @@ -69,6 +75,7 @@ const policyCloseCode = 1008; const watchdogCloseCode = 4000; const safeCloseReasonMaximumBytes = 123; const taskNotificationIdempotencyKeyPrefix = "tasks-notify-"; +const requestTimeoutMaximumDefaultMs = 5 * 60_000; export const persistentGatewayChatEventQueueMaximum = 256; /** Per-listener encoded projection budget retained while async delivery is blocked. */ export const persistentGatewayChatEventQueueMaximumBytes = 2 * 1024 * 1024; @@ -224,6 +231,8 @@ interface PersistentGatewayOneShotRequestOptions extends PersistentGatewayReques interface GatewaySocketLaneRequestOptions extends PersistentGatewayRequestOptions { /** Runs synchronously only after the native socket accepted the encoded frame. */ readonly onDispatched?: () => void; + /** Internal per-method ceiling; callers retain the five-minute default. */ + readonly timeoutMaximumMs?: number; } export type PersistentGatewayWebSocketFactory = (url: string) => WebSocket; @@ -934,7 +943,7 @@ class GatewaySocketLane { options.timeoutMs, this.#resolved.requestTimeoutMs, 1, - 5 * 60 * 1000, + options.timeoutMaximumMs ?? requestTimeoutMaximumDefaultMs, "Persistent Gateway request timeout" ); let id: string; @@ -1615,9 +1624,14 @@ export interface PersistentGatewayTransport extends PersistentGatewayTransportLi ): () => void; } -/** Worker-only port with no generic read or admin request capability. */ +/** Worker-only port with notification sending and exact privileged operation methods. */ export interface PersistentGatewayTaskNotificationTransport extends PersistentGatewayTransportLifecycle { readonly taskNotificationSender: TaskNotificationChatSender; + requestOpenClawServiceAction( + method: PersistentGatewayOpenClawServiceActionMethod, + parameters: Readonly>, + options?: PersistentGatewayRequestOptions + ): Promise; } class PersistentGatewayTransportImplementation @@ -1897,6 +1911,39 @@ class PersistentGatewayTransportImplementation ); } + async requestOpenClawServiceAction( + method: PersistentGatewayOpenClawServiceActionMethod, + parameters: Readonly>, + options: PersistentGatewayRequestOptions = {} + ): Promise { + if ( + this.#resolved.profile !== "task-notification-worker" || + !isPersistentGatewayOpenClawServiceActionMethod(method) + ) { + throw new PersistentGatewayUnavailableError(); + } + try { + assertPersistentGatewayOpenClawServiceActionParameters(method, parameters); + } catch { + throw new PersistentGatewayUnavailableError(); + } + this.#assertOneShotAdmission(options); + const response = await this.#runOneShotRequest( + "admin", + method, + parameters, + options, + this.#resolved.bufferedAmountMaximumBytes, + this.#resolved.outboundFrameMaximumBytes, + persistentGatewayOpenClawServiceActionRequestTimeoutMs[method] + ); + try { + return parsePersistentGatewayOpenClawServiceActionResponse(method, response); + } catch { + throw new PersistentGatewayUnknownOutcomeError(); + } + } + start(): void { if (this.#permanentlyStopped) { throw new TypeError("Persistent Gateway transport is stopped"); @@ -2552,12 +2599,14 @@ class PersistentGatewayTransportImplementation | PersistentGatewayAdminMethod | PersistentGatewayChatReadMutationMethod | PersistentGatewayChatWriteMethod + | PersistentGatewayOpenClawServiceActionMethod | PersistentGatewayOpenClawSettingsWriteMethod | PersistentGatewayTaskWriteMethod, parameters: Readonly>, options: PersistentGatewayOneShotRequestOptions, bufferedAmountMaximumBytes: number, - outboundFrameMaximumBytes: number + outboundFrameMaximumBytes: number, + timeoutMaximumMs: number = requestTimeoutMaximumDefaultMs ): Promise { let dispatched = false; let settled = false; @@ -2587,14 +2636,20 @@ class PersistentGatewayTransportImplementation }, onConnected: () => { const dispatch = (): void => { - void lane - .request(method, parameters, { + let request: Promise; + try { + request = lane.request(method, parameters, { ...options, onDispatched: () => { dispatched = true; }, - }) - .then(resolveOutcome, rejectOutcome); + timeoutMaximumMs, + }); + } catch (error) { + rejectOutcome(error); + return; + } + void request.then(resolveOutcome, rejectOutcome); }; const beforeDispatch = options.beforeDispatch; if (beforeDispatch === undefined) { diff --git a/greenfield/src/server/platform/observability/structuredLogger.test.ts b/greenfield/src/server/platform/observability/structuredLogger.test.ts index 8c272261f..243105788 100644 --- a/greenfield/src/server/platform/observability/structuredLogger.test.ts +++ b/greenfield/src/server/platform/observability/structuredLogger.test.ts @@ -337,6 +337,43 @@ test("records only fixed log-maintenance audit settlement fields", () => { }); }); +test("records only fixed Service Actions audit settlement fields", () => { + const lines: string[] = []; + const logger = createStructuredLogger({ + identity, + sink: { + write(line) { + lines.push(line); + }, + }, + }); + + logger.error({ + component: "service-actions-audit", + event: "service_actions.audit_settlement.failed", + failure: new Error("private provider detail"), + fields: { + actionId: "openclaw-update", + kind: "service-actions-audit-settlement", + settlement: "partial", + }, + outcome: "server-error", + }); + + expect(JSON.parse(lines[0] ?? "null")).toMatchObject({ + component: "service-actions-audit", + event: "service_actions.audit_settlement.failed", + fields: { + actionId: "openclaw-update", + settlement: "partial", + }, + level: "error", + outcome: "server-error", + }); + expect(JSON.parse(lines[0] ?? "null")).not.toHaveProperty("fields.kind"); + expect(lines[0]).not.toContain("private provider detail"); +}); + test("normalizes unknown events and drops extra fields instead of relying on secret names", () => { const lines: string[] = []; const logger = createStructuredLogger({ diff --git a/greenfield/src/server/platform/observability/structuredLogger.ts b/greenfield/src/server/platform/observability/structuredLogger.ts index 8313f2dff..e0a79a429 100644 --- a/greenfield/src/server/platform/observability/structuredLogger.ts +++ b/greenfield/src/server/platform/observability/structuredLogger.ts @@ -2,6 +2,7 @@ import { logMaintenancePolicyIds, type LogMaintenancePolicyId, } from "../../../contracts/logs.ts"; +import type { ServiceActionId } from "../../../contracts/serviceActions.ts"; import type { SafeFailureDescriptor } from "../errors/safeFailure.ts"; import { describeSafeFailure } from "../errors/safeFailure.ts"; @@ -88,6 +89,11 @@ export type StructuredLogFields = readonly kind: "openclaw-settings-mutation-queue"; readonly queueDepth: number; } + | { + readonly actionId: ServiceActionId; + readonly kind: "service-actions-audit-settlement"; + readonly settlement: "failed" | "partial" | "succeeded"; + } | { readonly kind: "http-request"; readonly method: string; @@ -165,6 +171,7 @@ const structuredEventComponents = Object.freeze({ "runtime.start_failed": "runtime", "runtime.started": "runtime", "runtime.stopped": "runtime", + "service_actions.audit_settlement.failed": "service-actions-audit", "trpc.request.defect": "trpc", } as const); @@ -377,6 +384,24 @@ function safeEventFields( } return { queueDepth: fields.queueDepth }; } + case "service-actions-audit-settlement": { + if ( + eventName !== "service_actions.audit_settlement.failed" || + (fields.actionId !== "openclaw-cleanup" && + fields.actionId !== "openclaw-update" && + fields.actionId !== "system-restart" && + fields.actionId !== "system-update") || + (fields.settlement !== "failed" && + fields.settlement !== "partial" && + fields.settlement !== "succeeded") + ) { + return undefined; + } + return { + actionId: fields.actionId, + settlement: fields.settlement, + }; + } case "realtime-runner-failure": { return eventName === "realtime.runner.failed" && fields.failureKind === "unexpected-runner-defect" diff --git a/greenfield/src/server/test/support/requestContext.ts b/greenfield/src/server/test/support/requestContext.ts index c435fd161..4e80591bb 100644 --- a/greenfield/src/server/test/support/requestContext.ts +++ b/greenfield/src/server/test/support/requestContext.ts @@ -43,6 +43,7 @@ import type { AutomationSecurityLifecycleService } from "../../domains/security/ import type { MfaAccountLifecycleService } from "../../domains/security/mfa/accountLifecycle.ts"; import type { MfaLoginLifecycleService } from "../../domains/security/mfa/loginLifecycle.ts"; import type { SecurityAuditLifecycleService } from "../../domains/security/securityAuditLifecycle.ts"; +import type { ServiceActionsService } from "../../domains/serviceActions/service.ts"; import type { SystemHealthDiagnosticsService } from "../../domains/system/healthDiagnosticsService.ts"; import { SystemMetricsUnavailableError, @@ -93,6 +94,10 @@ function unavailableOpenClawSettingsCall(): Promise { return Promise.reject(new Error("OpenClaw settings unavailable")); } +function unavailableServiceActionsCall(): Promise { + return Promise.reject(new Error("Service actions unavailable")); +} + /** * Creates a stable empty Gateway-session service for generic request and server tests. * @returns An inert current-session service. @@ -169,6 +174,17 @@ export function createTestOpenClawSettingsService(): OpenClawSettingsService { }); } +/** + * Creates a stable fail-closed Service Actions service for generic request tests. + * @returns An inert fixed-action service. + */ +export function createTestServiceActionsService(): ServiceActionsService { + return Object.freeze({ + getStatus: unavailableServiceActionsCall, + request: unavailableServiceActionsCall, + }); +} + /** * Creates an inert process logger for tests that compose runtime or server roots. * @returns A complete structured logger that discards every validated record. @@ -526,6 +542,7 @@ export interface TestServerSecurityServices { readonly openClawCronService: OpenClawCronService; readonly openClawSettingsService: OpenClawSettingsService; readonly securityAuditLifecycle: SecurityAuditLifecycleService; + readonly serviceActionsService: ServiceActionsService; readonly systemHealthDiagnosticsService: SystemHealthDiagnosticsService; readonly taskService: TaskService["Service"]; } @@ -568,6 +585,8 @@ export function createTestServerSecurityServices( overrides.openClawSettingsService ?? createTestOpenClawSettingsService(), securityAuditLifecycle: overrides.securityAuditLifecycle ?? createTestSecurityAuditLifecycleService(), + serviceActionsService: + overrides.serviceActionsService ?? createTestServiceActionsService(), systemHealthDiagnosticsService: overrides.systemHealthDiagnosticsService ?? createTestSystemHealthDiagnosticsService(), @@ -667,6 +686,7 @@ export function createTestRequestContext( readonly requestId?: string; readonly responseHeaders?: Headers; readonly securityAuditLifecycle?: SecurityAuditLifecycleService; + readonly serviceActionsService?: ServiceActionsService; readonly systemHealthDiagnosticsService?: SystemHealthDiagnosticsService; readonly taskService?: TaskService["Service"]; } = {} @@ -708,6 +728,8 @@ export function createTestRequestContext( responseHeaders: options.responseHeaders ?? new Headers(), securityAuditLifecycle: options.securityAuditLifecycle ?? createTestSecurityAuditLifecycleService(), + serviceActionsService: + options.serviceActionsService ?? createTestServiceActionsService(), systemHealthDiagnosticsService: options.systemHealthDiagnosticsService ?? createTestSystemHealthDiagnosticsService(), diff --git a/greenfield/src/server/trpc/appRouter.ts b/greenfield/src/server/trpc/appRouter.ts index 31222dbaf..7df6e7ca4 100644 --- a/greenfield/src/server/trpc/appRouter.ts +++ b/greenfield/src/server/trpc/appRouter.ts @@ -60,6 +60,10 @@ import { securityAuditProcedureNames, securityAuditRouter, } from "../domains/security/securityAuditProcedures.ts"; +import { + serviceActionsProcedureNames, + serviceActionsRouter, +} from "../domains/serviceActions/procedures.ts"; import { systemProcedureNames, systemRouter } from "../domains/system/procedures.ts"; import { taskProcedureNames, taskRouter } from "../domains/tasks/procedures.ts"; import { @@ -99,6 +103,7 @@ export const appRouter = router({ reports: reportRouter, schedules: scheduleRouter, securityAudit: securityAuditRouter, + serviceActions: serviceActionsRouter, system: systemRouter, tasks: taskRouter, terminal: terminalRouter, @@ -128,6 +133,7 @@ export const appRouterProcedureNames = Object.freeze([ ...namespacedProcedureNames("reports", reportProcedureNames), ...namespacedProcedureNames("schedules", scheduleProcedureNames), ...namespacedProcedureNames("securityAudit", securityAuditProcedureNames), + ...namespacedProcedureNames("serviceActions", serviceActionsProcedureNames), ...namespacedProcedureNames("system", systemProcedureNames), ...namespacedProcedureNames("tasks", taskProcedureNames), ...namespacedProcedureNames("terminal", terminalProcedureNames), diff --git a/greenfield/src/server/trpc/context.test.ts b/greenfield/src/server/trpc/context.test.ts index 7b0a1c906..99147cb9d 100644 --- a/greenfield/src/server/trpc/context.test.ts +++ b/greenfield/src/server/trpc/context.test.ts @@ -21,6 +21,7 @@ import { createTestOpenClawCronService, createTestOpenClawSettingsService, createTestSecurityAuditLifecycleService, + createTestServiceActionsService, createTestSystemHealthDiagnosticsService, } from "../test/support/requestContext.ts"; import { createRequestContext } from "./context.ts"; @@ -45,6 +46,7 @@ describe("tRPC request context", () => { const jobService = createTestJobService(); const cacheService = createTestCacheService(); const responseHeaders = new Headers(); + const serviceActionsService = createTestServiceActionsService(); const context = await createRequestContext({ agentService: createTestAgentService(), @@ -87,6 +89,7 @@ describe("tRPC request context", () => { requestId: "request-context-1", responseHeaders, securityAuditLifecycle: createTestSecurityAuditLifecycleService(), + serviceActionsService, systemHealthDiagnosticsService: createTestSystemHealthDiagnosticsService(), taskService: createTestTaskService(), }); @@ -121,6 +124,7 @@ describe("tRPC request context", () => { }, }); expect(context.responseHeaders).toBe(responseHeaders); + expect(context.serviceActionsService).toBe(serviceActionsService); expect(context.requestId).toBe("request-context-1"); expect(context.userAgent).toBe("Context Test Browser"); expect("dispose" in context.services).toBe(false); @@ -158,6 +162,7 @@ describe("tRPC request context", () => { requestId: "request-context-2", responseHeaders: new Headers(), securityAuditLifecycle: createTestSecurityAuditLifecycleService(), + serviceActionsService: createTestServiceActionsService(), systemHealthDiagnosticsService: createTestSystemHealthDiagnosticsService(), taskService: createTestTaskService(), }); @@ -206,6 +211,7 @@ describe("tRPC request context", () => { requestId: "request-context-3", responseHeaders: new Headers(), securityAuditLifecycle: createTestSecurityAuditLifecycleService(), + serviceActionsService: createTestServiceActionsService(), systemHealthDiagnosticsService: createTestSystemHealthDiagnosticsService(), taskService: createTestTaskService(), diff --git a/greenfield/src/server/trpc/context.ts b/greenfield/src/server/trpc/context.ts index e1897a66e..0dfa980e4 100644 --- a/greenfield/src/server/trpc/context.ts +++ b/greenfield/src/server/trpc/context.ts @@ -21,6 +21,7 @@ import type { AutomationSecurityLifecycleService } from "../domains/security/aut import type { MfaAccountLifecycleService } from "../domains/security/mfa/accountLifecycle.ts"; import type { MfaLoginLifecycleService } from "../domains/security/mfa/loginLifecycle.ts"; import type { SecurityAuditLifecycleService } from "../domains/security/securityAuditLifecycle.ts"; +import type { ServiceActionsService } from "../domains/serviceActions/service.ts"; import type { SystemHealthDiagnosticsService } from "../domains/system/healthDiagnosticsService.ts"; import type { TaskService } from "../domains/tasks/service.ts"; import type { TerminalService } from "../domains/terminal/service.ts"; @@ -64,6 +65,7 @@ export interface RequestContextOptions { readonly requestId: string; readonly responseHeaders: Headers; readonly securityAuditLifecycle: SecurityAuditLifecycleService; + readonly serviceActionsService: ServiceActionsService; readonly systemHealthDiagnosticsService: SystemHealthDiagnosticsService; readonly taskService: TaskService["Service"]; readonly terminalService?: TerminalService; @@ -95,6 +97,7 @@ export interface RequestContext { readonly requestId: string; readonly responseHeaders: Headers; readonly securityAuditLifecycle: SecurityAuditLifecycleService; + readonly serviceActionsService: ServiceActionsService; readonly systemHealthDiagnosticsService: SystemHealthDiagnosticsService; readonly taskService: TaskService["Service"]; readonly terminalService?: TerminalService; @@ -147,6 +150,7 @@ export async function createRequestContext( requestId: options.requestId, responseHeaders: options.responseHeaders, securityAuditLifecycle: options.securityAuditLifecycle, + serviceActionsService: options.serviceActionsService, systemHealthDiagnosticsService: options.systemHealthDiagnosticsService, taskService: options.taskService, ...(options.terminalService === undefined diff --git a/greenfield/src/server/trpc/procedureErrorPolicy.ts b/greenfield/src/server/trpc/procedureErrorPolicy.ts index b28453313..efff3df5d 100644 --- a/greenfield/src/server/trpc/procedureErrorPolicy.ts +++ b/greenfield/src/server/trpc/procedureErrorPolicy.ts @@ -528,6 +528,13 @@ export const procedureExpectedErrorPolicy = freezeProcedureExpectedErrorPolicy({ "UNAUTHORIZED", ], "securityAudit.listEvents": ["FORBIDDEN", "UNAUTHORIZED"], + "serviceActions.getStatus": ["FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], + "serviceActions.request": [ + "CONFLICT", + "FORBIDDEN", + "SERVICE_UNAVAILABLE", + "UNAUTHORIZED", + ], "system.healthDiagnostics": ["FORBIDDEN", "UNAUTHORIZED"], "system.metrics": ["FORBIDDEN", "SERVICE_UNAVAILABLE", "UNAUTHORIZED"], "system.runtimeIdentity": [], diff --git a/greenfield/src/shared/databaseMigrationManifest.ts b/greenfield/src/shared/databaseMigrationManifest.ts index 2c9ed36f1..562b87fc2 100644 --- a/greenfield/src/shared/databaseMigrationManifest.ts +++ b/greenfield/src/shared/databaseMigrationManifest.ts @@ -13,8 +13,8 @@ export const migrationManifest = Object.freeze + | Readonly<{ status: "completed" }>; + +/** Worker-only fixed-operation authority; no command or path crosses this port. */ +export interface FixedHostOperationsExecutionPort { + readonly availableOperations: ( + signal?: AbortSignal + ) => Promise; + readonly request: ( + operationId: HostOperationId, + signal?: AbortSignal + ) => Promise; +} diff --git a/greenfield/src/shared/openClawServiceActions.ts b/greenfield/src/shared/openClawServiceActions.ts new file mode 100644 index 000000000..afe165b7b --- /dev/null +++ b/greenfield/src/shared/openClawServiceActions.ts @@ -0,0 +1,45 @@ +/** Secret-free aggregate from one fixed source-owned OpenClaw cleanup. */ +export interface OpenClawSessionsCleanupSummary { + readonly artifactsRemoved: number; + readonly bytesFreed: number; + readonly diskEntriesRemoved: number; + readonly diskFilesRemoved: number; + readonly dmScopesRetired: number; + readonly entriesAfter: number; + readonly entriesBefore: number; + readonly entriesCapped: number; + readonly entriesPruned: number; + readonly missingEntriesRemoved: number; + readonly modelRunsPruned: number; + readonly status: "completed"; + readonly storesProcessed: number; +} + +/** Secret-free settlement from one fixed source-owned OpenClaw update. */ +export interface OpenClawInstallationUpdateSummary { + readonly afterVersion?: string; + readonly beforeVersion?: string; + readonly status: "accepted" | "completed"; +} + +export type OpenClawServiceActionsExecutionErrorReason = + | "operation-failed" + | "unknown-outcome" + | "unavailable"; + +/** Sanitized worker-domain failure without upstream details or process output. */ +export class OpenClawServiceActionsExecutionError extends Error { + public readonly reason: OpenClawServiceActionsExecutionErrorReason; + + public constructor(reason: OpenClawServiceActionsExecutionErrorReason) { + super("OpenClaw Service Action failed"); + this.name = "OpenClawServiceActionsExecutionError"; + this.reason = reason; + } +} + +/** Worker-only authority for the two reviewed OpenClaw Service Actions. */ +export interface OpenClawServiceActionsExecutionPort { + cleanupSessions(signal?: AbortSignal): Promise; + updateInstallation(signal?: AbortSignal): Promise; +} diff --git a/greenfield/src/test/integration/openclaw/sourceAudit.test.ts b/greenfield/src/test/integration/openclaw/sourceAudit.test.ts index 494ee649f..8c0946448 100644 --- a/greenfield/src/test/integration/openclaw/sourceAudit.test.ts +++ b/greenfield/src/test/integration/openclaw/sourceAudit.test.ts @@ -1213,6 +1213,359 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise } //#endregion `, + "cleanup-service-fixture.js": ` + function serializeSessionCleanupResult(params) { + if (params.summaries.length === 1) return params.summaries[0] ?? {}; + return { + allAgents: true, + mode: params.mode, + dryRun: params.dryRun, + stores: params.summaries + }; + } + function pruneMissingTranscriptEntries(params) { return 0; } + async function previewStoreCleanup(params) { return params; } + /** Runs session cleanup preview/apply for the selected store targets. */ + async function runSessionsCleanup(params) { + const { cfg, opts } = params; + const maintenance = resolveMaintenanceConfig(); + const mode = opts.enforce ? "enforce" : maintenance.mode; + previewStoreCleanup({ + fixMissing: Boolean(opts.fixMissing), + fixDmScope: Boolean(opts.fixDmScope) + }); + const lifecycleResult = await applySqliteSessionEntryLifecycleMutation({ + activeSessionKey: opts.activeKey, + maintenanceOverride: { + ...maintenance, + mode + } + }); + const appliedUnreferencedArtifacts = mode === "warn" ? null : await pruneUnreferencedSessionArtifacts({}); + const appliedDiskBudget = await enforceSqliteSessionHistoryDiskBudget({}); + const missingApplied = 0; + const dmScopeRetiredApplied = 0; + const unreferencedArtifacts = appliedUnreferencedArtifacts; + const appliedReport = { + mode, + beforeCount: 2, + afterCount: 1, + modelRunPruned: 0, + pruned: 1, + capped: 0 + }; + const summary = { + agentId: target.agentId, + storePath: target.storePath, + mode: appliedReport.mode, + dryRun: false, + beforeCount: appliedReport.beforeCount, + afterCount: appliedReport.afterCount, + missing: missingApplied, + dmScopeRetired: dmScopeRetiredApplied, + modelRunPruned: appliedReport.modelRunPruned, + pruned: appliedReport.pruned, + capped: appliedReport.capped, + unreferencedArtifacts, + diskBudget: appliedDiskBudget, + wouldMutate: true, + applied: true, + appliedCount: lifecycleResult.afterCount + }; + return { mode, previewResults: [], appliedSummaries: [summary] }; + } + /** Purge session store entries for a deleted agent (#65524). Best-effort. */ + `, + "session-entry-slot-keys-fixture.js": ` + function collectSessionMaintenancePreserveKeys(baseKeys) { return new Set(baseKeys); } + function collectActiveSessionWorkAdmissionKeys(params) { return new Set(); } + /** Collects every runtime and active-work key protected from automatic maintenance. */ + function collectSessionMaintenancePreserveKeysForStore(params) { + const keys = collectSessionMaintenancePreserveKeys(params.baseKeys) ?? new Set(); + for (const key of collectActiveSessionWorkAdmissionKeys({ + storePath: params.storePath, + store: params.store + }) ?? []) keys.add(key); + return keys.size > 0 ? keys : void 0; + } + //#endregion + function isPrimarySessionMaintenanceKey(sessionKey) { return sessionKey === "main"; } + function isTelegramTopicSessionKey(sessionKey) { return false; } + function isExternalGroupOrChannelSessionKey(sessionKey) { return false; } + function isProtectedSessionMaintenanceEntry(sessionKey, entry) { + if (isPrimarySessionMaintenanceKey(sessionKey)) return true; + if (parseSessionThreadInfoFast(sessionKey).threadId) return true; + if (isTelegramTopicSessionKey(sessionKey)) return true; + if (isExternalGroupOrChannelSessionKey(sessionKey)) return true; + const chatType = normalizeLowercaseStringOrEmpty(entry?.chatType ?? sessionDeliveryOrigin(entry)?.chatType); + return chatType === "group" || chatType === "channel" || chatType === "thread"; + } + function shouldPreserveMaintenanceEntry(params) { + if (params.entry?.archivedAt !== void 0) return true; + return params.entry?.modelSelectionLocked === true || + params.preserveKeys?.has(params.key) === true || + isProtectedSessionMaintenanceEntry(params.key, params.entry); + } + function getActiveSessionMaintenanceWarning(params) { return null; } + function resolveMaintenanceConfig() { + let maintenance; + try { + maintenance = getRuntimeConfig().session?.maintenance; + } catch {} + return resolveMaintenanceConfigFromInput(maintenance); + } + async function pruneUnreferencedSessionArtifacts(params) { + return { + scannedFiles: files.length + promptBlobFiles.length, + removedFiles, + freedBytes, + olderThanMs + }; + } + async function enforceSessionDiskBudget(params) { + return { + totalBytesBefore: totalBefore, + totalBytesAfter: total, + removedFiles, + removedEntries, + freedBytes, + maxBytes, + highWaterBytes, + overBudget: true + }; + } + //#endregion + `, + "session-accessor.sqlite-fixture.js": ` + function collectSqliteSessionMaintenanceBaseKeys(store, activeSessionKey) { + const keys = []; + let currentKey = normalizeStoreSessionKey(activeSessionKey); + while (currentKey) { + keys.push(currentKey); + currentKey = normalizeStoreSessionKey(store[currentKey]?.parentSessionKey ?? ""); + } + return keys; + } + function hasStaleSqliteSessionEntryCandidate() { return false; } + function applySqliteSessionEntryMaintenance(database, params) { + const store = {}; + const preserveKeys = collectSessionMaintenancePreserveKeysForStore({ + storePath: params.storePath, + store, + baseKeys: collectSqliteSessionMaintenanceBaseKeys(store, params.activeSessionKey) + }) ?? new Set(); + pruneStaleEntries(store, maintenance.pruneAfterMs, { preserveKeys }); + capEntryCount(store, maintenance.maxEntries, { preserveKeys }); + return { entryRemovals: [], stateDeletePlans: [] }; + } + function finalizeSqliteSessionEntryMaintenancePlansBestEffort(scope, plans) { return []; } + /** Applies exact lifecycle removals/upserts using SQLite session rows. */ + async function applySqliteSessionEntryLifecycleMutation(params) { + if (!sqliteSessionEntriesEqual(entry, removal.expectedEntry)) throw new Error("changed"); + applySqliteSessionEntryMaintenance(database, { + activeSessionKey: params.activeSessionKey ?? "", + forceMaintenance: params.maintenanceOverride !== void 0, + maintenanceConfig: params.maintenanceOverride ? { + ...resolveMaintenanceConfig(), + ...params.maintenanceOverride + } : void 0 + }); + return { afterCount: 1 }; + } + /** Purges entries owned by a deleted agent from SQLite session rows. */ + `, + "update-fixture.js": ` + const MANAGED_HANDOFF_RESTART_DELAY_MS = 2e3; + function hasManagedServiceHandoffContext(env, supervisor) { + if (supervisor === "systemd") return Boolean(env.OPENCLAW_SYSTEMD_UNIT?.trim()); + return false; + } + function resolveManagedServiceHandoffRestartDelayMs(restartDelayMs, supervisor) { + const resolvedDelayMs = restartDelayMs ?? MANAGED_HANDOFF_RESTART_DELAY_MS; + if (supervisor !== "systemd") return resolvedDelayMs; + return Math.max(resolvedDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS); + } + const updateHandlers = { + "update.status": async () => {}, + "update.run": async ({ params, respond, client, context }) => { + if (!assertValidParams(params, validateUpdateRunParams, "update.run", respond)) return; + const timeoutMsRaw = params.timeoutMs; + const timeoutMs = typeof timeoutMsRaw === "number" && Number.isFinite(timeoutMsRaw) ? Math.max(1e3, Math.floor(timeoutMsRaw)) : void 0; + const installSurface = {}; + const supervisor = "systemd"; + const hasHandoffContext = supervisor ? hasManagedServiceHandoffContext(process.env, supervisor) : false; + const requiresManagedServiceHandoff = installSurface.kind === "global" || installSurface.kind === "git" && supervisor !== null; + let result; + let handoff = null; + let managedHandoffRestart = null; + let ownsManagedServiceHandoff = true; + if (requiresManagedServiceHandoff && hasHandoffContext) { + const started = await startManagedServiceUpdateHandoff({ timeoutMs }); + ownsManagedServiceHandoff = started.status === "started"; + if (ownsManagedServiceHandoff) { + handoff = { + status: "started", + ...started.pid ? { pid: started.pid } : {}, + command: started.command + }; + managedHandoffRestart = scheduleGatewaySigusr1Restart({ + reason: "update.run", + skipDeferral: true, + skipCooldown: true + }); + } else handoff = { + status: "already-running", + command: started.command, + message: "Another managed update is already running; retry after it completes." + }; + } else { + result = await runGatewayUpdate({ + timeoutMs, + allowGatewayServiceRepair: false, + allowGatewayActivation: false + }); + } + const payload = buildUpdateRestartSentinelPayload({ result, meta: {} }); + let sentinelPersisted = false; + if (ownsManagedServiceHandoff) try { + await writeRestartSentinel(payload); + sentinelPersisted = true; + } catch {} + const updateWasPackageSwap = result.status === "ok" && result.mode !== "git"; + const restart = managedHandoffRestart ?? (result.status === "ok" ? scheduleGatewaySigusr1Restart({ + delayMs: updateWasPackageSwap ? 0 : undefined, + reason: "update.run", + skipDeferral: updateWasPackageSwap, + skipCooldown: updateWasPackageSwap + }) : null); + respond(true, { + ok: result.status === "ok" || handoff?.status === "started", + result, + ...handoff ? { handoff } : {}, + restart, + sentinel: { + persisted: sentinelPersisted, + payload + } + }, void 0); + } + }; + //#endregion + `, + "update-startup-fixture.js": ` + const HANDOFF_READY_TIMEOUT_MS = 3e4; + const HANDOFF_READY_MARKER = "OPENCLAW_UPDATE_HANDOFF_READY\\n"; + const HANDOFF_SCRIPT = String.raw\` + function cleanupSensitiveFiles() {} + cleanupSensitiveFiles(); + \`; + function resolveUpdateCliArgv(params) { + const updateArgs = ["update", "--yes", "--json"]; + if (typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) updateArgs.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1e3)))); + return ["openclaw", ...updateArgs]; + } + function formatManagedServiceUpdateCommand(params) { + const args = ["openclaw", "update", "--yes"]; + if (typeof params?.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) args.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1e3)))); + return args.join(" "); + } + function resolveGatewayServiceRecovery(supervisor, env) { return {}; } + async function waitForHandoffReady(child) { + if (buffered.includes(HANDOFF_READY_MARKER)) finish(); + setTimeout(() => finish(new Error("managed update handoff did not signal readiness within 30 seconds")), HANDOFF_READY_TIMEOUT_MS); + } + async function resolveHandoffSpawn(params) { + return { args: ["--user", "--scope", "--collect"] }; + } + async function spawnManagedServiceUpdateHandoff(params, onExit) { + const helperParams = { + sensitivePaths: [scriptPath, paramsPath, metaPath] + }; + const child = spawn(command, args, { detached: true }); + child.unref(); + return { status: "started", command: "openclaw update --yes" }; + } + async function startManagedServiceUpdateHandoff(params) { + const active = activeManagedServiceUpdateHandoff; + if (active) return { + ...await active, + status: "joined" + }; + return await spawnManagedServiceUpdateHandoff(params, () => {}); + } + function buildManagedServiceHandoffUnavailableMessage(command) { return command; } + `, + "update-runner-fixture.js": ` + const MAX_LOG_CHARS = 8e3; + async function runStep(opts) { + const { runCommand, name, argv, cwd, timeoutMs, progress, stepIndex, totalSteps } = opts; + const command = argv.join(" "); + const result = await runCommand(argv, { + cwd, + timeoutMs, + env + }); + const stderrTail = trimLogTail(result.stderr, MAX_LOG_CHARS); + return { + name, + command, + cwd, + durationMs: 1, + exitCode: result.code, + stdoutTail: trimLogTail(result.stdout, MAX_LOG_CHARS), + stderrTail, + signal: result.signal + }; + } + function normalizeFallbackFailureReason(stepName) { return "unexpected-error"; } + function successfulUpdateResult() { return { status: "ok" }; } + async function runGatewayUpdate(opts = {}) { + const timeoutMs = opts.timeoutMs ?? 12e5; + if (gitRoot) return await runGitUpdate({ timeoutMs }); + if (globalManager) return await runGlobalUpdate({ timeoutMs }); + return { + status: "skipped", + mode: "unknown", + root: pkgRoot, + reason: "not-git-install", + before: { version: beforeVersion }, + steps: [], + durationMs: 0 + }; + } + //#endregion + `, + "update-control-plane-sentinel-fixture.js": ` + function buildUpdateRestartSentinelPayload(params) { + const { result, meta } = params; + return { + kind: "update", + status: result.status, + message: meta.note ?? null, + doctorHint: formatDoctorNonInteractiveHint(), + stats: { + mode: result.mode, + ...result.root ? { root: result.root } : {}, + ...meta.handoffId ? { handoffId: meta.handoffId } : {}, + before: result.before ?? null, + after: result.after ?? null, + steps: result.steps.map((step) => ({ + command: step.command, + cwd: step.cwd, + log: { + stdoutTail: step.stdoutTail ?? null, + stderrTail: step.stderrTail ?? null + } + })) + } + }; + } + //#endregion + const CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON = "managed-service-handoff-started"; + function isPendingControlPlaneUpdateRestartSentinel(payload) { + return payload.stats?.reason === CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON; + } + `, "reset-policy-fixture.js": ` const DEFAULT_RESET_MODE = "none"; const DEFAULT_RESET_AT_HOUR = 4; @@ -1268,6 +1621,7 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise { name: "sessions.compact", scope: "operator.admin" }, { name: "sessions.delete", scope: "dynamic" }, { name: "sessions.list", scope: "operator.read" }, + { name: "sessions.cleanup", scope: "operator.admin" }, { name: "sessions.reset", scope: "operator.admin" }, { name: "sessions.subscribe", scope: "operator.read" }, { name: "cron.get", scope: "operator.read" }, @@ -1280,6 +1634,7 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise { name: "config.patch", scope: "operator.admin", controlPlaneWrite: true }, { name: "skills.status", scope: "operator.read" }, { name: "skills.update", scope: "operator.admin" }, + { name: "update.run", scope: "operator.admin", controlPlaneWrite: true }, `, "method-scopes-fixture.js": ` /** @@ -1350,6 +1705,17 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise \treplacePaths: Type.Optional(Type.Array(NonEmptyString, { maxItems: 256 })) }); /** Empty request payload for fetching the generated config schema. */ + const UpdateStatusParamsSchema = closedObject({}); + /** Request payload for running an update/restart flow with optional channel delivery context. */ + const UpdateRunParamsSchema = closedObject({ +\tsessionKey: Type.Optional(Type.String()), +\tdeliveryContext: Type.Optional(ConfigDeliveryContextSchema), +\tnote: Type.Optional(Type.String()), +\tcontinuationMessage: Type.Optional(Type.String()), +\trestartDelayMs: Type.Optional(Type.Integer({ minimum: 0 })), +\ttimeoutMs: Type.Optional(Type.Integer({ minimum: 1 })) + }); + /** UI metadata attached to config schema paths. */ /** Reads installed skill status, optionally for a selected agent. */ const SkillsStatusParamsSchema = closedObject({ agentId: Type.Optional(NonEmptyString) }); /** Empty request payload for listing available skill bins. */ @@ -1608,6 +1974,18 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise }); /** Searches one agent's indexed session transcripts */ + /** Repairs or removes invalid session records from the selected agent scope. */ + const SessionsCleanupParamsSchema = closedObject({ +\tagent: Type.Optional(NonEmptyString), +\tallAgents: Type.Optional(Type.Boolean()), +\tenforce: Type.Optional(Type.Boolean()), +\tactiveKey: Type.Optional(NonEmptyString), +\tfixMissing: Type.Optional(Type.Boolean()), +\tfixDmScope: Type.Optional(Type.Boolean()) + }); + /** Reads short previews for selected session keys. */ + const SessionsPreviewParamsSchema = closedObject({}); + /** Subscribes a client to live message updates for one session. */ const SessionsMessagesSubscribeParamsSchema = closedObject({ \tkey: NonEmptyString, @@ -2168,6 +2546,31 @@ async function writeSyntheticOpenClawPackage(sourceRoot: string): Promise activeRunIds: activeRunState.runIds } : {} }), + "sessions.cleanup": async ({ params, respond, context }) => { + if (!assertValidParams(params, validateSessionsCleanupParams, "sessions.cleanup", respond)) return; + try { + const { mode, appliedSummaries } = await runSessionsCleanup({ + cfg: context.getRuntimeConfig(), + opts: { + agent: params.agent, + allAgents: params.allAgents, + enforce: params.enforce, + activeKey: params.activeKey, + fixMissing: params.fixMissing, + fixDmScope: params.fixDmScope + } + }); + respond(true, serializeSessionCleanupResult({ + mode, + dryRun: false, + summaries: appliedSummaries + }), void 0); + emitSessionsChanged(context, { reason: "cleanup" }); + } catch (error) { + respond(false, void 0, errorShape(ErrorCodes.INVALID_REQUEST, formatErrorMessage(error))); + } + }, + "sessions.preview": async () => {}, "sessions.patch": async () => { const expectedSessionChanged = p.expectedSessionId !== void 0 && currentLifecycleEntry?.sessionId !== p.expectedSessionId; respond(true, { @@ -2611,7 +3014,7 @@ describe("reviewed OpenClaw protocol fixtures", () => { "chat-delta", "chat-terminal", ]); - expect(reviewed.audit.sourceArtifacts).toHaveLength(83); + expect(reviewed.audit.sourceArtifacts).toHaveLength(90); expect(reviewed.audit.chat.adapter.media.localHistory).toEqual({ canonical: { fields: [ @@ -3226,6 +3629,7 @@ describe("reviewed OpenClaw protocol fixtures", () => { "cron.json", "gateway.json", "manifest.json", + "operations.json", "sessions.json", "settings.json", "tasks.json", @@ -3418,7 +3822,72 @@ describe("explicit OpenClaw source audit", () => { schedulerSuccess: { ok: true }, sentinelRequiresRestartPath: "sentinel.payload.stats.requiresRestart", }); - expect(audit.sourceArtifacts).toHaveLength(83); + expect(audit.operations.methodAccess).toEqual([ + { + controlPlaneWrite: false, + lane: "one-shot-admin", + method: "sessions.cleanup", + scope: "operator.admin", + }, + { + controlPlaneWrite: true, + lane: "one-shot-admin", + method: "update.run", + scope: "operator.admin", + }, + ]); + expect(audit.operations.sessionsCleanup).toMatchObject({ + outcome: { + automaticReplaySafe: false, + handlerTimeoutParameter: false, + idempotencyParameter: false, + postDispatchTransportTimeout: "outcome-unknown", + }, + request: { + acceptedParams: [ + "activeKey", + "agent", + "allAgents", + "enforce", + "fixDmScope", + "fixMissing", + ], + closedObject: true, + requiredParams: [], + }, + response: { + sensitivePaths: ["storePath", "stores[].storePath"], + }, + }); + expect(audit.operations.updateRun).toMatchObject({ + managedHandoff: { + readyMarkerTimeoutMs: 30_000, + sensitiveTemporaryFilesRemoved: true, + startedHandoffCountsAsAccepted: true, + }, + outcome: { + automaticReplaySafe: false, + handlerAbortSignal: false, + idempotencyParameter: false, + operationalErrorsUseRpcSuccess: true, + postDispatchTransportTimeout: "outcome-unknown", + }, + response: { + sentinelPersistenceBestEffort: true, + sensitivePaths: expect.arrayContaining([ + "handoff.command", + "result.root", + "result.steps[].stdoutTail", + "sentinel.payload", + ]), + }, + timeout: { + defaultRunnerPerStepMs: 1_200_000, + handlerFloorMs: 1000, + perStepRatherThanWholeOperation: true, + }, + }); + expect(audit.sourceArtifacts).toHaveLength(90); expect(audit.chat.adapter.media.localHistory.precedence.url).toEqual([ "canonical.url", "MediaUrls[index]", @@ -3518,6 +3987,59 @@ describe("explicit OpenClaw source audit", () => { ); }); + test("rejects drift in privileged operations access and execution facts", async () => { + const driftCases = [ + { + expected: "permission descriptor changed for sessions.cleanup", + fileName: "core-descriptors-fixture.js", + from: '{ name: "sessions.cleanup", scope: "operator.admin" }', + to: '{ name: "sessions.cleanup", scope: "operator.read" }', + }, + { + expected: "update.run optional params changed", + fileName: "src-fixture.js", + from: "timeoutMs: Type.Optional(Type.Integer({ minimum: 1 }))", + to: "timeoutMs: Type.Optional(Type.Integer({ minimum: 0 }))", + }, + { + expected: "sessions.cleanup execution changed", + fileName: "cleanup-service-fixture.js", + from: 'const appliedUnreferencedArtifacts = mode === "warn" ? null', + to: 'const appliedUnreferencedArtifacts = mode === "enforce" ? null', + }, + { + expected: "update.run handler changed", + fileName: "update-fixture.js", + from: 'ok: result.status === "ok" || handoff?.status === "started"', + to: "ok: true", + }, + ] as const; + + for (const driftCase of driftCases) { + await withTemporaryDirectory( + "mira-openclaw-operations-drift-", + async (sourceRoot) => { + await writeSyntheticOpenClawPackage(sourceRoot); + const artifactPath = path.join( + sourceRoot, + "dist", + driftCase.fileName + ); + const source = await readFile(artifactPath, "utf8"); + expect(source).toContain(driftCase.from); + await writeFile( + artifactPath, + source.replace(driftCase.from, driftCase.to), + "utf8" + ); + + const error = await rejectedError(auditInstalledOpenClaw(sourceRoot)); + expect(error.message).toContain(driftCase.expected); + } + ); + } + }); + test("rejects drift in the system.info read permission", async () => { await withTemporaryDirectory( "mira-openclaw-system-scope-", diff --git a/greenfield/src/test/parity/fixtures/greenfield-contracts.json b/greenfield/src/test/parity/fixtures/greenfield-contracts.json index bc4c4a84d..b890938e0 100644 --- a/greenfield/src/test/parity/fixtures/greenfield-contracts.json +++ b/greenfield/src/test/parity/fixtures/greenfield-contracts.json @@ -478,6 +478,14 @@ "kind": "query", "name": "securityAudit.listEvents" }, + { + "kind": "query", + "name": "serviceActions.getStatus" + }, + { + "kind": "mutation", + "name": "serviceActions.request" + }, { "kind": "query", "name": "system.healthDiagnostics" diff --git a/greenfield/src/test/parity/fixtures/legacy-endpoints.json b/greenfield/src/test/parity/fixtures/legacy-endpoints.json index e25f62b79..5aeca4cab 100644 --- a/greenfield/src/test/parity/fixtures/legacy-endpoints.json +++ b/greenfield/src/test/parity/fixtures/legacy-endpoints.json @@ -506,12 +506,16 @@ "id": "GET /api/exec/:jobId", "method": "GET", "path": "/api/exec/:jobId", - "purpose": "Reads persisted exec output/state.", + "purpose": "Reads durable fixed-action state or the caller's active interactive terminal state without retaining PTY contents.", "section": "Exec And Terminal", "target": { - "delivery": "planned", + "delivery": "implemented", "kind": "procedure", - "names": ["terminal.getExecution"], + "names": [ + "jobs.getRun", + "serviceActions.getStatus", + "terminal.getActiveSession" + ], "phase": "phase-5" } }, @@ -1567,10 +1571,9 @@ "purpose": "Queues one command and observes its persisted result.", "section": "Exec And Terminal", "target": { - "delivery": "planned", - "kind": "procedure", - "names": ["terminal.startExecution"], - "phase": "phase-5" + "consumerEvidence": "no-current-consumers", + "kind": "reviewed-removal", + "reason": "No production browser route or scoped automation caller uses the synchronous generic command endpoint; interactive work runs inside the bounded PTY and fixed privileged operations use purpose-built durable Service Actions jobs." } }, { @@ -1580,9 +1583,9 @@ "purpose": "Requests exec cancellation.", "section": "Exec And Terminal", "target": { - "delivery": "planned", + "delivery": "implemented", "kind": "procedure", - "names": ["terminal.stopExecution"], + "names": ["terminal.terminateSession"], "phase": "phase-5" } }, @@ -1593,9 +1596,9 @@ "purpose": "Queues a worker-owned long-running exec job.", "section": "Exec And Terminal", "target": { - "delivery": "planned", + "delivery": "implemented", "kind": "procedure", - "names": ["terminal.startExecution"], + "names": ["serviceActions.request", "terminal.prepareSession"], "phase": "phase-5" } }, @@ -1945,9 +1948,10 @@ "purpose": "Resolves validated directory changes.", "section": "Exec And Terminal", "target": { - "delivery": "planned", - "kind": "procedure", - "names": ["terminal.resolveDirectory"], + "delivery": "implemented", + "kind": "raw-http", + "method": "GET", + "path": "/api/terminal/sessions/:sessionId/socket", "phase": "phase-5" } }, @@ -1958,9 +1962,10 @@ "purpose": "Returns shell/path completions.", "section": "Exec And Terminal", "target": { - "delivery": "planned", - "kind": "procedure", - "names": ["terminal.complete"], + "delivery": "implemented", + "kind": "raw-http", + "method": "GET", + "path": "/api/terminal/sessions/:sessionId/socket", "phase": "phase-5" } }, diff --git a/greenfield/src/test/parity/parityInventory.test.ts b/greenfield/src/test/parity/parityInventory.test.ts index ce158fe13..c42f9f69a 100644 --- a/greenfield/src/test/parity/parityInventory.test.ts +++ b/greenfield/src/test/parity/parityInventory.test.ts @@ -90,7 +90,7 @@ describe("reviewed pre-cutover parity inventory", () => { "phase-2": 28, "phase-3": 39, "phase-4": 15, - "phase-5": 68, + "phase-5": 67, }); }); @@ -265,6 +265,67 @@ describe("reviewed pre-cutover parity inventory", () => { ]); }); + test("records the purpose-built Service Actions and interactive PTY exec replacement", async () => { + const reviewed = await loadReviewedParityInventory(); + const endpoints = reviewed.legacyEndpoints.endpoints.filter( + ({ section }) => section === "Exec And Terminal" + ); + + expect( + endpoints.map(({ id, target }) => { + const identity = + target.kind === "procedure" + ? target.names + : target.kind === "raw-http" + ? `${target.method} ${target.path}` + : target.consumerEvidence; + return [ + id, + target.kind === "reviewed-removal" + ? target.kind + : target.delivery, + identity, + ]; + }) + ).toEqual([ + [ + "GET /api/exec/:jobId", + "implemented", + [ + "jobs.getRun", + "serviceActions.getStatus", + "terminal.getActiveSession", + ], + ], + ["POST /api/exec", "reviewed-removal", "no-current-consumers"], + [ + "POST /api/exec/:jobId/stop", + "implemented", + ["terminal.terminateSession"], + ], + [ + "POST /api/exec/start", + "implemented", + ["serviceActions.request", "terminal.prepareSession"], + ], + [ + "POST /api/terminal/cd", + "implemented", + "GET /api/terminal/sessions/:sessionId/socket", + ], + [ + "POST /api/terminal/complete", + "implemented", + "GET /api/terminal/sessions/:sessionId/socket", + ], + ]); + expect(endpoints[1]?.target).toMatchObject({ + consumerEvidence: "no-current-consumers", + kind: "reviewed-removal", + reason: expect.stringContaining("synchronous generic command endpoint"), + }); + }); + test("records the bounded OpenClaw settings and operations slice", async () => { const reviewed = await loadReviewedParityInventory(); const settingsRoute = reviewed.frontend.routes.find( From 1abd736367ea30beda9207331482ed6deb79eea9 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 11:49:25 +0200 Subject: [PATCH 02/13] fix(greenfield): close service actions lint gate --- .../src/server/domains/jobs/repository.ts | 2 +- .../server/domains/serviceActions/service.ts | 28 ++++++++--------- .../src/test/parity/parityInventory.test.ts | 30 +++++++------------ 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index 44724f1ff..4e8323ccf 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -91,9 +91,9 @@ import { scheduledJobInsertSchema, scheduledJobSelectSchema, } from "../../database/validation/scheduledJobs.ts"; -import { parseWorkerActionKeysJson } from "../../database/validation/workerActionKeys.ts"; import { canonicalWorkerActionKeys, + parseWorkerActionKeysJson, workerActionKeysSchema, } from "../../database/validation/workerActionKeys.ts"; import { diff --git a/greenfield/src/server/domains/serviceActions/service.ts b/greenfield/src/server/domains/serviceActions/service.ts index 1e46942ec..9d74494d1 100644 --- a/greenfield/src/server/domains/serviceActions/service.ts +++ b/greenfield/src/server/domains/serviceActions/service.ts @@ -116,11 +116,11 @@ export function createServiceActionsService( ...(jobRunId === undefined ? {} : { jobRunId }), settlement, }); - } catch (cause) { + } catch (error) { try { options.onAuditSettlementFailure?.({ actionId: input.actionId, - cause, + cause: error, settlement, }); } catch { @@ -197,18 +197,18 @@ export function createServiceActionsService( await settleAudit(parsed, context, "failed"); throw error; } - const mapped = - error instanceof ServiceActionQueueError - ? queueFailure(error) - : error instanceof v.ValiError - ? new ServiceActionsServiceError("unknown-outcome", { - cause: error, - }) - : signal?.aborted - ? error - : new ServiceActionsServiceError("unavailable", { - cause: error, - }); + let mapped: unknown; + if (error instanceof ServiceActionQueueError) { + mapped = queueFailure(error); + } else if (error instanceof v.ValiError) { + mapped = new ServiceActionsServiceError("unknown-outcome", { + cause: error, + }); + } else { + mapped = signal?.aborted + ? error + : new ServiceActionsServiceError("unavailable", { cause: error }); + } await settleAudit( parsed, context, diff --git a/greenfield/src/test/parity/parityInventory.test.ts b/greenfield/src/test/parity/parityInventory.test.ts index c42f9f69a..116bb7596 100644 --- a/greenfield/src/test/parity/parityInventory.test.ts +++ b/greenfield/src/test/parity/parityInventory.test.ts @@ -273,17 +273,17 @@ describe("reviewed pre-cutover parity inventory", () => { expect( endpoints.map(({ id, target }) => { - const identity = - target.kind === "procedure" - ? target.names - : target.kind === "raw-http" - ? `${target.method} ${target.path}` - : target.consumerEvidence; + let identity: readonly string[] | string; + if (target.kind === "procedure") { + identity = target.names; + } else if (target.kind === "raw-http") { + identity = `${target.method} ${target.path}`; + } else { + identity = target.consumerEvidence; + } return [ id, - target.kind === "reviewed-removal" - ? target.kind - : target.delivery, + target.kind === "reviewed-removal" ? target.kind : target.delivery, identity, ]; }) @@ -291,18 +291,10 @@ describe("reviewed pre-cutover parity inventory", () => { [ "GET /api/exec/:jobId", "implemented", - [ - "jobs.getRun", - "serviceActions.getStatus", - "terminal.getActiveSession", - ], + ["jobs.getRun", "serviceActions.getStatus", "terminal.getActiveSession"], ], ["POST /api/exec", "reviewed-removal", "no-current-consumers"], - [ - "POST /api/exec/:jobId/stop", - "implemented", - ["terminal.terminateSession"], - ], + ["POST /api/exec/:jobId/stop", "implemented", ["terminal.terminateSession"]], [ "POST /api/exec/start", "implemented", From 2f88a415e1c5574f440fc250775175094aef82c5 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 12:25:17 +0200 Subject: [PATCH 03/13] test(greenfield): advertise cache claim action --- greenfield/src/server/domains/cache/repository.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/greenfield/src/server/domains/cache/repository.test.ts b/greenfield/src/server/domains/cache/repository.test.ts index 9dd25aa61..b108c6cf1 100644 --- a/greenfield/src/server/domains/cache/repository.test.ts +++ b/greenfield/src/server/domains/cache/repository.test.ts @@ -63,7 +63,9 @@ async function runningClaim( await jobs.registerWorker({ ...noSideEffects, worker: { - actionKeysJson: "[]", + actionKeysJson: JSON.stringify([ + options.actionKey ?? "cache.refresh.system-host", + ]), capacity: 1, drainingAt: null, heartbeatAt: new Date(1000), From 506d852bc25a239a8f1c621735316d286a08f982 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 12:36:02 +0200 Subject: [PATCH 04/13] fix(greenfield): settle retired immutable jobs --- .../src/server/domains/jobs/repository.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index 4e8323ccf..f412c52a7 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -5,6 +5,7 @@ import { count, desc, eq, + exists, gte, gt, inArray, @@ -2147,6 +2148,21 @@ class DrizzleJobWriter extends DrizzleJobReader { if (activeCount >= worker.capacity) return { kind: "worker-unavailable" }; const workerActionKeys = parseWorkerActionKeysJson(worker.actionKeysJson); if (workerActionKeys.length === 0) return { kind: "empty" }; + const retiredNeverRun = and( + eq(jobRuns.cancellationPolicy, "never"), + isNotNull(jobRuns.scheduledJobId), + exists( + this.#transaction + .select({ id: scheduledJobs.id }) + .from(scheduledJobs) + .where( + and( + eq(scheduledJobs.id, jobRuns.scheduledJobId), + eq(scheduledJobs.enabled, false) + ) + ) + ) + ); const availableThrough = input.cursor?.availableThrough ?? input.at; const candidates: JobRunRecord[] = []; @@ -2161,7 +2177,10 @@ class DrizzleJobWriter extends DrizzleJobReader { and( eq(jobRuns.state, "queued"), lte(jobRuns.availableAt, availableThrough), - inArray(jobRuns.actionKey, workerActionKeys), + or( + inArray(jobRuns.actionKey, workerActionKeys), + retiredNeverRun + ), range ) ) From dacc3661b71ae0241bd34c0c3b070dc0d426c268 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 13:45:01 +0200 Subject: [PATCH 05/13] fix(greenfield): close service action release gates --- .../application-architecture.md | 16 ++- .../greenfield-rewrite/data-and-security.md | 5 + .../greenfield-rewrite/implementation-plan.md | 4 +- .../greenfield-rewrite/progress.md | 32 ++--- .../runtime-and-delivery.md | 5 + .../migration.sql | 31 ++++- .../snapshot.json | 2 +- .../sourceBoundaries/sourceTopologyPolicy.ts | 1 + greenfield/src/app/dashboardServer.ts | 6 +- greenfield/src/app/worker.ts | 2 +- .../browser/overview/OverviewRoute.test.tsx | 7 +- greenfield/src/contracts/jobModel.ts | 33 +++++- .../src/contracts/serviceActions.test.ts | 34 ++++++ .../database/migrations/jobsSchema.test.ts | 88 ++++++++++++++ .../src/server/database/schema/jobChecks.ts | 31 +++++ .../src/server/database/schema/jobRuns.ts | 3 +- .../server/database/schema/workerInstances.ts | 3 +- .../database/validation/jobRunEvents.ts | 8 +- .../src/server/database/validation/jobRuns.ts | 4 +- .../database/validation/rowSchemas.test.ts | 29 +++++ .../database/validation/workerActionKeys.ts | 61 ---------- .../database/validation/workerInstances.ts | 61 +++++++++- .../server/database/workerActionKeyPolicy.ts | 4 - .../server/domains/jobs/actionExecutors.ts | 26 +++- .../server/domains/jobs/coordinator.test.ts | 4 +- .../src/server/domains/jobs/coordinator.ts | 14 ++- .../server/domains/jobs/repository.test.ts | 74 +++++++++++- .../src/server/domains/jobs/repository.ts | 106 ++++++++++++----- .../src/server/domains/jobs/service.test.ts | 30 +++-- greenfield/src/server/domains/jobs/service.ts | 12 ++ .../domains/jobs/serviceActionQueue.test.ts | 43 +++++-- .../server/domains/jobs/serviceActionQueue.ts | 23 +++- .../src/server/domains/jobs/workerRuntime.ts | 6 +- .../serviceActions/operationAudit.test.ts | 72 ------------ .../domains/serviceActions/operationAudit.ts | 86 -------------- .../domains/serviceActions/procedures.ts | 10 -- .../server/domains/serviceActions/routes.ts | 9 ++ .../domains/serviceActions/service.test.ts | 83 ++++++++++++- .../server/domains/serviceActions/service.ts | 111 +++++++++++++++--- greenfield/src/server/trpc/appRouter.ts | 2 +- .../src/shared/databaseMigrationManifest.ts | 4 +- greenfield/src/shared/hostOperations.ts | 24 ---- .../parity/fixtures/legacy-endpoints.json | 4 +- .../src/test/parity/parityInventory.test.ts | 7 +- 44 files changed, 831 insertions(+), 389 deletions(-) delete mode 100644 greenfield/src/server/database/validation/workerActionKeys.ts delete mode 100644 greenfield/src/server/database/workerActionKeyPolicy.ts delete mode 100644 greenfield/src/server/domains/serviceActions/operationAudit.test.ts delete mode 100644 greenfield/src/server/domains/serviceActions/operationAudit.ts delete mode 100644 greenfield/src/server/domains/serviceActions/procedures.ts delete mode 100644 greenfield/src/shared/hostOperations.ts diff --git a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md index a2e1f4b6e..66498227c 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md +++ b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md @@ -464,7 +464,7 @@ is exclusive, caller-idempotent, single-attempt, non-retry-safe, and non-cancell worker owns its fixed no-shell lifecycle command. Ambiguous enqueue or terminal settlement is reconciled by durable run identity and never blindly dispatches a second restart. -### Purpose-built Service Actions replace generic exec +### Purpose-built Service Actions partially replace generic exec consumers The Overview exposes exactly four fixed Service Actions through `serviceActions.getStatus` and `serviceActions.request`: OpenClaw session cleanup, OpenClaw @@ -478,9 +478,12 @@ OpenClaw cleanup and update are implemented worker-only through the hash-pinned `sessions.cleanup` and `update.run` Gateway methods. Their providers accept no browser parameters, persist only bounded schema-validated summaries, never return raw Gateway results, and never blindly replay a post-dispatch unknown outcome. Cleanup deliberately uses OpenClaw's source-owned -session/artifact maintenance instead of reproducing legacy recursive deletion. The legacy broad -`system_cleanup` behavior is not restored: package, journal, and Docker deletion cross separate -ownership domains, and Docker cleanup remains part of the Docker domain. +session/artifact maintenance instead of reproducing legacy recursive deletion. These safe +replacements do not yet close `POST /api/exec/start`: the legacy broad `system_cleanup` behavior +crosses separate ownership domains and remains planned as three explicit capabilities. Docker +prune belongs to the Docker slice, apt cleanup belongs to host/package maintenance, and journald +vacuum belongs to log maintenance. The row stays open until all three are delivered or separately +reviewed without removing their operator-visible behavior. The contract and Overview retain fixed rows for host restart and host update, but production marks both unavailable. The current web and worker processes share one Unix identity, so a group- or @@ -493,8 +496,9 @@ The interactive PTY remains the sole terminal boundary. Shell `cd` and completio connected shell/readline protocol, termination uses the bounded terminal session control, and no new generic command, cwd, or completion API is introduced. The unused synchronous `POST /api/exec` endpoint is a reviewed removal because no current browser or scoped automation consumer depends on -it; legacy long-running exec consumers map to either the PTY or the fixed durable Service Actions -queue. +it. Implemented long-running exec consumers map to either the PTY or the fixed durable Service +Actions queue, while the inventory keeps `POST /api/exec/start` planned for the outstanding +cleanup decomposition. ### Current-protocol Control UI projections diff --git a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md index fb0af76fa..61eb23c84 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md +++ b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md @@ -299,6 +299,11 @@ enablement requires a distinct worker OS identity, root-owned immutable worker e subject and operation policy, and reviewed install/rollback evidence before either action key can be advertised. +This boundary is a partial secure replacement for `POST /api/exec/start`, not a feature-removal +claim. The legacy `system_cleanup` intent remains planned as three separately authorized effects: +Docker prune in the Docker slice, apt cleanup in host/package maintenance, and journald vacuum in +log maintenance. None may be smuggled back through a generic shell or shared-user privilege grant. + The `cache:read` automation heartbeat is a separate sanitized projection, not a shortcut around session, task, job, or cron detail authorization. It reads process-local validated Gateway summaries plus bounded payload-free cache status and purpose-built SQLite task/Dashboard-job diff --git a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md index 3b033d126..3e8d046ee 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md +++ b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md @@ -103,7 +103,9 @@ including restart during streaming. filesystem isolation requires a separate mount, namespace, or container sandbox. - keep shell `cd`, completion, and termination inside the implemented bounded PTY. Replace consumed legacy exec behavior only with purpose-built durable Service Actions; do not restore a generic - command, shell, or cwd API for the unused synchronous exec route. + command, shell, or cwd API for the unused synchronous exec route. Keep `POST /api/exec/start` + planned until `system_cleanup` is decomposed without feature loss: Docker prune in the Docker + slice, apt cleanup in host/package maintenance, and journald vacuum in log maintenance. - expose the four fixed Service Action intents in contract/UI, but advertise only exact executors owned by a fresh worker on the current release. OpenClaw cleanup/update use reviewed worker-only Gateway methods. Host restart/update remain unavailable until web and worker have distinct OS diff --git a/greenfield/docs/architecture/greenfield-rewrite/progress.md b/greenfield/docs/architecture/greenfield-rewrite/progress.md index dce0ebe09..ddb03b4b9 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/progress.md +++ b/greenfield/docs/architecture/greenfield-rewrite/progress.md @@ -7,15 +7,15 @@ This matrix is the living phase status. Update it in the same change that materially advances or closes a phase; dated entries below provide the evidence, not a second status source. -| Phase | Status | Current evidence and remaining gate | -| ----------------------------------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | -| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | -| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | -| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | -| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | -| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Overview Service Actions replace the consumed legacy exec flows with four fixed intents; OpenClaw cleanup/update are worker-owned, while host restart/update remain explicitly unavailable pending a distinct worker OS identity and reviewed root boundary. Docker control, database, GitHub, deployment, database backup/restore, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | -| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | +| Phase | Status | Current evidence and remaining gate | +| ----------------------------------- | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | +| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | +| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | +| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | +| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | +| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Overview Service Actions partially replace consumed legacy exec flows with four fixed intents; OpenClaw cleanup/update are worker-owned, host restart/update remain explicitly unavailable pending a distinct worker OS identity, and `POST /api/exec/start` stays planned until system cleanup is decomposed across Docker, host/package, and log-maintenance authorities. Docker control, database, GitHub, deployment, database backup/restore, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | +| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | ### 2026-08-03 — Phase 0 started @@ -1553,7 +1553,7 @@ full-browser parity, production rehearsal, cutover, and legacy deletion remain o Gateway smoke/restart evidence and the remaining Phase 5 domains still kept their aggregate exit gates open. -### 2026-08-12 — Purpose-built Service Actions close consumed exec parity +### 2026-08-12 — Purpose-built Service Actions narrow consumed exec authority - The Overview now exposes exactly four fixed Service Actions through session-only `serviceActions.getStatus` and recent-MFA `serviceActions.request`: OpenClaw cleanup, OpenClaw @@ -1576,10 +1576,12 @@ full-browser parity, production rehearsal, cutover, and legacy deletion remain o - The interactive PTY already owns shell `cd`, completion, and bounded termination. Legacy long-running exec consumers map to either that PTY or the purpose-built durable Service Actions queue. The unused synchronous `POST /api/exec` route is a reviewed removal with no current - browser or scoped automation consumer; no generic shell/command replacement was added. The broad - legacy `system_cleanup` behavior is not restored because it mixed package, journal, and Docker - deletion across separately owned domains. -- The living inventory is now **113 implemented, 41 planned, and three reviewed removals** out of + browser or scoped automation consumer; no generic shell/command replacement was added. + `POST /api/exec/start` remains planned because the broad legacy `system_cleanup` consumer is only + partially replaced: Docker prune belongs to the Docker slice, apt cleanup to host/package + maintenance, and journald vacuum to log maintenance. Keeping the row open preserves all three + operator capabilities without restoring their unsafe shared shell boundary. +- The living inventory is now **112 implemented, 42 planned, and three reviewed removals** out of 157 legacy endpoints. Browser routes remain **12 implemented and four planned**. This advances - Phase 5 without claiming host-operation enablement, Docker/database/delivery parity, or the + Phase 5 without claiming complete exec, host-operation, Docker/database/delivery parity, or the aggregate Phase 5 exit gate. diff --git a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md index 177e463f8..9601db765 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md +++ b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md @@ -440,6 +440,11 @@ OpenClaw cleanup and update do not use this deferred host authority: their exact Gateway operations are already implemented and remain available only when a fresh exact-release worker advertises them. +Those fixed operations do not close the legacy `POST /api/exec/start` row. Its `system_cleanup` +consumer remains planned as Docker prune in the Docker slice, apt cleanup in host/package +maintenance, and journald vacuum in log maintenance. Delivery must preserve each capability behind +its own reviewed authority rather than recreate the old shared shell boundary. + The web process also derives the fixed `/media` descriptor boundary from that same reviewed root. It exposes no configurable media directory, recursive listing, or browser-supplied path route. Local-history transcript carriers become opaque session/message-bound diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql index 59ea2c7b8..88831f5e3 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql +++ b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql @@ -998,7 +998,7 @@ CREATE TABLE `job_runs` ( CONSTRAINT "job_runs_result_json_check" CHECK("result_json" IS NULL OR (length(CAST("result_json" AS BLOB)) <= 65536 AND CASE WHEN json_valid("result_json") THEN json_type("result_json") = 'object' ELSE 0 END)), CONSTRAINT "job_runs_retry_safe_check" CHECK("retry_safe" IN (0, 1)), CONSTRAINT "job_runs_schedule_check" CHECK(("trigger_type" = 'schedule' AND "scheduled_job_id" IS NOT NULL AND "scheduled_job_version" BETWEEN 1 AND 9007199254740991 AND "scheduled_for_at" IS NOT NULL AND "scheduled_for_at" BETWEEN 0 AND 8640000000000000 AND "scheduled_for_at" <= "queued_at") OR ("trigger_type" = 'manual' AND ((("scheduled_job_id" IS NOT NULL AND "scheduled_job_version" BETWEEN 1 AND 9007199254740991) OR ("scheduled_job_id" IS NULL AND "scheduled_job_version" IS NULL)) AND "scheduled_for_at" IS NULL)) OR ("trigger_type" IN ('startup', 'system') AND "scheduled_job_id" IS NULL AND "scheduled_job_version" IS NULL AND "scheduled_for_at" IS NULL)), - CONSTRAINT "job_runs_state_check" CHECK("state" IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND (("state" = 'queued' AND "finished_at" IS NULL AND "result_json" IS NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" = 'running' AND "attempt_count" > 0 AND "finished_at" IS NULL AND "result_json" IS NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" = 'succeeded' AND "attempt_count" > 0 AND "finished_at" IS NOT NULL AND "result_json" IS NOT NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" IN ('failed', 'timed-out') AND "attempt_count" > 0 AND "finished_at" IS NOT NULL AND "result_json" IS NULL AND "terminal_code" IS NOT NULL AND "terminal_message" IS NOT NULL) OR ("state" = 'cancelled' AND "finished_at" IS NOT NULL AND "result_json" IS NULL AND "terminal_code" IS NOT NULL AND "terminal_message" IS NOT NULL))), + CONSTRAINT "job_runs_state_check" CHECK("state" IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND (("state" = 'queued' AND "finished_at" IS NULL AND "result_json" IS NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" = 'running' AND "attempt_count" > 0 AND "finished_at" IS NULL AND "result_json" IS NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" = 'succeeded' AND "attempt_count" > 0 AND "finished_at" IS NOT NULL AND "result_json" IS NOT NULL AND "terminal_code" IS NULL AND "terminal_message" IS NULL) OR ("state" IN ('failed', 'timed-out') AND ("attempt_count" > 0 OR ("state" = 'failed' AND "attempt_count" = 0 AND "cancellation_policy" = 'never' AND "trigger_type" = 'schedule' AND "terminal_code" = 'action-unavailable' AND "terminal_message" = 'The scheduled action is no longer available')) AND "finished_at" IS NOT NULL AND "result_json" IS NULL AND "terminal_code" IS NOT NULL AND "terminal_message" IS NOT NULL) OR ("state" = 'cancelled' AND "finished_at" IS NOT NULL AND "result_json" IS NULL AND "terminal_code" IS NOT NULL AND "terminal_message" IS NOT NULL))), CONSTRAINT "job_runs_state_version_check" CHECK("state_version" BETWEEN 1 AND 9007199254740991), CONSTRAINT "job_runs_terminal_code_check" CHECK(("terminal_code" IS NULL OR (length("terminal_code") BETWEEN 1 AND 128 AND instr("terminal_code", char(0)) = 0 AND "terminal_code" = lower("terminal_code") AND substr("terminal_code", 1, 1) GLOB '[a-z0-9]' AND "terminal_code" NOT GLOB '*[^a-z0-9._/-]*'))), CONSTRAINT "job_runs_terminal_message_check" CHECK(("terminal_message" IS NULL OR (length("terminal_message") BETWEEN 1 AND 2000 AND instr("terminal_message", char(0)) = 0 AND length(trim("terminal_message", char(9, 10, 11, 12, 13, 32, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8232, 8233, 8239, 8287, 12288, 65279))) > 0 AND "terminal_message" NOT GLOB ('*[' || char(1) || '-' || char(31) || char(127) || '-' || char(159) || char(173) || char(1536) || '-' || char(1541) || char(1564) || char(1757) || char(1807) || char(2192) || '-' || char(2193) || char(2274) || char(6158) || char(8203) || '-' || char(8207) || char(8232) || '-' || char(8238) || char(8288) || '-' || char(8292) || char(8294) || '-' || char(8303) || char(65279) || char(65529) || '-' || char(65531) || char(69821) || char(69837) || char(78896) || '-' || char(78911) || char(113824) || '-' || char(113827) || char(119155) || '-' || char(119162) || char(917505) || char(917536) || '-' || char(917631) || ']*') AND length(CAST("terminal_message" AS BLOB)) <= 8000))), @@ -1977,12 +1977,28 @@ WHEN ( AND NEW.attempt_count <> OLD.attempt_count + 1 ) OR ( - NOT (OLD.state = 'queued' AND NEW.state = 'running') + NOT ( + OLD.state = 'queued' + AND NEW.state = 'running' + ) AND NEW.attempt_count <> OLD.attempt_count ) OR ( OLD.state = 'queued' AND NEW.state NOT IN ('queued', 'running', 'cancelled') + AND NOT ( + NEW.state = 'failed' + AND OLD.cancellation_policy = 'never' + AND OLD.trigger_type = 'schedule' + AND NEW.terminal_code = 'action-unavailable' + AND NEW.terminal_message = 'The scheduled action is no longer available' + AND EXISTS ( + SELECT 1 + FROM scheduled_jobs AS schedule + WHERE schedule.id = OLD.scheduled_job_id + AND schedule.enabled = 0 + ) + ) ) OR ( OLD.state = 'running' @@ -2127,6 +2143,17 @@ WHEN NOT EXISTS ( AND ( NEW.kind IN ('cancel-requested', 'cancelled', 'queued') OR NEW.attempt > 0 + OR ( + NEW.kind = 'failed' + AND NEW.attempt = 0 + AND NEW.worker_instance_id IS NULL + AND run.state = 'failed' + AND run.attempt_count = 0 + AND run.cancellation_policy = 'never' + AND run.trigger_type = 'schedule' + AND run.terminal_code = 'action-unavailable' + AND NEW.message = 'The scheduled action is no longer available' + ) ) AND ( NEW.kind <> 'queued' diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json index f8e324254..1812b319e 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json +++ b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json @@ -7731,7 +7731,7 @@ "table": "job_runs" }, { - "value": "\"state\" IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND ((\"state\" = 'queued' AND \"finished_at\" IS NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" = 'running' AND \"attempt_count\" > 0 AND \"finished_at\" IS NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" = 'succeeded' AND \"attempt_count\" > 0 AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NOT NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" IN ('failed', 'timed-out') AND \"attempt_count\" > 0 AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NOT NULL AND \"terminal_message\" IS NOT NULL) OR (\"state\" = 'cancelled' AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NOT NULL AND \"terminal_message\" IS NOT NULL))", + "value": "\"state\" IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND ((\"state\" = 'queued' AND \"finished_at\" IS NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" = 'running' AND \"attempt_count\" > 0 AND \"finished_at\" IS NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" = 'succeeded' AND \"attempt_count\" > 0 AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NOT NULL AND \"terminal_code\" IS NULL AND \"terminal_message\" IS NULL) OR (\"state\" IN ('failed', 'timed-out') AND (\"attempt_count\" > 0 OR (\"state\" = 'failed' AND \"attempt_count\" = 0 AND \"cancellation_policy\" = 'never' AND \"trigger_type\" = 'schedule' AND \"terminal_code\" = 'action-unavailable' AND \"terminal_message\" = 'The scheduled action is no longer available')) AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NOT NULL AND \"terminal_message\" IS NOT NULL) OR (\"state\" = 'cancelled' AND \"finished_at\" IS NOT NULL AND \"result_json\" IS NULL AND \"terminal_code\" IS NOT NULL AND \"terminal_message\" IS NOT NULL))", "name": "job_runs_state_check", "entityType": "checks", "table": "job_runs" diff --git a/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts b/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts index 7df3f9842..64e74a696 100644 --- a/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts +++ b/greenfield/scripts/sourceBoundaries/sourceTopologyPolicy.ts @@ -52,6 +52,7 @@ const reviewedApplicationServerTargets: ReadonlyMap< [ "src/app/worker.ts", new Set([ + "src/server/domains/jobs/actionExecutors.ts", "src/server/domains/jobs/workerRuntime.ts", "src/server/domains/moltbook/provider.ts", "src/server/platform/configuration/workerConfiguration.ts", diff --git a/greenfield/src/app/dashboardServer.ts b/greenfield/src/app/dashboardServer.ts index b497e2f03..d5226eafd 100644 --- a/greenfield/src/app/dashboardServer.ts +++ b/greenfield/src/app/dashboardServer.ts @@ -119,8 +119,10 @@ import { createRequestAuthenticator } from "../server/domains/security/requestAu import { createRequestAuthenticationRepository } from "../server/domains/security/requestAuthenticationRepository.ts"; import { createSecurityAuditLifecycleService } from "../server/domains/security/securityAuditLifecycle.ts"; import { createSecurityAuditLifecycleRepository } from "../server/domains/security/securityAuditLifecycleRepository.ts"; -import { createSqliteServiceActionAuditWriter } from "../server/domains/serviceActions/operationAudit.ts"; -import { createServiceActionsService } from "../server/domains/serviceActions/service.ts"; +import { + createServiceActionsService, + createSqliteServiceActionAuditWriter, +} from "../server/domains/serviceActions/service.ts"; import { createSqliteServiceActionStatusReader } from "../server/domains/serviceActions/statusReader.ts"; import { createSystemHealthDiagnosticsService } from "../server/domains/system/healthDiagnosticsService.ts"; import { createTaskRepository } from "../server/domains/tasks/repository.ts"; diff --git a/greenfield/src/app/worker.ts b/greenfield/src/app/worker.ts index b7e24cf37..bb7e5e9a4 100644 --- a/greenfield/src/app/worker.ts +++ b/greenfield/src/app/worker.ts @@ -1,6 +1,7 @@ import { realpath } from "node:fs/promises"; import path from "node:path"; +import type { FixedHostOperationsExecutionPort } from "../server/domains/jobs/actionExecutors.ts"; import { createDashboardWorkerRuntime, createSystemJobWorkerSideEffects, @@ -39,7 +40,6 @@ import { createProcessTerminationController, type ProcessTerminationController, } from "../server/platform/runtime/processSignals.ts"; -import type { FixedHostOperationsExecutionPort } from "../shared/hostOperations.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../shared/openClawGatewayLifecycle.ts"; import type { OpenClawServiceActionsExecutionPort } from "../shared/openClawServiceActions.ts"; import { diff --git a/greenfield/src/browser/overview/OverviewRoute.test.tsx b/greenfield/src/browser/overview/OverviewRoute.test.tsx index fc84a6f24..88b82b167 100644 --- a/greenfield/src/browser/overview/OverviewRoute.test.tsx +++ b/greenfield/src/browser/overview/OverviewRoute.test.tsx @@ -484,6 +484,10 @@ describe("Dashboard operational overview foundation", () => { const transport = new OverviewTransport(); const { user } = renderOverview(transport); + expect( + await screen.findByRole("heading", { level: 2, name: "Service actions" }) + ).toBeTruthy(); + expect( await screen.findByRole("heading", { level: 1, name: "Mira Dashboard" }) ).toBeTruthy(); @@ -565,9 +569,6 @@ describe("Dashboard operational overview foundation", () => { for (const call of jobSummaryCalls) { expect(call).toEqual({ input: { limit: 1 }, path: "jobs.listRuns" }); } - expect( - await screen.findByRole("heading", { level: 2, name: "Service actions" }) - ).toBeTruthy(); expect( transport.queryCalls.filter(({ path }) => path === "serviceActions.getStatus") ).toEqual([{ input: {}, path: "serviceActions.getStatus" }]); diff --git a/greenfield/src/contracts/jobModel.ts b/greenfield/src/contracts/jobModel.ts index 7e947a10d..42ed641c9 100644 --- a/greenfield/src/contracts/jobModel.ts +++ b/greenfield/src/contracts/jobModel.ts @@ -78,6 +78,10 @@ export const jobResourceKeysMaximumBytes = 4 * 1024; export const jobRunResultMaximumBytes = 64 * 1024; export const jobRunTerminalCodeMaximumLength = 128; export const jobRunTerminalMessageMaximumLength = 2000; +/** Exact terminal identity for a never-started run whose code-owned schedule retired. */ +export const retiredScheduledActionTerminalCode = "action-unavailable"; +export const retiredScheduledActionTerminalMessage = + "The scheduled action is no longer available"; export const jobRunAttemptMaximum = 10; export const jobRunEventMaximum = 1000; /** Payload slots left after reserving every worst-case structural lifecycle event. */ @@ -489,6 +493,32 @@ const jobRunSummaryObjectSchema = v.strictObject({ export type JobRunSummary = v.InferOutput; +interface UnstartedRetiredScheduleFailure { + readonly attemptCount: number; + readonly cancellationPolicy: JobCancellationPolicy; + readonly state: JobRunState; + readonly terminalCode?: string | null; + readonly terminalMessage?: string | null; + readonly triggerType: JobTriggerType; +} + +/** + * @param run Stored or public run projection to inspect. + * @returns Whether it is the one canonical terminal state allowed before attempt one. + */ +export function isUnstartedRetiredScheduleFailure( + run: UnstartedRetiredScheduleFailure +): boolean { + return ( + run.state === "failed" && + run.attemptCount === 0 && + run.cancellationPolicy === "never" && + run.triggerType === "schedule" && + run.terminalCode === retiredScheduledActionTerminalCode && + run.terminalMessage === retiredScheduledActionTerminalMessage + ); +} + /** * @param run Public run projection to inspect. * @returns Whether it preserves lifecycle and timestamp invariants. @@ -532,7 +562,8 @@ export function jobRunSummaryIsConsistent(run: JobRunSummary): boolean { } if ( ["failed", "running", "succeeded", "timed-out"].includes(run.state) && - run.attemptCount === 0 + run.attemptCount === 0 && + !isUnstartedRetiredScheduleFailure(run) ) { return false; } diff --git a/greenfield/src/contracts/serviceActions.test.ts b/greenfield/src/contracts/serviceActions.test.ts index 054e2bc00..c2f3f1f59 100644 --- a/greenfield/src/contracts/serviceActions.test.ts +++ b/greenfield/src/contracts/serviceActions.test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test"; import * as v from "valibot"; import { procedureContracts } from "./contractRegistry.ts"; +import { jobRunSummarySchema } from "./jobModel.ts"; import { getServiceActionsStatusResultSchema, requestServiceActionInputSchema, @@ -38,6 +39,39 @@ function queuedRun(actionKey: string) { } describe("service action contracts", () => { + test("accepts only the exact zero-attempt scheduled-action retirement", () => { + const retired = { + ...queuedRun("cache.refresh.retired"), + eventCount: 2, + finishedAtMs: 2000, + scheduledForAtMs: 1000, + scheduledJobId: "cache.refresh.retired", + scheduledJobVersion: 1, + state: "failed" as const, + stateVersion: 2, + terminalCode: "action-unavailable", + terminalMessage: "The scheduled action is no longer available", + triggerType: "schedule" as const, + updatedAtMs: 2000, + }; + + expect(v.parse(jobRunSummarySchema, retired)).toEqual(retired); + for (const invalid of [ + { ...retired, cancellationPolicy: "queued-only" }, + { ...retired, terminalCode: "failed/provider" }, + { ...retired, terminalMessage: "Scheduled action unavailable" }, + { + ...retired, + scheduledForAtMs: undefined, + scheduledJobId: undefined, + scheduledJobVersion: undefined, + triggerType: "manual", + }, + ]) { + expect(v.safeParse(jobRunSummarySchema, invalid).success).toBeFalse(); + } + }); + test("registers one session-only status query and one recent-MFA mutation", () => { expect( serviceActionProcedureContracts.map( diff --git a/greenfield/src/server/database/migrations/jobsSchema.test.ts b/greenfield/src/server/database/migrations/jobsSchema.test.ts index b48bf2f76..ad583c8e0 100644 --- a/greenfield/src/server/database/migrations/jobsSchema.test.ts +++ b/greenfield/src/server/database/migrations/jobsSchema.test.ts @@ -1034,6 +1034,94 @@ describe("jobs baseline schema", () => { ) ).toThrow("job_runs lifecycle transition is invalid"); + const retiredScheduleId = "system.worker-smoke-retired-never"; + const retiredRunId = uuid(29); + insertSchedule(database, { + cancellationPolicy: "never", + id: retiredScheduleId, + }); + insertQueuedRun(database, { + cancellationPolicy: "never", + id: retiredRunId, + idempotencyKey: idempotencyKey(29), + scheduledForAt: 1000, + scheduledJobId: retiredScheduleId, + scheduledJobVersion: 1, + triggerType: "schedule", + }); + insertEvent(database, { + jobRunId: retiredRunId, + kind: "queued", + sequence: 1, + }); + expect(() => + database.sqlite.run( + `UPDATE job_runs + SET finished_at = 1500, state = 'failed', state_version = 2, + terminal_code = 'action-unavailable', + terminal_message = 'The scheduled action is no longer available', + updated_at = 1500 + WHERE id = ?`, + [retiredRunId] + ) + ).toThrow("job_runs lifecycle transition is invalid"); + database.sqlite.run( + `UPDATE scheduled_jobs + SET enabled = 0, next_run_at = NULL, updated_at = 1500, version = 2 + WHERE id = ?`, + [retiredScheduleId] + ); + expect(() => + database.sqlite.run( + `UPDATE job_runs + SET finished_at = 1500, state = 'failed', state_version = 2, + terminal_code = 'action-unavailable', + terminal_message = 'A different bounded failure message', + updated_at = 1500 + WHERE id = ?`, + [retiredRunId] + ) + ).toThrow(); + database.sqlite.run( + `UPDATE job_runs + SET finished_at = 1500, state = 'failed', state_version = 2, + terminal_code = 'action-unavailable', + terminal_message = 'The scheduled action is no longer available', + updated_at = 1500 + WHERE id = ?`, + [retiredRunId] + ); + insertEvent(database, { + attempt: 0, + jobRunId: retiredRunId, + kind: "failed", + message: "The scheduled action is no longer available", + occurredAt: 1500, + sequence: 2, + }); + expect( + database.sqlite + .query< + { + attempt_count: number; + first_started_at: number | null; + last_attempt_started_at: number | null; + state: string; + }, + [string] + >( + `SELECT attempt_count, first_started_at, + last_attempt_started_at, state + FROM job_runs WHERE id = ?` + ) + .get(retiredRunId) + ).toEqual({ + attempt_count: 0, + first_started_at: null, + last_attempt_started_at: null, + state: "failed", + }); + const queuedOnlyRunId = uuid(27); insertQueuedRun(database, { cancellationPolicy: "queued-only", diff --git a/greenfield/src/server/database/schema/jobChecks.ts b/greenfield/src/server/database/schema/jobChecks.ts index 5458e28de..e4ab3f4e1 100644 --- a/greenfield/src/server/database/schema/jobChecks.ts +++ b/greenfield/src/server/database/schema/jobChecks.ts @@ -1,5 +1,9 @@ import { sql, type SQLWrapper } from "drizzle-orm"; +import { + retiredScheduledActionTerminalCode, + retiredScheduledActionTerminalMessage, +} from "../../../contracts/jobModel.ts"; import { boundedControlSafeTextCheck, boundedNonBlankTextCheck, @@ -7,6 +11,33 @@ import { uuidV7TextCheck, } from "./checks.ts"; +/** Maximum executable action identities advertised by one worker process. */ +export const workerActionKeyMaximum = 32; +/** Maximum canonical UTF-8 JSON representation retained in one worker row. */ +export const workerActionKeysMaximumBytes = 4 * 1024; + +const retiredScheduledActionTerminalCodeSql = sql.raw( + `'${retiredScheduledActionTerminalCode}'` +); +const retiredScheduledActionTerminalMessageSql = sql.raw( + `'${retiredScheduledActionTerminalMessage}'` +); + +/** + * Exact SQL form of the only failed lifecycle admitted before attempt one. + * @returns Canonical predicate shared by durable job-row constraints. + */ +export function unstartedRetiredScheduleFailureCheck(columns: { + readonly attemptCount: SQLWrapper; + readonly cancellationPolicy: SQLWrapper; + readonly state: SQLWrapper; + readonly terminalCode: SQLWrapper; + readonly terminalMessage: SQLWrapper; + readonly triggerType: SQLWrapper; +}) { + return sql`${columns.state} = 'failed' AND ${columns.attemptCount} = 0 AND ${columns.cancellationPolicy} = 'never' AND ${columns.triggerType} = 'schedule' AND ${columns.terminalCode} = ${retiredScheduledActionTerminalCodeSql} AND ${columns.terminalMessage} = ${retiredScheduledActionTerminalMessageSql}`; +} + /** * Canonical lowercase identifier used by schedules, actions, and resource leases. * @returns SQL predicate for the bounded canonical key. diff --git a/greenfield/src/server/database/schema/jobRuns.ts b/greenfield/src/server/database/schema/jobRuns.ts index 6d1dddeac..70f5f8ff3 100644 --- a/greenfield/src/server/database/schema/jobRuns.ts +++ b/greenfield/src/server/database/schema/jobRuns.ts @@ -31,6 +31,7 @@ import { jobActorCheck, optionalJobMessageCheck, optionalJobTerminalCodeCheck, + unstartedRetiredScheduleFailureCheck, } from "./jobChecks.ts"; import { scheduledJobs } from "./scheduledJobs.ts"; import { workerInstances } from "./workerInstances.ts"; @@ -170,7 +171,7 @@ export const jobRuns = sqliteTable( ), check( "job_runs_state_check", - sql`${table.state} IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND ((${table.state} = 'queued' AND ${table.finishedAt} IS NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} = 'running' AND ${table.attemptCount} > 0 AND ${table.finishedAt} IS NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} = 'succeeded' AND ${table.attemptCount} > 0 AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NOT NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} IN ('failed', 'timed-out') AND ${table.attemptCount} > 0 AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NOT NULL AND ${table.terminalMessage} IS NOT NULL) OR (${table.state} = 'cancelled' AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NOT NULL AND ${table.terminalMessage} IS NOT NULL))` + sql`${table.state} IN ('cancelled', 'failed', 'queued', 'running', 'succeeded', 'timed-out') AND ((${table.state} = 'queued' AND ${table.finishedAt} IS NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} = 'running' AND ${table.attemptCount} > 0 AND ${table.finishedAt} IS NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} = 'succeeded' AND ${table.attemptCount} > 0 AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NOT NULL AND ${table.terminalCode} IS NULL AND ${table.terminalMessage} IS NULL) OR (${table.state} IN ('failed', 'timed-out') AND (${table.attemptCount} > 0 OR (${unstartedRetiredScheduleFailureCheck(table)})) AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NOT NULL AND ${table.terminalMessage} IS NOT NULL) OR (${table.state} = 'cancelled' AND ${table.finishedAt} IS NOT NULL AND ${table.resultJson} IS NULL AND ${table.terminalCode} IS NOT NULL AND ${table.terminalMessage} IS NOT NULL))` ), check( "job_runs_state_version_check", diff --git a/greenfield/src/server/database/schema/workerInstances.ts b/greenfield/src/server/database/schema/workerInstances.ts index 3cbca61dd..79d1cbe02 100644 --- a/greenfield/src/server/database/schema/workerInstances.ts +++ b/greenfield/src/server/database/schema/workerInstances.ts @@ -1,13 +1,12 @@ import { sql } from "drizzle-orm"; import { check, index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; -import { workerActionKeysMaximumBytes } from "../workerActionKeyPolicy.ts"; import { lowercaseHexTextCheck, timestampMillisecondsCheck, uuidV7TextCheck, } from "./checks.ts"; -import { boundedJsonArrayCheck } from "./jobChecks.ts"; +import { boundedJsonArrayCheck, workerActionKeysMaximumBytes } from "./jobChecks.ts"; /** Durable worker registration and heartbeat state shared across rolling releases. */ export const workerInstances = sqliteTable( diff --git a/greenfield/src/server/database/validation/jobRunEvents.ts b/greenfield/src/server/database/validation/jobRunEvents.ts index b660028d1..9e9f12b72 100644 --- a/greenfield/src/server/database/validation/jobRunEvents.ts +++ b/greenfield/src/server/database/validation/jobRunEvents.ts @@ -9,6 +9,7 @@ import { jobRunEventProgressMaximumBytes, jobRunEventProgressSchema, jobRunEventSequenceSchema, + retiredScheduledActionTerminalMessage, } from "../../../contracts/jobModel.ts"; import { utf8ByteLength } from "../../../shared/encoding.ts"; import { parseJsonText } from "../../../shared/json.ts"; @@ -69,10 +70,15 @@ function eventAttemptIsConsistent(event: StoredJobRunEvent): boolean { if (event.kind === "queued") { return event.attempt === 0 && event.workerInstanceId == null; } + if (event.kind === "failed" && event.attempt === 0) { + return ( + event.workerInstanceId == null && + event.message === retiredScheduledActionTerminalMessage + ); + } if ( [ "claimed", - "failed", "lease-expired", "output-truncated", "progress", diff --git a/greenfield/src/server/database/validation/jobRuns.ts b/greenfield/src/server/database/validation/jobRuns.ts index eda8d9bc4..14915bccf 100644 --- a/greenfield/src/server/database/validation/jobRuns.ts +++ b/greenfield/src/server/database/validation/jobRuns.ts @@ -8,6 +8,7 @@ import { jobCancellationPolicySchema, jobDisplayNameSchema, jobIdempotencyKeySchema, + isUnstartedRetiredScheduleFailure, jobPayloadMaximumBytes, jobPayloadSchema, jobPrioritySchema, @@ -151,7 +152,8 @@ function attemptsAreConsistent(run: StoredJobRun): boolean { } return ( !["failed", "running", "succeeded", "timed-out"].includes(run.state) || - run.attemptCount > 0 + run.attemptCount > 0 || + isUnstartedRetiredScheduleFailure(run) ); } diff --git a/greenfield/src/server/database/validation/rowSchemas.test.ts b/greenfield/src/server/database/validation/rowSchemas.test.ts index b1c6e3526..a878ddaf4 100644 --- a/greenfield/src/server/database/validation/rowSchemas.test.ts +++ b/greenfield/src/server/database/validation/rowSchemas.test.ts @@ -680,6 +680,35 @@ describe("Drizzle-generated Valibot row schemas", () => { workerInstanceId: null, }) ).toThrow(); + const retiredUnstartedRun = { + ...validJobRunRow, + cancellationPolicy: "never" as const, + finishedAt: jobUpdatedAt, + scheduledForAt: jobCreatedAt, + state: "failed" as const, + terminalCode: "action-unavailable", + terminalMessage: "The scheduled action is no longer available", + triggerType: "schedule" as const, + }; + expect(v.parse(jobRunSelectSchema, retiredUnstartedRun)).toBeDefined(); + expect(() => + v.parse(jobRunSelectSchema, { + ...retiredUnstartedRun, + terminalCode: "action-failed", + }) + ).toThrow(); + expect(() => + v.parse(jobRunSelectSchema, { + ...retiredUnstartedRun, + terminalMessage: "A different bounded failure message", + }) + ).toThrow(); + expect(() => + v.parse(jobRunSelectSchema, { + ...retiredUnstartedRun, + triggerType: "manual", + }) + ).toThrow(); expect(() => v.parse(workerInstanceSelectSchema, { actionKeysJson: "[]", diff --git a/greenfield/src/server/database/validation/workerActionKeys.ts b/greenfield/src/server/database/validation/workerActionKeys.ts deleted file mode 100644 index dac92ce40..000000000 --- a/greenfield/src/server/database/validation/workerActionKeys.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as v from "valibot"; - -import { jobActionKeySchema } from "../../../contracts/jobModel.ts"; -import { utf8ByteLength } from "../../../shared/encoding.ts"; -import { parseJsonText } from "../../../shared/json.ts"; -import { compareStrings, hasUniqueArrayItems } from "../../../shared/validation.ts"; -import { - workerActionKeyMaximum, - workerActionKeysMaximumBytes, -} from "../workerActionKeyPolicy.ts"; - -function actionKeysAreCanonical(keys: string[]): boolean { - return ( - hasUniqueArrayItems(keys) && - keys.every((key, index) => { - const previous = keys[index - 1]; - return previous === undefined || compareStrings(previous, key) < 0; - }) && - utf8ByteLength(JSON.stringify(keys)) <= workerActionKeysMaximumBytes - ); -} - -/** Strict bounded canonical worker action inventory. */ -export const workerActionKeysSchema = v.pipe( - v.array(jobActionKeySchema, "Worker action keys are invalid"), - v.maxLength(workerActionKeyMaximum, "Worker action keys are outside their budget"), - v.check(actionKeysAreCanonical, "Worker action keys are not canonical") -); - -/** - * Canonicalizes one validated release-owned executable-action inventory. - * @param actionKeys Candidate action identities from worker action definitions. - * @returns Frozen sorted unique keys safe to persist as worker identity. - */ -export function canonicalWorkerActionKeys( - actionKeys: readonly string[] -): readonly string[] { - const sorted = actionKeys - .map((actionKey) => v.parse(jobActionKeySchema, actionKey)) - .toSorted(compareStrings); - return Object.freeze([...v.parse(workerActionKeysSchema, sorted)]); -} - -/** - * Serializes one canonical inventory without whitespace or unbounded fields. - * @param actionKeys Candidate action identities. - * @returns Canonical JSON text accepted by the worker persistence boundary. - */ -export function serializeWorkerActionKeys(actionKeys: readonly string[]): string { - return JSON.stringify(canonicalWorkerActionKeys(actionKeys)); -} - -/** - * Parses the immutable action inventory stored on one worker row. - * @param value Stored JSON text. - * @returns Frozen validated canonical action identities. - */ -export function parseWorkerActionKeysJson(value: string): readonly string[] { - const parsed = v.parse(workerActionKeysSchema, parseJsonText(value)); - return Object.freeze([...parsed]); -} diff --git a/greenfield/src/server/database/validation/workerInstances.ts b/greenfield/src/server/database/validation/workerInstances.ts index 8b9f32c49..4e952af34 100644 --- a/greenfield/src/server/database/validation/workerInstances.ts +++ b/greenfield/src/server/database/validation/workerInstances.ts @@ -2,16 +2,75 @@ import { createInsertSchema, createSelectSchema } from "drizzle-orm/valibot"; import * as v from "valibot"; import { + jobActionKeySchema, jobWorkerCapacityMaximum, jobWorkerStateSchema, } from "../../../contracts/jobModel.ts"; +import { utf8ByteLength } from "../../../shared/encoding.ts"; +import { parseJsonText } from "../../../shared/json.ts"; import { + compareStrings, fullCommitShaSchema, + hasUniqueArrayItems, positiveSafeIntegerSchema, } from "../../../shared/validation.ts"; +import { + workerActionKeyMaximum, + workerActionKeysMaximumBytes, +} from "../schema/jobChecks.ts"; import { workerInstances } from "../schema/workerInstances.ts"; import { nonnegativeDateSchema, uuidV7TextSchema } from "./scalars.ts"; -import { parseWorkerActionKeysJson } from "./workerActionKeys.ts"; + +function actionKeysAreCanonical(keys: string[]): boolean { + return ( + hasUniqueArrayItems(keys) && + keys.every((key, index) => { + const previous = keys[index - 1]; + return previous === undefined || compareStrings(previous, key) < 0; + }) && + utf8ByteLength(JSON.stringify(keys)) <= workerActionKeysMaximumBytes + ); +} + +/** Strict bounded canonical worker action inventory. */ +export const workerActionKeysSchema = v.pipe( + v.array(jobActionKeySchema, "Worker action keys are invalid"), + v.maxLength(workerActionKeyMaximum, "Worker action keys are outside their budget"), + v.check(actionKeysAreCanonical, "Worker action keys are not canonical") +); + +/** + * Canonicalizes one validated release-owned executable-action inventory. + * @param actionKeys Candidate action identities from worker action definitions. + * @returns Frozen sorted unique keys safe to persist as worker identity. + */ +export function canonicalWorkerActionKeys( + actionKeys: readonly string[] +): readonly string[] { + const sorted = actionKeys + .map((actionKey) => v.parse(jobActionKeySchema, actionKey)) + .toSorted(compareStrings); + return Object.freeze([...v.parse(workerActionKeysSchema, sorted)]); +} + +/** + * Serializes one canonical inventory without whitespace or unbounded fields. + * @param actionKeys Candidate action identities. + * @returns Canonical JSON text accepted by the worker persistence boundary. + */ +export function serializeWorkerActionKeys(actionKeys: readonly string[]): string { + return JSON.stringify(canonicalWorkerActionKeys(actionKeys)); +} + +/** + * Parses the immutable action inventory stored on one worker row. + * @param value Stored JSON text. + * @returns Frozen validated canonical action identities. + */ +export function parseWorkerActionKeysJson(value: string): readonly string[] { + const parsed = v.parse(workerActionKeysSchema, parseJsonText(value)); + return Object.freeze([...parsed]); +} const workerCapacitySchema = v.pipe( positiveSafeIntegerSchema("Stored worker capacity is invalid"), diff --git a/greenfield/src/server/database/workerActionKeyPolicy.ts b/greenfield/src/server/database/workerActionKeyPolicy.ts deleted file mode 100644 index f369620f0..000000000 --- a/greenfield/src/server/database/workerActionKeyPolicy.ts +++ /dev/null @@ -1,4 +0,0 @@ -/** Maximum executable action identities advertised by one worker process. */ -export const workerActionKeyMaximum = 32; -/** Maximum canonical UTF-8 JSON representation retained in one worker row. */ -export const workerActionKeysMaximumBytes = 4 * 1024; diff --git a/greenfield/src/server/domains/jobs/actionExecutors.ts b/greenfield/src/server/domains/jobs/actionExecutors.ts index f14ffcdd8..ca1157d84 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.ts @@ -7,7 +7,6 @@ import { type LogMaintenanceExecutionSummary, logMaintenancePolicyIdSchema, } from "../../../contracts/logs.ts"; -import type { FixedHostOperationsExecutionPort } from "../../../shared/hostOperations.ts"; import type { JsonObject } from "../../../shared/json.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; import { @@ -48,6 +47,31 @@ import { workspaceFileWriteJobActionKey, } from "./actionRegistry.ts"; +/** Complete contract-ordered inventory of reviewed privileged host operations. */ +export const hostOperationIds = Object.freeze([ + "system-restart", + "system-update", +] as const); + +/** One exact reviewed privileged host operation. */ +export type HostOperationId = (typeof hostOperationIds)[number]; + +/** Secret-free result returned by one future, separately privileged host adapter. */ +export type FixedHostOperationResult = + | Readonly<{ status: "accepted" }> + | Readonly<{ status: "completed" }>; + +/** Worker-only fixed-operation authority; no command or path crosses this port. */ +export interface FixedHostOperationsExecutionPort { + readonly availableOperations: ( + signal?: AbortSignal + ) => Promise; + readonly request: ( + operationId: HostOperationId, + signal?: AbortSignal + ) => Promise; +} + const emptyPayloadSchema = v.strictObject({}); const systemHostActionPayloadSchema = v.strictObject({ key: v.literal("system.host") }); const moltbookDashboardActionPayloadSchema = v.strictObject({ diff --git a/greenfield/src/server/domains/jobs/coordinator.test.ts b/greenfield/src/server/domains/jobs/coordinator.test.ts index 6e68920df..a2c9e8979 100644 --- a/greenfield/src/server/domains/jobs/coordinator.test.ts +++ b/greenfield/src/server/domains/jobs/coordinator.test.ts @@ -734,7 +734,7 @@ describe("durable job worker coordinator", () => { ]); }); - test("starts after retiring a queued never-cancellable schedule run", async () => { + test("fails a queued never-cancellable run before starting after schedule retirement", async () => { const database = await openFreshMigratedDatabase(); const repository = createJobRepository( database.orm, @@ -808,7 +808,7 @@ describe("durable job worker coordinator", () => { }); expect(repository.findRun(run.id)).toMatchObject({ cancelRequestedAt: null, - eventCount: 3, + eventCount: 2, state: "failed", terminalCode: "action-unavailable", }); diff --git a/greenfield/src/server/domains/jobs/coordinator.ts b/greenfield/src/server/domains/jobs/coordinator.ts index 06b1f51cc..44deb20c1 100644 --- a/greenfield/src/server/domains/jobs/coordinator.ts +++ b/greenfield/src/server/domains/jobs/coordinator.ts @@ -7,10 +7,12 @@ import { jobPayloadSchema, jobRunResultSchema, jobWorkerFreshnessMs, + retiredScheduledActionTerminalCode, + retiredScheduledActionTerminalMessage, } from "../../../contracts/jobModel.ts"; import type { JsonObject } from "../../../shared/json.ts"; import { parseJsonText } from "../../../shared/json.ts"; -import { serializeWorkerActionKeys } from "../../database/validation/workerActionKeys.ts"; +import { serializeWorkerActionKeys } from "../../database/validation/workerInstances.ts"; import { sha256Hex } from "../../shared/crypto.ts"; import { type JobActionDefinition, @@ -1099,6 +1101,16 @@ export function createJobWorkerCoordinator( terminalMessage: "Cancelled because the schedule was retired from the action registry", }, + retiredRunFailure: { + sideEffectsForRun: (run) => + durableRunTransitionSideEffects( + options.sideEffects, + "jobs.run.action-unavailable", + run + ), + terminalCode: retiredScheduledActionTerminalCode, + terminalMessage: retiredScheduledActionTerminalMessage, + }, schedules, sideEffectsForSchedule: (schedule) => options.sideEffects.forSchedule({ diff --git a/greenfield/src/server/domains/jobs/repository.test.ts b/greenfield/src/server/domains/jobs/repository.test.ts index 40ad34086..2768546ba 100644 --- a/greenfield/src/server/domains/jobs/repository.test.ts +++ b/greenfield/src/server/domains/jobs/repository.test.ts @@ -142,6 +142,57 @@ function worker( } describe("durable jobs repository", () => { + test("rechecks authority after write admission and rolls back a rejected enqueue", async () => { + const database = await openFreshMigratedDatabase(); + let releaseAdmission: (() => void) | undefined; + let enteredAdmission: (() => void) | undefined; + const admissionEntered = new Promise((resolve) => { + enteredAdmission = resolve; + }); + const admissionRelease = new Promise((resolve) => { + releaseAdmission = resolve; + }); + const repository = createJobRepository(database.orm, { + async run(operation) { + enteredAdmission?.(); + await admissionRelease; + return operation(() => {}); + }, + }); + const run = queuedRun(8, { + actionKey: "host.system.update", + displayName: "Update host system", + scheduledJobId: null, + scheduledJobVersion: null, + }); + const authorizationFailure = new Error("authorization expired"); + let authorized = true; + + try { + const pending = repository.enqueueManualRun( + { + ...noSideEffects, + queuedEvent: queuedEvent(run), + run, + }, + () => { + if (!authorized) throw authorizationFailure; + } + ); + await admissionEntered; + authorized = false; + releaseAdmission?.(); + + expect(await pending.catch((error: unknown) => error)).toBe( + authorizationFailure + ); + expect(repository.findRun(run.id)).toBeUndefined(); + } finally { + releaseAdmission?.(); + database.sqlite.close(true); + } + }); + test("reconciles code metadata and enforces caller-scoped manual idempotency", async () => { const database = await openFreshMigratedDatabase(); const repository = createJobRepository( @@ -1309,6 +1360,11 @@ describe("durable jobs repository", () => { terminalCode: "cancelled/schedule-retired", terminalMessage: "Cancelled because the schedule was retired", }, + retiredRunFailure: { + sideEffectsForRun: () => noSideEffects, + terminalCode: "action-unavailable", + terminalMessage: "The scheduled action is no longer available", + }, schedules: [], sideEffectsForSchedule: () => { throw new Error("reject retirement schedule side effects"); @@ -1335,6 +1391,7 @@ describe("durable jobs repository", () => { readonly updatedAt: Date; readonly version: number; }> = []; + let failureSideEffectAt: Date | undefined; await repository.reconcileSchedules({ at: new Date(900), retiredRunCancellation: { @@ -1345,6 +1402,14 @@ describe("durable jobs repository", () => { terminalCode: "cancelled/schedule-retired", terminalMessage: "Cancelled because the schedule was retired", }, + retiredRunFailure: { + sideEffectsForRun: (failed) => { + failureSideEffectAt = failed.updatedAt; + return noSideEffects; + }, + terminalCode: "action-unavailable", + terminalMessage: "The scheduled action is no longer available", + }, schedules: [], sideEffectsForSchedule: (retired) => { retiredSchedules.push({ @@ -1371,11 +1436,16 @@ describe("durable jobs repository", () => { version: 2, }); expect(repository.findRun(run.id)).toMatchObject({ + attemptCount: 0, cancelRequestedAt: null, - eventCount: 1, - state: "queued", + eventCount: 2, + firstStartedAt: null, + lastAttemptStartedAt: null, + state: "failed", + terminalCode: "action-unavailable", updatedAt: new Date(100_000), }); + expect(failureSideEffectAt).toEqual(new Date(100_000)); await repository.reconcileSchedules({ at: new Date(130_000), diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index f412c52a7..221ddb436 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -5,7 +5,6 @@ import { count, desc, eq, - exists, gte, gt, inArray, @@ -96,8 +95,6 @@ import { canonicalWorkerActionKeys, parseWorkerActionKeysJson, workerActionKeysSchema, -} from "../../database/validation/workerActionKeys.ts"; -import { workerInstanceInsertSchema, workerInstanceSelectSchema, } from "../../database/validation/workerInstances.ts"; @@ -245,6 +242,11 @@ export interface ReconcileSchedulesInput { readonly terminalCode: string; readonly terminalMessage: string; }; + readonly retiredRunFailure?: { + readonly sideEffectsForRun: (run: JobRunRecord) => JobMutationSideEffects; + readonly terminalCode: string; + readonly terminalMessage: string; + }; readonly schedules: readonly ScheduledJobInsert[]; readonly sideEffectsForSchedule: ( schedule: ScheduledJobRecord @@ -536,7 +538,10 @@ export interface JobRepository extends JobRepositoryReader { beginWorkerDrain(input: WorkerLifecycleMutationInput): Promise; cancelRun(input: CancelRunRepositoryInput): Promise; claimNextRun(input: ClaimNextRunInput): Promise; - enqueueManualRun(input: EnqueueManualRunInput): Promise; + enqueueManualRun( + input: EnqueueManualRunInput, + beforeInsert?: () => void + ): Promise; enqueueNextDueSchedule( input: DueScheduleEnqueueInput ): Promise; @@ -1405,14 +1410,14 @@ class DrizzleJobWriter extends DrizzleJobReader { .filter((schedule) => !registeredScheduleIds.has(schedule.id)); for (const schedule of retiredSchedules) { const queuedScheduleRun = this.#findQueuedScheduleRun(schedule.id); - // Registry retirement disables future scheduling. A queued `never` run keeps - // its immutable execution snapshot and completes through the normal worker - // action-availability path; retirement must not reinterpret it as cancellable. - const cancellableQueuedScheduleRun = + const neverCancellableQueuedRun = queuedScheduleRun?.cancellationPolicy === "never" - ? undefined - : queuedScheduleRun; + ? queuedScheduleRun + : undefined; + const cancellableQueuedScheduleRun = + neverCancellableQueuedRun === undefined ? queuedScheduleRun : undefined; const retiredRunCancellation = input.retiredRunCancellation; + const retiredRunFailure = input.retiredRunFailure; if ( cancellableQueuedScheduleRun !== undefined && retiredRunCancellation === undefined @@ -1421,6 +1426,14 @@ class DrizzleJobWriter extends DrizzleJobReader { "Removed schedule retirement requires queued-run cancellation metadata" ); } + if ( + neverCancellableQueuedRun !== undefined && + retiredRunFailure === undefined + ) { + throw new Error( + "Removed schedule retirement requires immutable-run failure metadata" + ); + } const retired = this.#transaction .update(scheduledJobs) .set({ @@ -1457,12 +1470,26 @@ class DrizzleJobWriter extends DrizzleJobReader { retiredRunCancellation.sideEffectsForRun(cancelled) ); } + if ( + neverCancellableQueuedRun !== undefined && + retiredRunFailure !== undefined + ) { + const failed = this.#failQueuedRun(neverCancellableQueuedRun, { + at: retiredSchedule.updatedAt, + terminalCode: retiredRunFailure.terminalCode, + terminalMessage: retiredRunFailure.terminalMessage, + }); + this.#insertSideEffects(retiredRunFailure.sideEffectsForRun(failed)); + } this.#insertSideEffects(input.sideEffectsForSchedule(retiredSchedule)); } return records; } - public enqueueManualRun(input: EnqueueManualRunInput): EnqueueManualRunResult { + public enqueueManualRun( + input: EnqueueManualRunInput, + beforeInsert?: () => void + ): EnqueueManualRunResult { const run = v.parse(jobRunInsertSchema, input.run); if (input.queuedEvent.jobRunId !== run.id) { throw new Error("Queued event does not belong to the inserted manual run"); @@ -1496,6 +1523,7 @@ class DrizzleJobWriter extends DrizzleJobReader { .get(); if (active !== undefined) return { kind: "active", run: parseRun(active) }; } + beforeInsert?.(); const inserted = this.#transaction.insert(jobRuns).values(run).returning().get(); const record = parseRun(requiredRow(inserted, "manual run insert")); this.#insertSuppliedEvent(input.queuedEvent); @@ -2148,21 +2176,6 @@ class DrizzleJobWriter extends DrizzleJobReader { if (activeCount >= worker.capacity) return { kind: "worker-unavailable" }; const workerActionKeys = parseWorkerActionKeysJson(worker.actionKeysJson); if (workerActionKeys.length === 0) return { kind: "empty" }; - const retiredNeverRun = and( - eq(jobRuns.cancellationPolicy, "never"), - isNotNull(jobRuns.scheduledJobId), - exists( - this.#transaction - .select({ id: scheduledJobs.id }) - .from(scheduledJobs) - .where( - and( - eq(scheduledJobs.id, jobRuns.scheduledJobId), - eq(scheduledJobs.enabled, false) - ) - ) - ) - ); const availableThrough = input.cursor?.availableThrough ?? input.at; const candidates: JobRunRecord[] = []; @@ -2177,10 +2190,7 @@ class DrizzleJobWriter extends DrizzleJobReader { and( eq(jobRuns.state, "queued"), lte(jobRuns.availableAt, availableThrough), - or( - inArray(jobRuns.actionKey, workerActionKeys), - retiredNeverRun - ), + inArray(jobRuns.actionKey, workerActionKeys), range ) ) @@ -2577,6 +2587,38 @@ class DrizzleJobWriter extends DrizzleJobReader { return requiredRow(this.findRun(run.id), "cancelled run refresh"); } + #failQueuedRun(run: JobRunRecord, input: ScheduleQueuedCancellation): JobRunRecord { + const at = maximumDate(run.updatedAt, input.at); + const row = this.#transaction + .update(jobRuns) + .set({ + finishedAt: at, + state: "failed", + stateVersion: run.stateVersion + 1, + terminalCode: input.terminalCode, + terminalMessage: input.terminalMessage, + updatedAt: at, + }) + .where( + and( + eq(jobRuns.id, run.id), + eq(jobRuns.state, "queued"), + eq(jobRuns.stateVersion, run.stateVersion) + ) + ) + .returning() + .get(); + parseRun(requiredRow(row, "queued run failure")); + this.#appendEvent(run.id, { + attempt: run.attemptCount, + kind: "failed", + message: boundedStructuralMessage(input.terminalMessage), + occurredAt: at, + workerInstanceId: null, + }); + return requiredRow(this.findRun(run.id), "failed queued run refresh"); + } + #claimFence(input: ClaimFenceInput): SQL { return and( eq(jobRuns.id, input.runId), @@ -2814,8 +2856,8 @@ export function createJobRepository( write((writer) => writer.cancelRun(input)), claimNextRun: (input: ClaimNextRunInput) => write((writer) => writer.claimNextRun(input)), - enqueueManualRun: (input: EnqueueManualRunInput) => - write((writer) => writer.enqueueManualRun(input)), + enqueueManualRun: (input: EnqueueManualRunInput, beforeInsert?: () => void) => + write((writer) => writer.enqueueManualRun(input, beforeInsert)), enqueueNextDueSchedule: (input: DueScheduleEnqueueInput) => write((writer) => writer.enqueueNextDueSchedule(input)), expireDisableIntents: (input: ExpireDisableIntentsInput) => diff --git a/greenfield/src/server/domains/jobs/service.test.ts b/greenfield/src/server/domains/jobs/service.test.ts index dcba23a99..1d74699ee 100644 --- a/greenfield/src/server/domains/jobs/service.test.ts +++ b/greenfield/src/server/domains/jobs/service.test.ts @@ -824,7 +824,7 @@ describe("durable jobs service", () => { } }); - test("retires a schedule without cancelling its queued never-cancellable run", async () => { + test("fails a queued never-cancellable run whose action leaves the registry", async () => { const fixture = await openAuthenticationTestDatabase(authenticationTestNow); const repository = createJobRepository( fixture.database.orm, @@ -881,10 +881,13 @@ describe("durable jobs service", () => { await reconcileJobSchedules({ generateId, nowMs: serviceNowMs, repository }); expect(repository.findRun(run.id)).toMatchObject({ + attemptCount: 0, cancelRequestedAt: null, - eventCount: 1, - state: "queued", - terminalCode: null, + eventCount: 2, + firstStartedAt: null, + lastAttemptStartedAt: null, + state: "failed", + terminalCode: "action-unavailable", updatedAt: runAt, }); expect(repository.findSchedule(retiredScheduleId)?.schedule).toMatchObject({ @@ -904,6 +907,16 @@ describe("durable jobs service", () => { .orderBy(asc(realtimeEvents.id)) .all() ).toEqual([ + { + entityId: run.id, + occurredAt: runAt, + topic: "jobs.runs", + }, + { + entityId: retiredScheduleId, + occurredAt: runAt, + topic: "schedules.records", + }, { entityId: retiredScheduleId, occurredAt: authenticationTestNow, @@ -914,14 +927,9 @@ describe("durable jobs service", () => { fixture.database.orm .select({ action: auditEvents.action }) .from(auditEvents) - .where( - and( - eq(auditEvents.action, "jobs.run.cancel"), - eq(auditEvents.targetId, run.id) - ) - ) + .where(eq(auditEvents.targetId, run.id)) .all() - ).toEqual([]); + ).toEqual([{ action: "jobs.run.action-unavailable" }]); const eventCount = fixture.database.orm .select() diff --git a/greenfield/src/server/domains/jobs/service.ts b/greenfield/src/server/domains/jobs/service.ts index 46b3849f9..26ad85f38 100644 --- a/greenfield/src/server/domains/jobs/service.ts +++ b/greenfield/src/server/domains/jobs/service.ts @@ -9,6 +9,8 @@ import { type ScheduleSummary, jobWorkerFreshnessMs, jobTimestampSchema, + retiredScheduledActionTerminalCode, + retiredScheduledActionTerminalMessage, } from "../../../contracts/jobModel.ts"; import { type JobRunDetail, @@ -876,6 +878,16 @@ export async function reconcileJobSchedules( terminalMessage: "Cancelled because the schedule was retired from the action registry", }, + retiredRunFailure: { + sideEffectsForRun: (run) => + durableRunMutationSideEffects(generateId, run, { + action: "jobs.run.action-unavailable", + actor: systemActor, + outcome: "failed", + }), + terminalCode: retiredScheduledActionTerminalCode, + terminalMessage: retiredScheduledActionTerminalMessage, + }, schedules: jobActionDefinitions.map((definition) => scheduleInsertShape(definition, at) ), diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts index 331c085ea..a9399f6d0 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts @@ -49,7 +49,8 @@ function repositoryFixture() { const idempotencyReads: [JobRunRecord["requestedByKind"], string, string][] = []; let stored: JobRunRecord | undefined; const repository: ServiceActionQueueDependencies["repository"] = { - enqueueManualRun(input): Promise { + enqueueManualRun(input, beforeInsert): Promise { + beforeInsert?.(); enqueues.push(input); stored = { ...input.run, @@ -88,7 +89,7 @@ function request( return { actionId, actor, - authorizeDispatch: () => Promise.resolve(), + authorizeDispatch: () => Promise.resolve(() => {}), idempotencyKey, requestId: "request-1", ...overrides, @@ -130,11 +131,11 @@ describe("Service Action durable queue", () => { const result = await queue.enqueue( request(actionId, { - authorizeDispatch: () => { - authorizationChecks += 1; - expect(fixture.enqueues).toHaveLength(0); - return Promise.resolve(); - }, + authorizeDispatch: () => + Promise.resolve(() => { + authorizationChecks += 1; + expect(fixture.enqueues).toHaveLength(0); + }), }) ); @@ -178,10 +179,10 @@ describe("Service Action durable queue", () => { const replay = await queue.enqueue( request("system-update", { - authorizeDispatch: () => { - authorizationChecks += 1; - return Promise.resolve(); - }, + authorizeDispatch: () => + Promise.resolve(() => { + authorizationChecks += 1; + }), }) ); @@ -298,6 +299,26 @@ describe("Service Action durable queue", () => { expect(fixture.run()).toBeUndefined(); }); + test("does not insert when the final admitted authorization fence rejects", async () => { + const { fixture, queue } = queueFixture(); + const authorizationFailure = new Error("authorization changed during admission"); + + const failure = await queue + .enqueue( + request("system-restart", { + authorizeDispatch: () => + Promise.resolve(() => { + throw authorizationFailure; + }), + }) + ) + .catch((error: unknown) => error); + + expect(failure).toBe(authorizationFailure); + expect(fixture.enqueues).toEqual([]); + expect(fixture.run()).toBeUndefined(); + }); + test("rejects unsafe injected action mappings at composition", () => { expect(() => createServiceActionQueue({ diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.ts index 4714e14f2..d97dd7ff2 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.ts @@ -57,7 +57,8 @@ export interface ServiceActionQueueActor { export interface ServiceActionQueueRequest { readonly actionId: ServiceActionId; readonly actor: ServiceActionQueueActor; - readonly authorizeDispatch: () => Promise; + /** Preflights asynchronous availability and returns the final synchronous auth fence. */ + readonly authorizeDispatch: () => Promise<() => void>; readonly idempotencyKey: string; readonly requestId: string; readonly signal?: AbortSignal; @@ -272,13 +273,29 @@ export function createServiceActionQueue( }); request.signal?.throwIfAborted(); - await request.authorizeDispatch(); + const authorizeEnqueue = await request.authorizeDispatch(); request.signal?.throwIfAborted(); let enqueued: Awaited>; + let authorizationFailed = false; + let authorizationFailure: unknown; try { - enqueued = await dependencies.repository.enqueueManualRun(enqueueInput); + enqueued = await dependencies.repository.enqueueManualRun( + enqueueInput, + () => { + request.signal?.throwIfAborted(); + try { + authorizeEnqueue(); + request.signal?.throwIfAborted(); + } catch (error) { + authorizationFailed = true; + authorizationFailure = error; + throw error; + } + } + ); } catch { + if (authorizationFailed) throw authorizationFailure; let recovered: JobRunRecord | undefined; try { recovered = dependencies.repository.findRunByIdempotency( diff --git a/greenfield/src/server/domains/jobs/workerRuntime.ts b/greenfield/src/server/domains/jobs/workerRuntime.ts index 0f84b8659..f87b23a1a 100644 --- a/greenfield/src/server/domains/jobs/workerRuntime.ts +++ b/greenfield/src/server/domains/jobs/workerRuntime.ts @@ -1,9 +1,5 @@ import { Cause, Effect, Exit, Fiber, ManagedRuntime } from "effect"; -import { - hostOperationIds, - type FixedHostOperationsExecutionPort, -} from "../../../shared/hostOperations.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; import type { OpenClawServiceActionsExecutionPort } from "../../../shared/openClawServiceActions.ts"; import type { @@ -24,6 +20,8 @@ import type { MoltbookDashboardCollector } from "../moltbook/provider.ts"; import { createTaskNotificationQueue } from "../tasks/taskNotificationQueue.ts"; import { createJobWorkerActionResolver, + hostOperationIds, + type FixedHostOperationsExecutionPort, type LogMaintenanceExecutionPort, type WorkspaceFileWriteExecutionPort, } from "./actionExecutors.ts"; diff --git a/greenfield/src/server/domains/serviceActions/operationAudit.test.ts b/greenfield/src/server/domains/serviceActions/operationAudit.test.ts deleted file mode 100644 index 0be3d4f8f..000000000 --- a/greenfield/src/server/domains/serviceActions/operationAudit.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { asc } from "drizzle-orm"; - -import { auditEvents } from "../../database/schema/auditEvents.ts"; -import { testImmediateDatabaseWriteAdmission } from "../../test/support/databaseWriteAdmission.ts"; -import { openFreshMigratedDatabase } from "../../test/support/freshDatabase.ts"; -import { createSqliteServiceActionAuditWriter } from "./operationAudit.ts"; - -describe("service action operation audit", () => { - test("persists only fixed action, run identity, and classified settlement", async () => { - const database = await openFreshMigratedDatabase(); - const ids = [ - "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a1", - "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a2", - ]; - const writer = createSqliteServiceActionAuditWriter({ - clock: () => new Date(1000), - database: database.orm, - generateId: () => { - const id = ids.shift(); - if (id === undefined) throw new Error("Audit id budget exhausted"); - return id; - }, - writeAdmission: testImmediateDatabaseWriteAdmission, - }); - const context = { - actionId: "system-update", - actor: { - authenticatorId: "a".repeat(32), - id: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b4", - kind: "user", - }, - requestId: "request-1", - } as const; - - try { - await writer.record({ ...context, settlement: "attempted" }); - await writer.record({ - ...context, - jobRunId: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b5", - settlement: "succeeded", - }); - const rows = database.orm - .select() - .from(auditEvents) - .orderBy(asc(auditEvents.id)) - .all(); - - expect(rows).toMatchObject([ - { - action: "service-actions.system-update.request", - metadataJson: '{"settlement":"attempted"}', - outcome: "attempted", - requestId: "request-1", - targetId: "system-update", - targetType: "service-action", - }, - { - action: "service-actions.system-update.request", - metadataJson: '{"settlement":"succeeded"}', - outcome: "succeeded", - targetId: "019ff1c6-1a9b-7770-8f1b-d5b863b0e7b5", - targetType: "job-run", - }, - ]); - expect(JSON.stringify(rows)).not.toContain("apt-get"); - } finally { - database.sqlite.close(true); - } - }); -}); diff --git a/greenfield/src/server/domains/serviceActions/operationAudit.ts b/greenfield/src/server/domains/serviceActions/operationAudit.ts deleted file mode 100644 index eba23fc60..000000000 --- a/greenfield/src/server/domains/serviceActions/operationAudit.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"; - -import type { ServiceActionId } from "../../../contracts/serviceActions.ts"; -import type { ImmediateDatabaseWriteAdmission } from "../../database/immediateWriteAdmission.ts"; -import { createSecurityAuditEvent } from "../security/audit.ts"; -import { DrizzleSecurityAuditStore } from "../security/securityAuditStore.ts"; - -export type ServiceActionAuditSettlement = - | "attempted" - | "failed" - | "partial" - | "succeeded"; - -export interface ServiceActionAuditContext { - readonly actor: { - readonly authenticatorId: string; - readonly id: string; - readonly kind: "user"; - }; - readonly requestId: string; -} - -export interface ServiceActionAuditEvent extends ServiceActionAuditContext { - readonly actionId: ServiceActionId; - readonly jobRunId?: string; - readonly settlement: ServiceActionAuditSettlement; -} - -/** Durable audit append port. Commands, provider results, and host details are absent. */ -export interface ServiceActionAuditWriter { - readonly record: (event: ServiceActionAuditEvent) => Promise; -} - -export interface SqliteServiceActionAuditWriterOptions { - readonly clock?: () => Date; - readonly database: SQLiteBunDatabase; - readonly generateId?: () => string; - readonly writeAdmission: ImmediateDatabaseWriteAdmission; -} - -function auditOutcome( - settlement: ServiceActionAuditSettlement -): "attempted" | "failed" | "succeeded" { - if (settlement === "attempted") return "attempted"; - if (settlement === "succeeded") return "succeeded"; - return "failed"; -} - -/** - * Creates a fail-closed admitted audit writer for fixed privileged service actions. - * @param options Database, admission, clock, and identity dependencies. - * @returns A sanitized append-only audit writer. - */ -export function createSqliteServiceActionAuditWriter({ - clock = () => new Date(), - database, - generateId = () => Bun.randomUUIDv7(), - writeAdmission, -}: SqliteServiceActionAuditWriterOptions): ServiceActionAuditWriter { - return Object.freeze({ - record(input: ServiceActionAuditEvent) { - const event = createSecurityAuditEvent({ - action: `service-actions.${input.actionId}.request`, - actor: input.actor, - id: generateId(), - metadata: { settlement: input.settlement }, - occurredAt: clock(), - outcome: auditOutcome(input.settlement), - requestId: input.requestId, - targetId: input.jobRunId ?? input.actionId, - targetType: input.jobRunId === undefined ? "service-action" : "job-run", - }); - return writeAdmission.run((markTransactionStarted) => - database.transaction( - (transaction) => { - markTransactionStarted(); - new DrizzleSecurityAuditStore(transaction).insertAuditEvent( - event - ); - }, - { behavior: "immediate" } - ) - ); - }, - }); -} diff --git a/greenfield/src/server/domains/serviceActions/procedures.ts b/greenfield/src/server/domains/serviceActions/procedures.ts deleted file mode 100644 index e81336a44..000000000 --- a/greenfield/src/server/domains/serviceActions/procedures.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { router } from "../../trpc/trpc.ts"; -import { serviceActionsRoutes } from "./routes.ts"; - -/** Leaf procedure names owned by the fixed Service Actions router. */ -export const serviceActionsProcedureNames = Object.freeze( - Object.keys(serviceActionsRoutes) -); - -/** Session-only status and recent-MFA fixed-operation queue controls. */ -export const serviceActionsRouter = router(serviceActionsRoutes); diff --git a/greenfield/src/server/domains/serviceActions/routes.ts b/greenfield/src/server/domains/serviceActions/routes.ts index 7e560af78..09319f498 100644 --- a/greenfield/src/server/domains/serviceActions/routes.ts +++ b/greenfield/src/server/domains/serviceActions/routes.ts @@ -11,6 +11,7 @@ import type { RequestContext } from "../../trpc/context.ts"; import { authenticationPolicyError, operationOutcomeUnknownError, + router, sessionCapabilityProcedure, } from "../../trpc/trpc.ts"; import type { AuthenticatedBrowserIdentity } from "../security/authenticationSession.ts"; @@ -123,3 +124,11 @@ export const serviceActionsRoutes = { } }), }; + +/** Leaf procedure names owned by the fixed Service Actions router. */ +export const serviceActionsProcedureNames = Object.freeze( + Object.keys(serviceActionsRoutes) +); + +/** Session-only status and recent-MFA fixed-operation queue controls. */ +export const serviceActionsRouter = router(serviceActionsRoutes); diff --git a/greenfield/src/server/domains/serviceActions/service.test.ts b/greenfield/src/server/domains/serviceActions/service.test.ts index 2aeef3c43..5ff185f93 100644 --- a/greenfield/src/server/domains/serviceActions/service.test.ts +++ b/greenfield/src/server/domains/serviceActions/service.test.ts @@ -1,8 +1,17 @@ import { describe, expect, test } from "bun:test"; +import { asc } from "drizzle-orm"; + +import { auditEvents as storedAuditEvents } from "../../database/schema/auditEvents.ts"; +import { testImmediateDatabaseWriteAdmission } from "../../test/support/databaseWriteAdmission.ts"; +import { openFreshMigratedDatabase } from "../../test/support/freshDatabase.ts"; import { ServiceActionQueueError } from "../jobs/serviceActionQueue.ts"; -import type { ServiceActionAuditEvent } from "./operationAudit.ts"; -import { createServiceActionsService, ServiceActionsServiceError } from "./service.ts"; +import { + createServiceActionsService, + createSqliteServiceActionAuditWriter, + type ServiceActionAuditEvent, + ServiceActionsServiceError, +} from "./service.ts"; const actor = Object.freeze({ authenticatorId: "a".repeat(32), @@ -72,7 +81,8 @@ function fixture( }, queue: options.queue ?? { enqueue: async (request) => { - await request.authorizeDispatch(); + const authorizeEnqueue = await request.authorizeDispatch(); + authorizeEnqueue(); return queuedResult; }, }, @@ -115,6 +125,58 @@ async function captureFailure(work: () => Promise): Promise { } describe("service actions service", () => { + test("persists only fixed action, run identity, and classified settlement", async () => { + const database = await openFreshMigratedDatabase(); + const ids = [ + "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a1", + "019ff1c6-1a9b-7775-8f1b-d5b863b0e7a2", + ]; + const writer = createSqliteServiceActionAuditWriter({ + clock: () => new Date(1000), + database: database.orm, + generateId: () => { + const id = ids.shift(); + if (id === undefined) throw new Error("Audit id budget exhausted"); + return id; + }, + writeAdmission: testImmediateDatabaseWriteAdmission, + }); + const context = { + actionId: "system-update", + actor, + requestId: "request-1", + } as const; + + try { + await writer.record({ ...context, settlement: "attempted" }); + await writer.record({ ...context, jobRunId, settlement: "succeeded" }); + const rows = database.orm + .select() + .from(storedAuditEvents) + .orderBy(asc(storedAuditEvents.id)) + .all(); + expect(rows).toMatchObject([ + { + action: "service-actions.system-update.request", + metadataJson: '{"settlement":"attempted"}', + outcome: "attempted", + requestId: "request-1", + targetId: "system-update", + targetType: "service-action", + }, + { + action: "service-actions.system-update.request", + metadataJson: '{"settlement":"succeeded"}', + outcome: "succeeded", + targetId: jobRunId, + targetType: "job-run", + }, + ]); + expect(JSON.stringify(rows)).not.toContain("apt-get"); + } finally { + database.sqlite.close(true); + } + }); test("projects the exact bounded status inventory", async () => { const result = await fixture().service.getStatus(); expect(result).toMatchObject({ @@ -146,7 +208,9 @@ describe("service actions service", () => { queue: { enqueue: async (request) => { order.push("queue:preflight"); - await request.authorizeDispatch(); + const authorizeEnqueue = await request.authorizeDispatch(); + order.push("queue:admitted"); + authorizeEnqueue(); order.push("queue:enqueue"); return queuedResult; }, @@ -175,6 +239,7 @@ describe("service actions service", () => { expect(order).toEqual([ "audit:attempted", "queue:preflight", + "queue:admitted", "authorize", "queue:enqueue", "audit:succeeded", @@ -204,7 +269,8 @@ describe("service actions service", () => { const state = fixture({ queue: { enqueue: async (request) => { - await request.authorizeDispatch(); + const authorizeEnqueue = await request.authorizeDispatch(); + authorizeEnqueue(); durableEnqueue = true; return queuedResult; }, @@ -236,7 +302,12 @@ describe("service actions service", () => { const state = fixture({ queue: { enqueue: async (request) => { - await request.authorizeDispatch().catch(() => {}); + const authorizeEnqueue = await request.authorizeDispatch(); + try { + authorizeEnqueue(); + } catch { + // Deliberately emulate a queue bug swallowing the rejection. + } return queuedResult; }, }, diff --git a/greenfield/src/server/domains/serviceActions/service.ts b/greenfield/src/server/domains/serviceActions/service.ts index 9d74494d1..6889f62b0 100644 --- a/greenfield/src/server/domains/serviceActions/service.ts +++ b/greenfield/src/server/domains/serviceActions/service.ts @@ -1,3 +1,4 @@ +import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"; import * as v from "valibot"; import { @@ -9,15 +10,92 @@ import { requestServiceActionInputSchema, requestServiceActionResultSchema, } from "../../../contracts/serviceActions.ts"; +import type { ImmediateDatabaseWriteAdmission } from "../../database/immediateWriteAdmission.ts"; import { ServiceActionQueueError, type ServiceActionQueue, } from "../jobs/serviceActionQueue.ts"; -import { - type ServiceActionAuditContext, - type ServiceActionAuditSettlement, - type ServiceActionAuditWriter, -} from "./operationAudit.ts"; +import { createSecurityAuditEvent } from "../security/audit.ts"; +import { DrizzleSecurityAuditStore } from "../security/securityAuditStore.ts"; + +export type ServiceActionAuditSettlement = + | "attempted" + | "failed" + | "partial" + | "succeeded"; + +export interface ServiceActionAuditContext { + readonly actor: { + readonly authenticatorId: string; + readonly id: string; + readonly kind: "user"; + }; + readonly requestId: string; +} + +export interface ServiceActionAuditEvent extends ServiceActionAuditContext { + readonly actionId: RequestServiceActionInput["actionId"]; + readonly jobRunId?: string; + readonly settlement: ServiceActionAuditSettlement; +} + +/** Durable audit append port. Commands, provider results, and host details are absent. */ +export interface ServiceActionAuditWriter { + readonly record: (event: ServiceActionAuditEvent) => Promise; +} + +export interface SqliteServiceActionAuditWriterOptions { + readonly clock?: () => Date; + readonly database: SQLiteBunDatabase; + readonly generateId?: () => string; + readonly writeAdmission: ImmediateDatabaseWriteAdmission; +} + +function auditOutcome( + settlement: ServiceActionAuditSettlement +): "attempted" | "failed" | "succeeded" { + if (settlement === "attempted") return "attempted"; + if (settlement === "succeeded") return "succeeded"; + return "failed"; +} + +/** + * Creates a fail-closed admitted audit writer for fixed privileged service actions. + * @returns A sanitized append-only audit writer. + */ +export function createSqliteServiceActionAuditWriter({ + clock = () => new Date(), + database, + generateId = () => Bun.randomUUIDv7(), + writeAdmission, +}: SqliteServiceActionAuditWriterOptions): ServiceActionAuditWriter { + return Object.freeze({ + record(input: ServiceActionAuditEvent) { + const event = createSecurityAuditEvent({ + action: `service-actions.${input.actionId}.request`, + actor: input.actor, + id: generateId(), + metadata: { settlement: input.settlement }, + occurredAt: clock(), + outcome: auditOutcome(input.settlement), + requestId: input.requestId, + targetId: input.jobRunId ?? input.actionId, + targetType: input.jobRunId === undefined ? "service-action" : "job-run", + }); + return writeAdmission.run((markTransactionStarted) => + database.transaction( + (transaction) => { + markTransactionStarted(); + new DrizzleSecurityAuditStore(transaction).insertAuditEvent( + event + ); + }, + { behavior: "immediate" } + ) + ); + }, + }); +} export type ServiceActionsServiceErrorReason = | "audit-unavailable" @@ -37,7 +115,7 @@ export class ServiceActionsServiceError extends Error { } export interface ServiceActionControlContext extends ServiceActionAuditContext { - /** Re-checks the current session and recent MFA at durable enqueue handoff. */ + /** Re-checks the current session and recent MFA inside durable enqueue admission. */ readonly reauthorize: () => void; } @@ -171,14 +249,17 @@ export function createServiceActionsService( throw new ServiceActionsServiceError("unavailable"); } signal?.throwIfAborted(); - try { - context.reauthorize(); + return () => { signal?.throwIfAborted(); - } catch (error) { - authorizationFailed = true; - authorizationFailure = error; - throw error; - } + try { + context.reauthorize(); + signal?.throwIfAborted(); + } catch (error) { + authorizationFailed = true; + authorizationFailure = error; + throw error; + } + }; }, idempotencyKey: parsed.idempotencyKey, requestId: context.requestId, @@ -193,9 +274,9 @@ export function createServiceActionsService( await settleAudit(parsed, context, "succeeded", output.jobRunId); return output; } catch (error) { - if (authorizationFailed && error === authorizationFailure) { + if (authorizationFailed) { await settleAudit(parsed, context, "failed"); - throw error; + throw authorizationFailure; } let mapped: unknown; if (error instanceof ServiceActionQueueError) { diff --git a/greenfield/src/server/trpc/appRouter.ts b/greenfield/src/server/trpc/appRouter.ts index 7df6e7ca4..3edd75b83 100644 --- a/greenfield/src/server/trpc/appRouter.ts +++ b/greenfield/src/server/trpc/appRouter.ts @@ -63,7 +63,7 @@ import { import { serviceActionsProcedureNames, serviceActionsRouter, -} from "../domains/serviceActions/procedures.ts"; +} from "../domains/serviceActions/routes.ts"; import { systemProcedureNames, systemRouter } from "../domains/system/procedures.ts"; import { taskProcedureNames, taskRouter } from "../domains/tasks/procedures.ts"; import { diff --git a/greenfield/src/shared/databaseMigrationManifest.ts b/greenfield/src/shared/databaseMigrationManifest.ts index 562b87fc2..dd081b3da 100644 --- a/greenfield/src/shared/databaseMigrationManifest.ts +++ b/greenfield/src/shared/databaseMigrationManifest.ts @@ -13,8 +13,8 @@ export const migrationManifest = Object.freeze - | Readonly<{ status: "completed" }>; - -/** Worker-only fixed-operation authority; no command or path crosses this port. */ -export interface FixedHostOperationsExecutionPort { - readonly availableOperations: ( - signal?: AbortSignal - ) => Promise; - readonly request: ( - operationId: HostOperationId, - signal?: AbortSignal - ) => Promise; -} diff --git a/greenfield/src/test/parity/fixtures/legacy-endpoints.json b/greenfield/src/test/parity/fixtures/legacy-endpoints.json index 5aeca4cab..84ece83ab 100644 --- a/greenfield/src/test/parity/fixtures/legacy-endpoints.json +++ b/greenfield/src/test/parity/fixtures/legacy-endpoints.json @@ -1593,10 +1593,10 @@ "id": "POST /api/exec/start", "method": "POST", "path": "/api/exec/start", - "purpose": "Queues a worker-owned long-running exec job.", + "purpose": "Partially replaced by the bounded PTY and fixed Service Actions. This row stays planned until legacy system_cleanup is decomposed without feature loss: Docker prune in Docker control, apt cleanup in host/package maintenance, and journald vacuum in log maintenance.", "section": "Exec And Terminal", "target": { - "delivery": "implemented", + "delivery": "planned", "kind": "procedure", "names": ["serviceActions.request", "terminal.prepareSession"], "phase": "phase-5" diff --git a/greenfield/src/test/parity/parityInventory.test.ts b/greenfield/src/test/parity/parityInventory.test.ts index 116bb7596..c0750e2b9 100644 --- a/greenfield/src/test/parity/parityInventory.test.ts +++ b/greenfield/src/test/parity/parityInventory.test.ts @@ -297,7 +297,7 @@ describe("reviewed pre-cutover parity inventory", () => { ["POST /api/exec/:jobId/stop", "implemented", ["terminal.terminateSession"]], [ "POST /api/exec/start", - "implemented", + "planned", ["serviceActions.request", "terminal.prepareSession"], ], [ @@ -316,6 +316,11 @@ describe("reviewed pre-cutover parity inventory", () => { kind: "reviewed-removal", reason: expect.stringContaining("synchronous generic command endpoint"), }); + expect(endpoints[3]?.purpose).toContain("Docker prune in Docker control"); + expect(endpoints[3]?.purpose).toContain( + "apt cleanup in host/package maintenance" + ); + expect(endpoints[3]?.purpose).toContain("journald vacuum in log maintenance"); }); test("records the bounded OpenClaw settings and operations slice", async () => { From 3d4b80678885a87acb40e8376a606997e37c9577 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 13:54:16 +0200 Subject: [PATCH 06/13] fix(greenfield): document retired schedule check input --- greenfield/src/server/database/schema/jobChecks.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/greenfield/src/server/database/schema/jobChecks.ts b/greenfield/src/server/database/schema/jobChecks.ts index e4ab3f4e1..67baf2bc9 100644 --- a/greenfield/src/server/database/schema/jobChecks.ts +++ b/greenfield/src/server/database/schema/jobChecks.ts @@ -25,6 +25,7 @@ const retiredScheduledActionTerminalMessageSql = sql.raw( /** * Exact SQL form of the only failed lifecycle admitted before attempt one. + * @param columns Durable run columns participating in the canonical predicate. * @returns Canonical predicate shared by durable job-row constraints. */ export function unstartedRetiredScheduleFailureCheck(columns: { From d639cbd379c7d85cc2107d07feee5aed7f96d748 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 14:25:20 +0200 Subject: [PATCH 07/13] fix(greenfield): close service action review findings --- .../migration.sql | 2 +- .../snapshot.json | 2 +- .../scripts/audits/openclaw/sourceAudit.ts | 16 ++- .../overview/OverviewServiceActionsCard.tsx | 4 +- .../OverviewServiceActionsSection.test.tsx | 72 +++++++++-- .../OverviewServiceActionsSection.tsx | 32 ++++- .../overview/serviceActionsOperations.test.ts | 17 +++ .../overview/serviceActionsOperations.ts | 5 + .../migrations/migrationGraph.test.ts | 3 + .../server/database/schema/workerInstances.ts | 10 +- .../database/validation/rowSchemas.test.ts | 40 +++--- .../database/validation/workerInstances.ts | 16 ++- .../domains/jobs/actionExecutors.test.ts | 8 +- .../server/domains/jobs/actionExecutors.ts | 98 +++++++-------- .../domains/jobs/actionRegistry.test.ts | 31 +++++ .../src/server/domains/jobs/actionRegistry.ts | 4 +- .../server/domains/jobs/coordinator.test.ts | 20 +++ .../src/server/domains/jobs/coordinator.ts | 4 +- .../src/server/domains/jobs/repository.ts | 38 +++--- .../domains/jobs/serviceActionQueue.test.ts | 115 +++++++++++++++++- .../server/domains/jobs/serviceActionQueue.ts | 5 +- .../domains/serviceActions/service.test.ts | 24 ++++ .../server/domains/serviceActions/service.ts | 4 +- .../serviceActions/statusReader.test.ts | 17 ++- ...ewayOpenClawServiceActionsProvider.test.ts | 10 +- .../gateway/persistentGatewayProtocol.test.ts | 18 ++- .../gateway/persistentGatewayProtocol.ts | 10 +- .../observability/structuredLogger.test.ts | 32 +++++ .../observability/structuredLogger.ts | 18 ++- .../src/shared/databaseMigrationManifest.ts | 4 +- .../integration/openclaw/sourceAudit.test.ts | 35 ++++++ 31 files changed, 581 insertions(+), 133 deletions(-) diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql index 88831f5e3..2888300a7 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql +++ b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql @@ -1094,7 +1094,7 @@ CREATE TABLE `worker_instances` ( `started_at` integer NOT NULL, `state` text NOT NULL, `stopped_at` integer, - CONSTRAINT "worker_instances_action_keys_json_check" CHECK(length(CAST("action_keys_json" AS BLOB)) <= 4096 AND CASE WHEN json_valid("action_keys_json") THEN json_type("action_keys_json") = 'array' ELSE 0 END), + CONSTRAINT "worker_instances_action_keys_json_check" CHECK(length(CAST("action_keys_json" AS BLOB)) <= 4096 AND CASE WHEN json_valid("action_keys_json") THEN json_type("action_keys_json") = 'array' ELSE 0 END AND CASE WHEN json_valid("action_keys_json") THEN json_array_length("action_keys_json") <= 32 ELSE 0 END), CONSTRAINT "worker_instances_capacity_check" CHECK("capacity" BETWEEN 1 AND 16), CONSTRAINT "worker_instances_id_check" CHECK(length("id") = 36 AND instr("id", char(0)) = 0 AND length(replace("id", '-', '')) = 32 AND replace("id", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr("id", 9, 1) = '-' AND substr("id", 14, 1) = '-' AND substr("id", 15, 1) = '7' AND substr("id", 19, 1) = '-' AND substr("id", 20, 1) GLOB '[89ab]' AND substr("id", 24, 1) = '-'), CONSTRAINT "worker_instances_pid_check" CHECK("pid" BETWEEN 1 AND 2147483647), diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json index 1812b319e..fb044f48f 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json +++ b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json @@ -8427,7 +8427,7 @@ "table": "users" }, { - "value": "length(CAST(\"action_keys_json\" AS BLOB)) <= 4096 AND CASE WHEN json_valid(\"action_keys_json\") THEN json_type(\"action_keys_json\") = 'array' ELSE 0 END", + "value": "length(CAST(\"action_keys_json\" AS BLOB)) <= 4096 AND CASE WHEN json_valid(\"action_keys_json\") THEN json_type(\"action_keys_json\") = 'array' ELSE 0 END AND CASE WHEN json_valid(\"action_keys_json\") THEN json_array_length(\"action_keys_json\") <= 32 ELSE 0 END", "name": "worker_instances_action_keys_json_check", "entityType": "checks", "table": "worker_instances" diff --git a/greenfield/scripts/audits/openclaw/sourceAudit.ts b/greenfield/scripts/audits/openclaw/sourceAudit.ts index 45a47d45f..2de666e84 100644 --- a/greenfield/scripts/audits/openclaw/sourceAudit.ts +++ b/greenfield/scripts/audits/openclaw/sourceAudit.ts @@ -3111,16 +3111,20 @@ function assertOpenClawOperationsSemantics( 32 * 1024, "sessions.cleanup execution" ); + const cleanupLifecycleMutationCall = + "const lifecycleResult = await applySqliteSessionEntryLifecycleMutation({"; + const cleanupDiskBudgetCall = + "const appliedDiskBudget = await enforceSqliteSessionHistoryDiskBudget({"; assertRequiredMarkers(cleanupExecution, "sessions.cleanup execution", [ "const maintenance = resolveMaintenanceConfig()", 'const mode = opts.enforce ? "enforce" : maintenance.mode', "fixMissing: Boolean(opts.fixMissing)", "fixDmScope: Boolean(opts.fixDmScope)", - "const lifecycleResult = await applySqliteSessionEntryLifecycleMutation({", + cleanupLifecycleMutationCall, "activeSessionKey: opts.activeKey", "maintenanceOverride: {", 'const appliedUnreferencedArtifacts = mode === "warn" ? null : await pruneUnreferencedSessionArtifacts({', - "const appliedDiskBudget = await enforceSqliteSessionHistoryDiskBudget({", + cleanupDiskBudgetCall, "agentId: target.agentId", "storePath: target.storePath", "mode: appliedReport.mode", @@ -3138,6 +3142,14 @@ function assertOpenClawOperationsSemantics( "applied: true", "appliedCount: lifecycleResult.afterCount", ]); + if ( + cleanupExecution.indexOf(cleanupLifecycleMutationCall) > + cleanupExecution.indexOf(cleanupDiskBudgetCall) + ) { + throw new Error( + "OpenClaw sessions.cleanup no longer applies lifecycle mutation before disk budget enforcement" + ); + } const maintenancePolicy = artifactByRole( artifacts, diff --git a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx index 2edca8591..604537a93 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx @@ -106,9 +106,7 @@ function ServiceActionRow({ onClick={() => onSelect(action.id)} variant={action.id === "system-restart" ? "danger" : "secondary"} > - {recoveryPending - ? `Retry ${presentation.actionLabel.toLowerCase()} request` - : presentation.buttonLabel} + {recoveryPending ? presentation.retryLabel : presentation.buttonLabel}
    diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx index e2929e564..459b2ed45 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx @@ -8,20 +8,26 @@ import { createRouter, RouterProvider, } from "@tanstack/react-router"; +import { act } from "react"; import type { AuthStatus } from "../../contracts/auth.ts"; +import type { RealtimeStreamOutput } from "../../contracts/events.ts"; import type { JobRunSummary } from "../../contracts/jobModel.ts"; +import { jobRealtimeTopics } from "../../contracts/jobRealtime.ts"; import type { GetServiceActionsStatusResult, RequestServiceActionResult, } from "../../contracts/serviceActions.ts"; import { createDashboardQueryClient } from "../api/queryClient.ts"; +import { DashboardRealtimeProvider } from "../api/realtimeContext.tsx"; import { createDashboardTrpcClient, type DashboardTrpcTransport, } from "../api/trpcClient.ts"; import { DashboardTrpcProvider } from "../api/trpcContext.tsx"; import { authStatusQueryKey } from "../auth/authQueries.ts"; +import { jobRealtimeRefreshDelayMs } from "../jobs/useJobRealtimeInvalidation.ts"; +import { ControlledDashboardRealtimeClient } from "../test/realtime.ts"; import { OverviewServiceActionsSection } from "./OverviewServiceActionsSection.tsx"; const { render, screen, waitFor } = await import("@testing-library/react"); @@ -196,6 +202,7 @@ class ServiceActionsTransport implements DashboardTrpcTransport { interface SectionHarness { readonly queryClient: ReturnType; + readonly realtimeClient: ControlledDashboardRealtimeClient; readonly transport: ServiceActionsTransport; readonly view: ReturnType; } @@ -225,6 +232,7 @@ function renderSection( queryClient.setQueryData(authStatusQueryKey, authenticatedStatus); const transport = new ServiceActionsTransport(queryOutputs, mutationOutputs); const trpcClient = createDashboardTrpcClient(transport); + const realtimeClient = new ControlledDashboardRealtimeClient(); const rootRoute = createRootRoute(); const overviewRoute = createRoute({ component: OverviewServiceActionsSection, @@ -242,16 +250,43 @@ function renderSection( }); const view = render( - - - + + + + + ); - const harness = { queryClient, transport, view }; + const harness = { queryClient, realtimeClient, transport, view }; harnesses.push(harness); return harness; } +async function emitJobRunChange( + realtimeClient: ControlledDashboardRealtimeClient +): Promise { + const output: RealtimeStreamOutput = { + data: { + event: { + entityId: runningRun.id, + entityType: "job-run", + occurredAtMs: timestampMs + 200, + operation: "updated", + payload: { id: runningRun.id }, + topic: jobRealtimeTopics.runs, + }, + kind: "change", + }, + id: "42", + }; + await act(async () => { + realtimeClient.emit(output); + await new Promise((resolve) => + setTimeout(resolve, jobRealtimeRefreshDelayMs + 20) + ); + }); +} + function operationOutcomeUnknownError(): Error { return Object.assign(new Error("private lost acknowledgement"), { data: { @@ -297,6 +332,29 @@ describe("OverviewServiceActionsSection", () => { ); }); + test("clears an active action after a same-tab job-run event", async () => { + const harness = renderSection([actionStatus, allAvailableStatus]); + + expect(await screen.findByText("Active job")).toBeTruthy(); + expect( + screen.getByRole("button", { name: "Queue system restart" }) + ).toBeDisabled(); + expect(harness.realtimeClient.input?.topics).toEqual([jobRealtimeTopics.runs]); + + const callCountBeforeRealtimeChange = harness.transport.queryCalls.length; + await emitJobRunChange(harness.realtimeClient); + + await waitFor(() => + expect( + screen.getByRole("button", { name: "Queue system restart" }) + ).toBeEnabled() + ); + expect(screen.queryByText("Active job")).toBeNull(); + expect(harness.transport.queryCalls.length).toBeGreaterThan( + callCountBeforeRealtimeChange + ); + }); + test("recovers initial errors and retains validated status after a refresh failure", async () => { const failure = new TypeError("private service-actions provider detail"); const harness = renderSection([failure, actionStatus, failure]); @@ -388,11 +446,11 @@ describe("OverviewServiceActionsSection", () => { ); expect( await screen.findByRole("button", { - name: "Retry openclaw cleanup request", + name: "Retry OpenClaw cleanup request", }) ).toBeTruthy(); await user.click( - screen.getByRole("button", { name: "Retry openclaw cleanup request" }) + screen.getByRole("button", { name: "Retry OpenClaw cleanup request" }) ); expect( screen.getByText(/retry uses the retained request identity/iu) @@ -450,7 +508,7 @@ describe("OverviewServiceActionsSection", () => { await screen.findByRole("heading", { name: "Service actions" }); await user.click( screen.getByRole("button", { - name: "Retry openclaw cleanup request", + name: "Retry OpenClaw cleanup request", }) ); await user.click(screen.getByRole("button", { name: "Retry request" })); diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx index 4a7acbe75..bb584b59f 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx @@ -1,6 +1,12 @@ -import { queryOptions, useMutation, useQuery } from "@tanstack/react-query"; +import { + type QueryClient, + queryOptions, + useMutation, + useQuery, +} from "@tanstack/react-query"; import { useState } from "react"; +import { jobRealtimeTopics } from "../../contracts/jobRealtime.ts"; import type { ServiceActionId } from "../../contracts/serviceActions.ts"; import type { DashboardProcedureOutput, DashboardTrpcClient } from "../api/trpcClient.ts"; import { useDashboardTrpcClient } from "../api/trpcContextValue.ts"; @@ -10,7 +16,12 @@ import { isDashboardOperationOutcomeUnknown, retryDashboardUnavailableRead, } from "../api/trpcError.ts"; +import { useRealtimeQueryInvalidation } from "../api/useRealtimeQueryInvalidation.ts"; import { useAuthenticatedMutationBoundary } from "../auth/useAuthenticatedMutationBoundary.ts"; +import { + jobRealtimeFallbackRefreshIntervalMs, + jobRealtimeRefreshDelayMs, +} from "../jobs/useJobRealtimeInvalidation.ts"; import { Alert } from "../ui/Alert.tsx"; import { Card } from "../ui/Card.tsx"; import { PageState } from "../ui/PageState.tsx"; @@ -40,6 +51,24 @@ function serviceActionsStatusQueryOptions(client: DashboardTrpcClient) { }); } +async function refreshServiceActionsStatus(queryClient: QueryClient): Promise { + await queryClient.invalidateQueries({ + exact: true, + queryKey: serviceActionsStatusQueryKey, + refetchType: "active", + }); +} + +/** Refreshes active fixed-action projections after durable job-run changes. */ +function useServiceActionsRealtimeInvalidation(): void { + useRealtimeQueryInvalidation({ + fallbackRefreshIntervalMs: jobRealtimeFallbackRefreshIntervalMs, + refreshDelayMs: jobRealtimeRefreshDelayMs, + refreshQueries: refreshServiceActionsStatus, + topic: jobRealtimeTopics.runs, + }); +} + /** * Owns session-bound fixed-action requests and lost-response recovery identities. * @returns One no-retry mutation plus safe feedback and recovery observations. @@ -125,6 +154,7 @@ function useServiceActionRequest() { /** @returns Fixed service-action status, requests, and partial-read handling. */ export function OverviewServiceActionsSection() { + useServiceActionsRealtimeInvalidation(); const client = useDashboardTrpcClient(); const query = useQuery(serviceActionsStatusQueryOptions(client)); const request = useServiceActionRequest(); diff --git a/greenfield/src/browser/overview/serviceActionsOperations.test.ts b/greenfield/src/browser/overview/serviceActionsOperations.test.ts index 509c1d2ab..b0c7d2c38 100644 --- a/greenfield/src/browser/overview/serviceActionsOperations.test.ts +++ b/greenfield/src/browser/overview/serviceActionsOperations.test.ts @@ -8,6 +8,7 @@ import { authenticatedServiceActionIdentity, clearServiceActionRecovery, readOrCreateServiceActionIdempotencyKey, + serviceActionPresentations, serviceActionRecoveryExists, serviceActionRequestInput, } from "./serviceActionsOperations.ts"; @@ -104,4 +105,20 @@ describe("service action browser operations", () => { idempotencyKey, }); }); + + test("keeps explicit retry labels for every fixed action", () => { + expect( + Object.fromEntries( + Object.entries(serviceActionPresentations).map(([id, presentation]) => [ + id, + presentation.retryLabel, + ]) + ) + ).toEqual({ + "openclaw-cleanup": "Retry OpenClaw cleanup request", + "openclaw-update": "Retry OpenClaw update request", + "system-restart": "Retry system restart request", + "system-update": "Retry system update request", + }); + }); }); diff --git a/greenfield/src/browser/overview/serviceActionsOperations.ts b/greenfield/src/browser/overview/serviceActionsOperations.ts index 27a776942..e6cca7273 100644 --- a/greenfield/src/browser/overview/serviceActionsOperations.ts +++ b/greenfield/src/browser/overview/serviceActionsOperations.ts @@ -18,6 +18,7 @@ export interface ServiceActionPresentation { readonly confirmationLabel: string; readonly confirmationTitle: string; readonly description: string; + readonly retryLabel: string; readonly warning: string; } @@ -29,6 +30,7 @@ export const serviceActionPresentations = Object.freeze({ confirmationTitle: "Queue OpenClaw cleanup?", description: "Runs source-owned OpenClaw session and artifact maintenance without generic filesystem or Docker cleanup.", + retryLabel: "Retry OpenClaw cleanup request", warning: "This queues OpenClaw's own bounded session and artifact maintenance. Review Dashboard jobs for the durable result.", }, @@ -39,6 +41,7 @@ export const serviceActionPresentations = Object.freeze({ confirmationTitle: "Queue OpenClaw update?", description: "Requests the source-owned OpenClaw update workflow through a fixed worker action.", + retryLabel: "Retry OpenClaw update request", warning: "OpenClaw updates can take time and may restart the Gateway. The Dashboard only confirms that the durable request was queued.", }, @@ -49,6 +52,7 @@ export const serviceActionPresentations = Object.freeze({ confirmationTitle: "Queue a system restart?", description: "Requests a fixed host restart through the separately provisioned worker boundary.", + retryLabel: "Retry system restart request", warning: "A system restart request interrupts Dashboard, OpenClaw, and other host services. Success here means the restart request was accepted for durable processing, not that the host restarted.", }, @@ -59,6 +63,7 @@ export const serviceActionPresentations = Object.freeze({ confirmationTitle: "Queue a system update?", description: "Runs the fixed host package-update workflow through the separately provisioned worker boundary.", + retryLabel: "Retry system update request", warning: "System updates can take a long time and may affect running services. Review Dashboard jobs for the durable result.", }, diff --git a/greenfield/src/server/database/migrations/migrationGraph.test.ts b/greenfield/src/server/database/migrations/migrationGraph.test.ts index a8a0e7519..64c59412d 100644 --- a/greenfield/src/server/database/migrations/migrationGraph.test.ts +++ b/greenfield/src/server/database/migrations/migrationGraph.test.ts @@ -184,6 +184,9 @@ describe("database migration graph", () => { expect(foundationSql).toContain( 'CONSTRAINT "monitor_runs_submission_sha256_check" CHECK(length("submission_sha256") = 64 AND instr("submission_sha256", char(0)) = 0' ); + expect(foundationSql).toContain( + 'CONSTRAINT "worker_instances_action_keys_json_check" CHECK(length(CAST("action_keys_json" AS BLOB)) <= 4096 AND CASE WHEN json_valid("action_keys_json") THEN json_type("action_keys_json") = \'array\' ELSE 0 END AND CASE WHEN json_valid("action_keys_json") THEN json_array_length("action_keys_json") <= 32 ELSE 0 END)' + ); expect(foundationSql).not.toContain("legacy"); expect(foundationSql).not.toContain("SET fingerprint = fingerprint"); expect(foundationSql).not.toContain("SET submission_sha256 = submission_sha256"); diff --git a/greenfield/src/server/database/schema/workerInstances.ts b/greenfield/src/server/database/schema/workerInstances.ts index 79d1cbe02..bf2f05837 100644 --- a/greenfield/src/server/database/schema/workerInstances.ts +++ b/greenfield/src/server/database/schema/workerInstances.ts @@ -6,7 +6,13 @@ import { timestampMillisecondsCheck, uuidV7TextCheck, } from "./checks.ts"; -import { boundedJsonArrayCheck, workerActionKeysMaximumBytes } from "./jobChecks.ts"; +import { + boundedJsonArrayCheck, + workerActionKeyMaximum, + workerActionKeysMaximumBytes, +} from "./jobChecks.ts"; + +const workerActionKeyMaximumSql = sql.raw(String(workerActionKeyMaximum)); /** Durable worker registration and heartbeat state shared across rolling releases. */ export const workerInstances = sqliteTable( @@ -26,7 +32,7 @@ export const workerInstances = sqliteTable( (table) => [ check( "worker_instances_action_keys_json_check", - boundedJsonArrayCheck(table.actionKeysJson, workerActionKeysMaximumBytes) + sql`${boundedJsonArrayCheck(table.actionKeysJson, workerActionKeysMaximumBytes)} AND CASE WHEN json_valid(${table.actionKeysJson}) THEN json_array_length(${table.actionKeysJson}) <= ${workerActionKeyMaximumSql} ELSE 0 END` ), check("worker_instances_capacity_check", sql`${table.capacity} BETWEEN 1 AND 16`), check("worker_instances_id_check", uuidV7TextCheck(table.id)), diff --git a/greenfield/src/server/database/validation/rowSchemas.test.ts b/greenfield/src/server/database/validation/rowSchemas.test.ts index a878ddaf4..c4059b9ee 100644 --- a/greenfield/src/server/database/validation/rowSchemas.test.ts +++ b/greenfield/src/server/database/validation/rowSchemas.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import * as v from "valibot"; +import { workerActionKeysMaximumBytes } from "../schema/jobChecks.ts"; import { incidentObservationInsertSchema } from "./incidentObservations.ts"; import { incidentInsertSchema, @@ -723,20 +724,31 @@ describe("Drizzle-generated Valibot row schemas", () => { stoppedAt: null, }) ).toThrow(); - expect(() => - v.parse(workerInstanceInsertSchema, { - actionKeysJson: '["host.system.update","host.system.restart"]', - capacity: 2, - drainingAt: null, - heartbeatAt: jobUpdatedAt, - id: jobWorkerId, - pid: 1234, - releaseId: "b".repeat(40), - startedAt: jobCreatedAt, - state: "online", - stoppedAt: null, - }) - ).toThrow("Stored worker action keys are invalid"); + const validWorkerInsert = { + actionKeysJson: '["host.system.restart","host.system.update"]', + capacity: 2, + drainingAt: null, + heartbeatAt: jobUpdatedAt, + id: jobWorkerId, + pid: 1234, + releaseId: "b".repeat(40), + startedAt: jobCreatedAt, + state: "online" as const, + stoppedAt: null, + }; + expect(v.parse(workerInstanceInsertSchema, validWorkerInsert)).toBeDefined(); + for (const actionKeysJson of [ + '["host.system.update","host.system.restart"]', + ' ["host.system.restart","host.system.update"]', + `${" ".repeat(workerActionKeysMaximumBytes)}[]`, + ]) { + expect(() => + v.parse(workerInstanceInsertSchema, { + ...validWorkerInsert, + actionKeysJson, + }) + ).toThrow("Stored worker action keys are invalid"); + } expect(() => v.parse(resourceLeaseSelectSchema, { acquiredAt: jobCreatedAt, diff --git a/greenfield/src/server/database/validation/workerInstances.ts b/greenfield/src/server/database/validation/workerInstances.ts index 4e952af34..045e6700e 100644 --- a/greenfield/src/server/database/validation/workerInstances.ts +++ b/greenfield/src/server/database/validation/workerInstances.ts @@ -68,7 +68,21 @@ export function serializeWorkerActionKeys(actionKeys: readonly string[]): string * @returns Frozen validated canonical action identities. */ export function parseWorkerActionKeysJson(value: string): readonly string[] { - const parsed = v.parse(workerActionKeysSchema, parseJsonText(value)); + const boundedValue = v.parse( + v.pipe( + v.string("Stored worker action keys are invalid"), + v.check( + (candidate) => utf8ByteLength(candidate) <= workerActionKeysMaximumBytes, + "Stored worker action keys are invalid" + ) + ), + value + ); + const parsed = v.parse(workerActionKeysSchema, parseJsonText(boundedValue)); + v.parse( + v.literal(JSON.stringify(parsed), "Stored worker action keys are invalid"), + boundedValue + ); return Object.freeze([...parsed]); } diff --git a/greenfield/src/server/domains/jobs/actionExecutors.test.ts b/greenfield/src/server/domains/jobs/actionExecutors.test.ts index ea186761b..dc9d2b528 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.test.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.test.ts @@ -79,6 +79,8 @@ describe("worker-only job executor registry", () => { }); expect(findAction("openclaw.sessions.cleanup")).toBeDefined(); expect(findAction("openclaw.installation.update")).toBeDefined(); + expect(findAction("host.system.restart")).toBeUndefined(); + expect(findAction("host.system.update")).toBeUndefined(); expect(findAction("system.shell")).toBeUndefined(); }); @@ -233,7 +235,8 @@ describe("worker-only job executor registry", () => { )(executionContext([]), {}) ).catch((error: unknown) => error); expect(failure).toBeInstanceOf(Error); - expect(JSON.stringify(failure)).not.toContain("../../private"); + expect(String(failure)).not.toContain("../../private"); + expect((failure as Error).message).not.toContain("private"); const unknownFailure = await Effect.runPromise( createOpenClawServiceActionJobExecutor( @@ -248,7 +251,8 @@ describe("worker-only job executor registry", () => { )(executionContext([]), {}) ).catch((error: unknown) => error); expect(unknownFailure).toBeInstanceOf(JobActionOutcomeUnknownError); - expect(JSON.stringify(unknownFailure)).not.toContain("Gateway"); + expect(String(unknownFailure)).not.toContain("Gateway"); + expect((unknownFailure as Error).message).not.toContain("Gateway"); }); test("fails closed for missing, extra, and duplicate executor keys", () => { diff --git a/greenfield/src/server/domains/jobs/actionExecutors.ts b/greenfield/src/server/domains/jobs/actionExecutors.ts index ca1157d84..f7196c6e0 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.ts @@ -341,7 +341,7 @@ export function createOpenClawGatewayRestartJobExecutor( */ export function createHostOperationJobExecutor( hostOperations: FixedHostOperationsExecutionPort, - operationId: "system-restart" | "system-update" + operationId: HostOperationId ): JobActionExecutor { return (context, payload) => Effect.tryPromise({ @@ -532,6 +532,14 @@ export function createJobWorkerActionResolver( workspaceFileReplaceJobActionDefinition, ]), ]); + const registeredActionKeys = new Set(definitions.map(({ actionKey }) => actionKey)); + const gatedExecutor = ( + actionKey: string, + execute: JobActionExecutor | undefined + ): readonly JobActionExecutorEntry[] => + execute === undefined || !registeredActionKeys.has(actionKey) + ? [] + : [Object.freeze({ actionKey, execute })]; const executors = [ Object.freeze({ actionKey: "cache.refresh.system-host", @@ -557,58 +565,42 @@ export function createJobWorkerActionResolver( ), }), ]), - ...(dependencies.openClawServiceActions === undefined || - !definitions.some( - ({ actionKey }) => actionKey === openClawSessionsCleanupJobActionKey - ) - ? [] - : [ - Object.freeze({ - actionKey: openClawSessionsCleanupJobActionKey, - execute: createOpenClawServiceActionJobExecutor( - dependencies.openClawServiceActions, - "openclaw-cleanup" - ), - }), - ]), - ...(dependencies.openClawServiceActions === undefined || - !definitions.some( - ({ actionKey }) => actionKey === openClawInstallationUpdateJobActionKey - ) - ? [] - : [ - Object.freeze({ - actionKey: openClawInstallationUpdateJobActionKey, - execute: createOpenClawServiceActionJobExecutor( - dependencies.openClawServiceActions, - "openclaw-update" - ), - }), - ]), - ...(dependencies.hostOperations === undefined || - !definitions.some(({ actionKey }) => actionKey === hostSystemRestartJobActionKey) - ? [] - : [ - Object.freeze({ - actionKey: hostSystemRestartJobActionKey, - execute: createHostOperationJobExecutor( - dependencies.hostOperations, - "system-restart" - ), - }), - ]), - ...(dependencies.hostOperations === undefined || - !definitions.some(({ actionKey }) => actionKey === hostSystemUpdateJobActionKey) - ? [] - : [ - Object.freeze({ - actionKey: hostSystemUpdateJobActionKey, - execute: createHostOperationJobExecutor( - dependencies.hostOperations, - "system-update" - ), - }), - ]), + ...gatedExecutor( + openClawSessionsCleanupJobActionKey, + dependencies.openClawServiceActions === undefined + ? undefined + : createOpenClawServiceActionJobExecutor( + dependencies.openClawServiceActions, + "openclaw-cleanup" + ) + ), + ...gatedExecutor( + openClawInstallationUpdateJobActionKey, + dependencies.openClawServiceActions === undefined + ? undefined + : createOpenClawServiceActionJobExecutor( + dependencies.openClawServiceActions, + "openclaw-update" + ) + ), + ...gatedExecutor( + hostSystemRestartJobActionKey, + dependencies.hostOperations === undefined + ? undefined + : createHostOperationJobExecutor( + dependencies.hostOperations, + "system-restart" + ) + ), + ...gatedExecutor( + hostSystemUpdateJobActionKey, + dependencies.hostOperations === undefined + ? undefined + : createHostOperationJobExecutor( + dependencies.hostOperations, + "system-update" + ) + ), Object.freeze({ actionKey: "system.worker-smoke", execute: workerSmokeExecutor, diff --git a/greenfield/src/server/domains/jobs/actionRegistry.test.ts b/greenfield/src/server/domains/jobs/actionRegistry.test.ts index a6c49084c..5dab2bbef 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.test.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test } from "bun:test"; +import * as v from "valibot"; + import { findJobActionDefinition, hostSystemRestartJobActionDefinition, @@ -7,7 +9,9 @@ import { isRegisteredJobSchedule, openClawGatewayRestartJobActionDefinition, openClawInstallationUpdateJobActionDefinition, + openClawInstallationUpdateJobResultSchema, openClawSessionsCleanupJobActionDefinition, + openClawSessionsCleanupJobResultSchema, parseJobActionOutputMessage, parseJobActionProgress, validateJobActionRegistration, @@ -186,4 +190,31 @@ describe("durable job action registry", () => { expect(openClawSessionsCleanupJobActionDefinition.timeoutMs).toBe(630_000); expect(openClawInstallationUpdateJobActionDefinition.timeoutMs).toBe(2_130_000); }); + + test("reports explicit validation errors for invalid OpenClaw result statuses", () => { + expect(() => + v.parse(openClawSessionsCleanupJobResultSchema, { + artifactsRemoved: 0, + bytesFreed: 0, + completedAtMs: 1, + diskEntriesRemoved: 0, + diskFilesRemoved: 0, + dmScopesRetired: 0, + entriesAfter: 0, + entriesBefore: 0, + entriesCapped: 0, + entriesPruned: 0, + missingEntriesRemoved: 0, + modelRunsPruned: 0, + status: "failed", + storesProcessed: 0, + }) + ).toThrow("OpenClaw cleanup result is invalid"); + expect(() => + v.parse(openClawInstallationUpdateJobResultSchema, { + completedAtMs: 1, + status: "failed", + }) + ).toThrow("OpenClaw update result is invalid"); + }); }); diff --git a/greenfield/src/server/domains/jobs/actionRegistry.ts b/greenfield/src/server/domains/jobs/actionRegistry.ts index 2d5c60a01..7c01c52ee 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.ts @@ -100,7 +100,7 @@ export const openClawSessionsCleanupJobResultSchema = v.strictObject({ entriesPruned: openClawOperationCountSchema, missingEntriesRemoved: openClawOperationCountSchema, modelRunsPruned: openClawOperationCountSchema, - status: v.literal("completed"), + status: v.literal("completed", "OpenClaw cleanup result is invalid"), storesProcessed: openClawOperationCountSchema, }); @@ -109,7 +109,7 @@ export const openClawInstallationUpdateJobResultSchema = v.strictObject({ afterVersion: v.optional(openClawOperationVersionSchema), beforeVersion: v.optional(openClawOperationVersionSchema), completedAtMs: jobTimestampSchema, - status: v.picklist(["accepted", "completed"]), + status: v.picklist(["accepted", "completed"], "OpenClaw update result is invalid"), }); export type JobCacheAttemptCommit = diff --git a/greenfield/src/server/domains/jobs/coordinator.test.ts b/greenfield/src/server/domains/jobs/coordinator.test.ts index a2c9e8979..88b94f208 100644 --- a/greenfield/src/server/domains/jobs/coordinator.test.ts +++ b/greenfield/src/server/domains/jobs/coordinator.test.ts @@ -508,6 +508,26 @@ describe("durable job worker coordinator", () => { expect(await coordinator.completion).toBeUndefined(); }); + test("advertises only action definitions backed by an executable resolver", async () => { + const workerId = Bun.randomUUIDv7(); + const fixture = repositoryFixture(); + const options = coordinatorOptions(fixture.repository, workerId); + const coordinator = createJobWorkerCoordinator({ + ...options, + actionDefinitions: Object.freeze([ + ...(options.actionDefinitions ?? []), + openClawGatewayRestartJobActionDefinition, + ]), + }); + + await coordinator.initialize(); + await coordinator.dispose(); + + expect(fixture.registrations[0]?.worker.actionKeysJson).toBe( + '["system.worker-smoke"]' + ); + }); + test("claims the conditionally registered Gateway restart without scheduling it", async () => { const workerId = Bun.randomUUIDv7(); const definition = openClawGatewayRestartJobActionDefinition; diff --git a/greenfield/src/server/domains/jobs/coordinator.ts b/greenfield/src/server/domains/jobs/coordinator.ts index 44deb20c1..ce76686da 100644 --- a/greenfield/src/server/domains/jobs/coordinator.ts +++ b/greenfield/src/server/domains/jobs/coordinator.ts @@ -785,7 +785,9 @@ export function createJobWorkerCoordinator( const findAction = options.findAction ?? findNoAction; const actionDefinitions = options.actionDefinitions ?? jobActionDefinitions; const actionKeysJson = serializeWorkerActionKeys( - actionDefinitions.map(({ actionKey }) => actionKey) + actionDefinitions.flatMap(({ actionKey }) => + findAction(actionKey) === undefined ? [] : [actionKey] + ) ); const abortController = new AbortController(); let activeExecution: Promise | undefined; diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index 221ddb436..8a8d8c1ad 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -272,7 +272,7 @@ export interface ScheduleUpdateChanges { readonly schedule?: ScheduleConfiguration; } -export interface ScheduleQueuedCancellation { +export interface ScheduleQueuedTermination { readonly at: Date; readonly terminalCode: string; readonly terminalMessage: string; @@ -286,7 +286,7 @@ export interface UpdateScheduleRepositoryInput extends JobMutationSideEffects { readonly id: string; readonly insertDisableIntent?: JobDisableIntentInsert; readonly patch: ScheduleUpdateChanges; - readonly queuedCancellation?: ScheduleQueuedCancellation; + readonly queuedCancellation?: ScheduleQueuedTermination; readonly queuedCancellationSideEffects?: ( run: JobRunRecord ) => JobMutationSideEffects; @@ -540,7 +540,7 @@ export interface JobRepository extends JobRepositoryReader { claimNextRun(input: ClaimNextRunInput): Promise; enqueueManualRun( input: EnqueueManualRunInput, - beforeInsert?: () => void + authorizeAdmittedEnqueue?: () => void ): Promise; enqueueNextDueSchedule( input: DueScheduleEnqueueInput @@ -1486,10 +1486,7 @@ class DrizzleJobWriter extends DrizzleJobReader { return records; } - public enqueueManualRun( - input: EnqueueManualRunInput, - beforeInsert?: () => void - ): EnqueueManualRunResult { + public enqueueManualRun(input: EnqueueManualRunInput): EnqueueManualRunResult { const run = v.parse(jobRunInsertSchema, input.run); if (input.queuedEvent.jobRunId !== run.id) { throw new Error("Queued event does not belong to the inserted manual run"); @@ -1523,7 +1520,6 @@ class DrizzleJobWriter extends DrizzleJobReader { .get(); if (active !== undefined) return { kind: "active", run: parseRun(active) }; } - beforeInsert?.(); const inserted = this.#transaction.insert(jobRuns).values(run).returning().get(); const record = parseRun(requiredRow(inserted, "manual run insert")); this.#insertSuppliedEvent(input.queuedEvent); @@ -2545,7 +2541,7 @@ class DrizzleJobWriter extends DrizzleJobReader { #cancelQueuedRun( run: JobRunRecord, actor: JobActor, - input: ScheduleQueuedCancellation + input: ScheduleQueuedTermination ): JobRunRecord { const at = maximumDate(run.updatedAt, input.at); const row = this.#transaction @@ -2587,7 +2583,7 @@ class DrizzleJobWriter extends DrizzleJobReader { return requiredRow(this.findRun(run.id), "cancelled run refresh"); } - #failQueuedRun(run: JobRunRecord, input: ScheduleQueuedCancellation): JobRunRecord { + #failQueuedRun(run: JobRunRecord, input: ScheduleQueuedTermination): JobRunRecord { const at = maximumDate(run.updatedAt, input.at); const row = this.#transaction .update(jobRuns) @@ -2608,7 +2604,7 @@ class DrizzleJobWriter extends DrizzleJobReader { ) .returning() .get(); - parseRun(requiredRow(row, "queued run failure")); + requiredRow(row, "queued run failure"); this.#appendEvent(run.id, { attempt: run.attemptCount, kind: "failed", @@ -2836,16 +2832,20 @@ export function createJobRepository( runTransaction((transaction) => callback(new DrizzleJobReader(transaction)), { behavior: "deferred", }); - const write = (callback: (writer: DrizzleJobWriter) => T): Promise => - writeAdmission.run((markTransactionStarted) => - runTransaction( + const write = ( + callback: (writer: DrizzleJobWriter) => T, + authorizeAdmittedWrite?: () => void + ): Promise => + writeAdmission.run((markTransactionStarted) => { + authorizeAdmittedWrite?.(); + return runTransaction( (transaction) => { markTransactionStarted(); return callback(new DrizzleJobWriter(transaction)); }, { behavior: "immediate" } - ) - ); + ); + }); return Object.freeze({ appendClaimEvent: (input: AppendClaimEventInput) => @@ -2856,8 +2856,10 @@ export function createJobRepository( write((writer) => writer.cancelRun(input)), claimNextRun: (input: ClaimNextRunInput) => write((writer) => writer.claimNextRun(input)), - enqueueManualRun: (input: EnqueueManualRunInput, beforeInsert?: () => void) => - write((writer) => writer.enqueueManualRun(input, beforeInsert)), + enqueueManualRun: ( + input: EnqueueManualRunInput, + authorizeAdmittedEnqueue?: () => void + ) => write((writer) => writer.enqueueManualRun(input), authorizeAdmittedEnqueue), enqueueNextDueSchedule: (input: DueScheduleEnqueueInput) => write((writer) => writer.enqueueNextDueSchedule(input)), expireDisableIntents: (input: ExpireDisableIntentsInput) => diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts index a9399f6d0..dd4c3501f 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts @@ -4,9 +4,22 @@ import { type ServiceActionId, serviceActionIds, } from "../../../contracts/serviceActions.ts"; +import { + securityUserId, + sessionSelector, + validAuthSessionInsert, + validUserInsert, +} from "../../database/validation/testSupport/securityRows.ts"; +import { testImmediateDatabaseWriteAdmission } from "../../test/support/databaseWriteAdmission.ts"; +import { openFreshMigratedDatabase } from "../../test/support/freshDatabase.ts"; +import { createAuthenticationLifecycleRepository } from "../security/authenticationLifecycleRepository.ts"; import type { JobUnscheduledActionDefinition } from "./actionRegistry.ts"; import type { JobRunRecord } from "./records.ts"; -import type { EnqueueManualRunInput, EnqueueManualRunResult } from "./repository.ts"; +import { + createJobRepository, + type EnqueueManualRunInput, + type EnqueueManualRunResult, +} from "./repository.ts"; import { createServiceActionQueue, serviceActionJobActionKeys, @@ -49,8 +62,11 @@ function repositoryFixture() { const idempotencyReads: [JobRunRecord["requestedByKind"], string, string][] = []; let stored: JobRunRecord | undefined; const repository: ServiceActionQueueDependencies["repository"] = { - enqueueManualRun(input, beforeInsert): Promise { - beforeInsert?.(); + enqueueManualRun( + input, + authorizeAdmittedEnqueue + ): Promise { + authorizeAdmittedEnqueue?.(); enqueues.push(input); stored = { ...input.run, @@ -147,6 +163,7 @@ describe("Service Action durable queue", () => { expect(authorizationChecks).toBe(1); expect(wakeCalls).toEqual([actionId]); expect(fixture.enqueues).toHaveLength(1); + expect(fixture.enqueues[0]?.rejectWhenActionActive).toBe(true); expect(fixture.enqueues[0]?.run).toMatchObject({ actionKey: serviceActionJobActionKeys[actionId], attemptLimit: 1, @@ -156,6 +173,7 @@ describe("Service Action durable queue", () => { requestedById: actor.id, requestedByKind: "user", resourceClass: "exclusive", + resourceKeysJson: JSON.stringify(definitions[actionId].resourceKeys), retrySafe: false, triggerType: "manual", }); @@ -319,6 +337,78 @@ describe("Service Action durable queue", () => { expect(fixture.run()).toBeUndefined(); }); + test("runs the final session read after admission but before the SQLite write transaction", async () => { + const database = await openFreshMigratedDatabase(); + const authenticationRepository = createAuthenticationLifecycleRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const jobRepository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const runIds = [ + "019fdf50-0000-7000-8000-000000000030", + "019fdf50-0000-7000-8000-000000000031", + "019fdf50-0000-7000-8000-000000000032", + "019fdf50-0000-7000-8000-000000000033", + ]; + const queue = createServiceActionQueue({ + definitions, + generateId: () => runIds.shift()!, + nowMs: () => 1000, + repository: jobRepository, + }); + const authorizationFailure = new Error("authorization expired"); + const authenticatedActor = Object.freeze({ + authenticatorId: sessionSelector, + id: securityUserId, + kind: "user" as const, + }); + const authorizeDispatch = () => + Promise.resolve(() => { + const session = authenticationRepository.withReadTransaction((reader) => + reader.findSession(securityUserId, sessionSelector) + ); + if (session === undefined) throw authorizationFailure; + }); + + try { + await authenticationRepository.withImmediateTransaction((unit) => { + unit.insertUser(validUserInsert); + unit.insertSession(validAuthSessionInsert); + }); + + const accepted = await queue.enqueue( + request("system-update", { + actor: authenticatedActor, + authorizeDispatch, + idempotencyKey: "019fdf50-0000-4000-8000-000000000040", + }) + ); + expect(jobRepository.findRun(accepted.jobRunId)).toBeDefined(); + + await authenticationRepository.withImmediateTransaction((unit) => { + expect(unit.deleteSession(securityUserId, sessionSelector)).toBe(true); + }); + const rejectedRunId = "019fdf50-0000-7000-8000-000000000032"; + const failure = await queue + .enqueue( + request("system-update", { + actor: authenticatedActor, + authorizeDispatch, + idempotencyKey: "019fdf50-0000-4000-8000-000000000041", + }) + ) + .catch((error: unknown) => error); + + expect(failure).toBe(authorizationFailure); + expect(jobRepository.findRun(rejectedRunId)).toBeUndefined(); + } finally { + database.sqlite.close(true); + } + }); + test("rejects unsafe injected action mappings at composition", () => { expect(() => createServiceActionQueue({ @@ -332,5 +422,24 @@ describe("Service Action durable queue", () => { repository: repositoryFixture().repository, }) ).toThrow("Service Action definition is invalid"); + + for (const unsafePolicy of [ + { attemptLimit: 2 }, + { cancellationPolicy: "queued-only" as const }, + { retrySafe: true }, + ]) { + expect(() => + createServiceActionQueue({ + definitions: { + ...definitions, + "system-update": { + ...definitions["system-update"], + ...unsafePolicy, + }, + }, + repository: repositoryFixture().repository, + }) + ).toThrow("Service Action definition is invalid"); + } }); }); diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.ts index d97dd7ff2..cb7bcafc4 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.ts @@ -107,7 +107,10 @@ function prepareDefinitions( ); if ( definition.actionKey !== serviceActionJobActionKeys[actionId] || - definition.manualExposure !== "none" + definition.manualExposure !== "none" || + definition.attemptLimit !== 1 || + definition.cancellationPolicy !== "never" || + definition.retrySafe ) { throw new TypeError("Service Action definition is invalid"); } diff --git a/greenfield/src/server/domains/serviceActions/service.test.ts b/greenfield/src/server/domains/serviceActions/service.test.ts index 5ff185f93..4c6716c2b 100644 --- a/greenfield/src/server/domains/serviceActions/service.test.ts +++ b/greenfield/src/server/domains/serviceActions/service.test.ts @@ -346,6 +346,30 @@ describe("service actions service", () => { expect(JSON.stringify(failure)).not.toContain("systemctl"); }); + test("preserves a classified service failure raised during dispatch preflight", async () => { + const classifiedFailure = new ServiceActionsServiceError("unknown-outcome"); + const state = fixture({ + queue: { + enqueue: async (request) => { + await request.authorizeDispatch(); + throw new Error("authorizeDispatch should have rejected"); + }, + }, + statuses: { + read: () => Promise.reject(classifiedFailure), + }, + }); + + const failure = await captureFailure(() => + state.service.request(input, state.context) + ); + expect(failure).toBe(classifiedFailure); + expect(state.auditEvents.map(({ settlement }) => settlement)).toEqual([ + "attempted", + "partial", + ]); + }); + test("does not replace a confirmed queued result when settlement audit fails", async () => { const state = fixture({ auditFailure: "succeeded" }); expect(await state.service.request(input, state.context)).toMatchObject({ diff --git a/greenfield/src/server/domains/serviceActions/service.ts b/greenfield/src/server/domains/serviceActions/service.ts index 6889f62b0..5f5083377 100644 --- a/greenfield/src/server/domains/serviceActions/service.ts +++ b/greenfield/src/server/domains/serviceActions/service.ts @@ -279,7 +279,9 @@ export function createServiceActionsService( throw authorizationFailure; } let mapped: unknown; - if (error instanceof ServiceActionQueueError) { + if (error instanceof ServiceActionsServiceError) { + mapped = error; + } else if (error instanceof ServiceActionQueueError) { mapped = queueFailure(error); } else if (error instanceof v.ValiError) { mapped = new ServiceActionsServiceError("unknown-outcome", { diff --git a/greenfield/src/server/domains/serviceActions/statusReader.test.ts b/greenfield/src/server/domains/serviceActions/statusReader.test.ts index 5f533e566..cf485fb41 100644 --- a/greenfield/src/server/domains/serviceActions/statusReader.test.ts +++ b/greenfield/src/server/domains/serviceActions/statusReader.test.ts @@ -118,13 +118,19 @@ describe("Service Action status reader", () => { test("rejects an already-aborted read before persistence", async () => { const controller = new AbortController(); controller.abort(new Error("request closed")); - let called = false; + const repositoryReads = { + actionSnapshots: 0, + workerAvailability: 0, + }; const reader = createSqliteServiceActionStatusReader({ expectedReleaseId, repository: { - readActionPayloadRunSnapshots: () => [], + readActionPayloadRunSnapshots: () => { + repositoryReads.actionSnapshots += 1; + return []; + }, readWorkerActionAvailability: () => { - called = true; + repositoryReads.workerAvailability += 1; return []; }, }, @@ -138,6 +144,9 @@ describe("Service Action status reader", () => { } expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toBe("request closed"); - expect(called).toBeFalse(); + expect(repositoryReads).toEqual({ + actionSnapshots: 0, + workerAvailability: 0, + }); }); }); diff --git a/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts index 21086888f..a7aad0a4a 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayOpenClawServiceActionsProvider.test.ts @@ -154,7 +154,15 @@ describe("persistent Gateway OpenClaw Service Actions provider", () => { message: "OpenClaw Service Action failed", reason: "unknown-outcome", }); - expect(JSON.stringify(unknownFailure)).not.toContain("systemctl"); + const renderedUnknownFailure = + unknownFailure instanceof Error + ? [ + unknownFailure.name, + unknownFailure.message, + unknownFailure.stack ?? "", + ].join("\n") + : String(unknownFailure); + expect(renderedUnknownFailure).not.toContain("systemctl"); expect(attempts).toBe(2); }); diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts index 774ca866b..ff36dbded 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts @@ -589,11 +589,27 @@ describe("persistent Gateway protocol-v4 boundary", () => { expect( parsePersistentGatewayOpenClawServiceActionResponse("update.run", { handoff: { status: "started" }, - ok: true, + ok: false, restart: { pid: 43 }, result: { status: "error" }, sentinel: {}, }) + ).toEqual({ method: "update.run", status: "accepted" }); + expect( + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + ok: true, + restart: null, + result: { status: "ok" }, + sentinel: {}, + }) + ).toEqual({ method: "update.run", status: "completed" }); + expect( + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + ok: false, + restart: null, + result: { status: "error" }, + sentinel: {}, + }) ).toEqual({ method: "update.run", status: "failed" }); expect(() => parsePersistentGatewayOpenClawServiceActionResponse("update.run", { diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts index 707eeb71e..9ab79c8a8 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts @@ -1501,14 +1501,10 @@ export function parsePersistentGatewayOpenClawServiceActionResponse( throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); } let status: "accepted" | "completed" | "failed" = "failed"; - if (parsed.output.ok && parsed.output.result.status === "ok") { - status = "completed"; - } else if ( - parsed.output.ok && - parsed.output.result.status === "skipped" && - parsed.output.handoff?.status === "started" - ) { + if (parsed.output.handoff?.status === "started") { status = "accepted"; + } else if (parsed.output.ok && parsed.output.result.status === "ok") { + status = "completed"; } return Object.freeze({ ...(parsed.output.result.after === undefined diff --git a/greenfield/src/server/platform/observability/structuredLogger.test.ts b/greenfield/src/server/platform/observability/structuredLogger.test.ts index 243105788..90a180a0c 100644 --- a/greenfield/src/server/platform/observability/structuredLogger.test.ts +++ b/greenfield/src/server/platform/observability/structuredLogger.test.ts @@ -1,5 +1,6 @@ import { expect, spyOn, test } from "bun:test"; +import { serviceActionIds } from "../../../contracts/serviceActions.ts"; import { createStructuredLogger, type StructuredLogSink } from "./structuredLogger.ts"; const identity = Object.freeze({ @@ -372,6 +373,37 @@ test("records only fixed Service Actions audit settlement fields", () => { }); expect(JSON.parse(lines[0] ?? "null")).not.toHaveProperty("fields.kind"); expect(lines[0]).not.toContain("private provider detail"); + + for (const actionId of serviceActionIds) { + logger.error({ + component: "service-actions-audit", + event: "service_actions.audit_settlement.failed", + fields: { + actionId, + kind: "service-actions-audit-settlement", + settlement: "failed", + }, + }); + } + logger.error({ + component: "service-actions-audit", + event: "service_actions.audit_settlement.failed", + fields: { + actionId: "unreviewed-action" as never, + kind: "service-actions-audit-settlement", + settlement: "failed", + }, + }); + + expect( + lines + .slice(1, 1 + serviceActionIds.length) + .map( + (line) => + (JSON.parse(line) as { fields: { actionId: string } }).fields.actionId + ) + ).toEqual(serviceActionIds); + expect(JSON.parse(lines.at(-1) ?? "null")).not.toHaveProperty("fields"); }); test("normalizes unknown events and drops extra fields instead of relying on secret names", () => { diff --git a/greenfield/src/server/platform/observability/structuredLogger.ts b/greenfield/src/server/platform/observability/structuredLogger.ts index e0a79a429..3886a6cd1 100644 --- a/greenfield/src/server/platform/observability/structuredLogger.ts +++ b/greenfield/src/server/platform/observability/structuredLogger.ts @@ -2,7 +2,10 @@ import { logMaintenancePolicyIds, type LogMaintenancePolicyId, } from "../../../contracts/logs.ts"; -import type { ServiceActionId } from "../../../contracts/serviceActions.ts"; +import { + type ServiceActionId, + serviceActionIds, +} from "../../../contracts/serviceActions.ts"; import type { SafeFailureDescriptor } from "../errors/safeFailure.ts"; import { describeSafeFailure } from "../errors/safeFailure.ts"; @@ -23,6 +26,14 @@ const defaultStructuredLogLimits: StructuredLogLimits = Object.freeze({ maximumSerializedBytes: 16 * 1024, }); const structuredLogEncoder = new TextEncoder(); +const serviceActionIdInventory: ReadonlySet = new Set(serviceActionIds); + +function isServiceActionId(value: unknown): value is ServiceActionId { + return ( + typeof value === "string" && + serviceActionIdInventory.has(value as ServiceActionId) + ); +} export type StructuredLogLevel = "debug" | "error" | "fatal" | "info" | "warn"; @@ -387,10 +398,7 @@ function safeEventFields( case "service-actions-audit-settlement": { if ( eventName !== "service_actions.audit_settlement.failed" || - (fields.actionId !== "openclaw-cleanup" && - fields.actionId !== "openclaw-update" && - fields.actionId !== "system-restart" && - fields.actionId !== "system-update") || + !isServiceActionId(fields.actionId) || (fields.settlement !== "failed" && fields.settlement !== "partial" && fields.settlement !== "succeeded") diff --git a/greenfield/src/shared/databaseMigrationManifest.ts b/greenfield/src/shared/databaseMigrationManifest.ts index dd081b3da..dbf125cef 100644 --- a/greenfield/src/shared/databaseMigrationManifest.ts +++ b/greenfield/src/shared/databaseMigrationManifest.ts @@ -13,8 +13,8 @@ export const migrationManifest = Object.freeze { } }); + test("rejects cleanup disk-budget enforcement before lifecycle mutation", async () => { + await withTemporaryDirectory( + "mira-openclaw-cleanup-order-drift-", + async (sourceRoot) => { + await writeSyntheticOpenClawPackage(sourceRoot); + const artifactPath = path.join( + sourceRoot, + "dist", + "cleanup-service-fixture.js" + ); + const source = await readFile(artifactPath, "utf8"); + const lifecycleCall = + "const lifecycleResult = await applySqliteSessionEntryLifecycleMutation({"; + const diskBudgetCall = + "const appliedDiskBudget = await enforceSqliteSessionHistoryDiskBudget({"; + const lifecycleIndex = source.indexOf(lifecycleCall); + const diskBudgetIndex = source.indexOf(diskBudgetCall); + expect(lifecycleIndex).toBeGreaterThanOrEqual(0); + expect(diskBudgetIndex).toBeGreaterThan(lifecycleIndex); + const reordered = + source.slice(0, lifecycleIndex) + + diskBudgetCall + + source.slice(lifecycleIndex + lifecycleCall.length, diskBudgetIndex) + + lifecycleCall + + source.slice(diskBudgetIndex + diskBudgetCall.length); + await writeFile(artifactPath, reordered, "utf8"); + + const error = await rejectedError(auditInstalledOpenClaw(sourceRoot)); + expect(error.message).toContain( + "sessions.cleanup no longer applies lifecycle mutation before disk budget enforcement" + ); + } + ); + }); + test("rejects drift in the system.info read permission", async () => { await withTemporaryDirectory( "mira-openclaw-system-scope-", From eb71e93ba3f3d4e1450b1c4d2a892fd20297da99 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 14:56:55 +0200 Subject: [PATCH 08/13] fix(greenfield): close service action recovery review --- .../overview/OverviewServiceActionsCard.tsx | 8 ++++++-- .../OverviewServiceActionsSection.test.tsx | 20 +++++++++++++------ .../gateway/persistentGatewayProtocol.test.ts | 9 +++++++++ .../gateway/persistentGatewayProtocol.ts | 7 +++++-- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx index 604537a93..bfd5e48f1 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx @@ -68,7 +68,10 @@ function ServiceActionRow({ }: ServiceActionRowProps) { const presentation = serviceActionPresentations[action.id]; const active = action.activeRun !== undefined; - const disabled = action.availability === "unavailable" || active || globalBusy; + const disabled = + active || + globalBusy || + (action.availability === "unavailable" && !recoveryPending); return (
  • @@ -220,7 +223,8 @@ export function OverviewServiceActionsCard({ busy={requestBusy && requestActionId === selectedActionId} confirmDisabled={ selectedAction === undefined || - selectedAction.availability === "unavailable" || + (selectedAction.availability === "unavailable" && + !selectedRecoveryPending) || selectedAction.activeRun !== undefined } confirmLabel={ diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx index 459b2ed45..8020549ab 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx @@ -135,6 +135,15 @@ const allAvailableStatus = Object.freeze({ observedAtMs: timestampMs + 3000, } satisfies GetServiceActionsStatusResult); +const unavailableCleanupStatus = Object.freeze({ + actions: allAvailableStatus.actions.map((action) => + action.id === "openclaw-cleanup" + ? { ...action, availability: "unavailable" as const } + : action + ), + observedAtMs: timestampMs + 4000, +} satisfies GetServiceActionsStatusResult); + const queuedResult = Object.freeze({ actionId: "openclaw-cleanup", jobRunId: queuedRun.id, @@ -441,14 +450,13 @@ describe("OverviewServiceActionsSection", () => { const firstIndex = harnesses.indexOf(first); if (firstIndex !== -1) harnesses.splice(firstIndex, 1); const second = renderSection( - [allAvailableStatus, allAvailableStatus], + [unavailableCleanupStatus, unavailableCleanupStatus], [queuedResult] ); - expect( - await screen.findByRole("button", { - name: "Retry OpenClaw cleanup request", - }) - ).toBeTruthy(); + const recoveryButton = await screen.findByRole("button", { + name: "Retry OpenClaw cleanup request", + }); + expect(recoveryButton).toBeEnabled(); await user.click( screen.getByRole("button", { name: "Retry OpenClaw cleanup request" }) ); diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts index ff36dbded..d72e4a932 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts @@ -595,6 +595,15 @@ describe("persistent Gateway protocol-v4 boundary", () => { sentinel: {}, }) ).toEqual({ method: "update.run", status: "accepted" }); + expect( + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + handoff: { status: "joined" }, + ok: false, + restart: null, + result: { status: "skipped" }, + sentinel: {}, + }) + ).toEqual({ method: "update.run", status: "accepted" }); expect( parsePersistentGatewayOpenClawServiceActionResponse("update.run", { ok: true, diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts index 9ab79c8a8..05d784ae8 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts @@ -738,7 +738,7 @@ const gatewayOpenClawUpdateResultSchema = v.object({ status: v.picklist(["error", "ok", "skipped"]), }); const gatewayOpenClawUpdateHandoffSchema = v.object({ - status: v.picklist(["already-running", "started", "unavailable"]), + status: v.picklist(["already-running", "joined", "started", "unavailable"]), }); const gatewayOpenClawUpdateResponseSchema = v.strictObject({ handoff: v.optional(gatewayOpenClawUpdateHandoffSchema), @@ -1501,7 +1501,10 @@ export function parsePersistentGatewayOpenClawServiceActionResponse( throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); } let status: "accepted" | "completed" | "failed" = "failed"; - if (parsed.output.handoff?.status === "started") { + if ( + parsed.output.handoff?.status === "started" || + parsed.output.handoff?.status === "joined" + ) { status = "accepted"; } else if (parsed.output.ok && parsed.output.result.status === "ok") { status = "completed"; From 76f61b1bf549430c68772a9bea4e35313b9c0ece Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 15:10:42 +0200 Subject: [PATCH 09/13] fix(greenfield): keep update handoff wire strict --- .../platform/gateway/persistentGatewayProtocol.test.ts | 4 ++-- .../server/platform/gateway/persistentGatewayProtocol.ts | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts index d72e4a932..e40a2421c 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts @@ -597,13 +597,13 @@ describe("persistent Gateway protocol-v4 boundary", () => { ).toEqual({ method: "update.run", status: "accepted" }); expect( parsePersistentGatewayOpenClawServiceActionResponse("update.run", { - handoff: { status: "joined" }, + handoff: { status: "already-running" }, ok: false, restart: null, result: { status: "skipped" }, sentinel: {}, }) - ).toEqual({ method: "update.run", status: "accepted" }); + ).toEqual({ method: "update.run", status: "failed" }); expect( parsePersistentGatewayOpenClawServiceActionResponse("update.run", { ok: true, diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts index 05d784ae8..9ab79c8a8 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.ts @@ -738,7 +738,7 @@ const gatewayOpenClawUpdateResultSchema = v.object({ status: v.picklist(["error", "ok", "skipped"]), }); const gatewayOpenClawUpdateHandoffSchema = v.object({ - status: v.picklist(["already-running", "joined", "started", "unavailable"]), + status: v.picklist(["already-running", "started", "unavailable"]), }); const gatewayOpenClawUpdateResponseSchema = v.strictObject({ handoff: v.optional(gatewayOpenClawUpdateHandoffSchema), @@ -1501,10 +1501,7 @@ export function parsePersistentGatewayOpenClawServiceActionResponse( throw new TypeError("Persistent Gateway OpenClaw operation response is invalid"); } let status: "accepted" | "completed" | "failed" = "failed"; - if ( - parsed.output.handoff?.status === "started" || - parsed.output.handoff?.status === "joined" - ) { + if (parsed.output.handoff?.status === "started") { status = "accepted"; } else if (parsed.output.ok && parsed.output.result.status === "ok") { status = "completed"; From 41129fce48454bfc6061b1ea1712c6fb0042b01b Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 15:45:41 +0200 Subject: [PATCH 10/13] fix(greenfield): preserve service action recovery semantics --- .../openclaw/fixtures/2026.7.2-beta.7/manifest.json | 2 +- .../openclaw/fixtures/2026.7.2-beta.7/operations.json | 2 ++ greenfield/scripts/audits/openclaw/sourceAudit.ts | 5 +++++ .../scripts/audits/openclaw/sourceAuditSchemas.ts | 2 ++ .../browser/overview/OverviewServiceActionsCard.tsx | 4 ++-- .../overview/OverviewServiceActionsSection.test.tsx | 10 +++++++--- .../platform/gateway/persistentGatewayProtocol.test.ts | 9 +++++++++ .../src/test/integration/openclaw/sourceAudit.test.ts | 2 ++ 8 files changed, 30 insertions(+), 6 deletions(-) diff --git a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json index 3cf27b03c..14c0a473c 100644 --- a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json +++ b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/manifest.json @@ -18,7 +18,7 @@ }, { "file": "operations.json", - "sha256": "bfb91bffba35131f3bdcde1900e6a33904ee9188ac32318b28aa78bc07ed69c7" + "sha256": "e7db22e0c9ab48c08da94cad87c3a2b8c8b59c9457a11b12486c5c41c71bb6e5" }, { "file": "sessions.json", diff --git a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json index 8532f1911..c968edabe 100644 --- a/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json +++ b/greenfield/scripts/audits/openclaw/fixtures/2026.7.2-beta.7/operations.json @@ -107,6 +107,8 @@ "detachedChild": true, "gitRequiresSupervisor": true, "globalInstallRequiresHandoff": true, + "internalJoinedStatusCrossesRpc": false, + "nonOwningWireStatus": "already-running", "readyMarkerTimeoutMs": 30000, "sensitiveTemporaryFilesRemoved": true, "startedHandoffCountsAsAccepted": true, diff --git a/greenfield/scripts/audits/openclaw/sourceAudit.ts b/greenfield/scripts/audits/openclaw/sourceAudit.ts index 2de666e84..e103e9520 100644 --- a/greenfield/scripts/audits/openclaw/sourceAudit.ts +++ b/greenfield/scripts/audits/openclaw/sourceAudit.ts @@ -3293,8 +3293,11 @@ function assertOpenClawOperationsSemantics( "const hasHandoffContext = supervisor ? hasManagedServiceHandoffContext(process.env, supervisor) : false", "const started = await startManagedServiceUpdateHandoff({", 'ownsManagedServiceHandoff = started.status === "started"', + "if (ownsManagedServiceHandoff) {", "...started.pid ? { pid: started.pid } : {}", "command: started.command", + "} else handoff = {", + 'status: "already-running"', 'message: "Another managed update is already running; retry after it completes."', "managedHandoffRestart = scheduleGatewaySigusr1Restart({", 'reason: "update.run"', @@ -3536,6 +3539,8 @@ function assertOpenClawOperationsSemantics( detachedChild: true, gitRequiresSupervisor: true, globalInstallRequiresHandoff: true, + internalJoinedStatusCrossesRpc: false, + nonOwningWireStatus: "already-running", readyMarkerTimeoutMs: 30_000, sensitiveTemporaryFilesRemoved: true, startedHandoffCountsAsAccepted: true, diff --git a/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts b/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts index 79ef6e8e0..a06e8152d 100644 --- a/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts +++ b/greenfield/scripts/audits/openclaw/sourceAuditSchemas.ts @@ -1022,6 +1022,8 @@ export const operationsFixtureSchema = v.strictObject({ detachedChild: v.literal(true), gitRequiresSupervisor: v.literal(true), globalInstallRequiresHandoff: v.literal(true), + internalJoinedStatusCrossesRpc: v.literal(false), + nonOwningWireStatus: v.literal("already-running"), readyMarkerTimeoutMs: v.literal(30_000), sensitiveTemporaryFilesRemoved: v.literal(true), startedHandoffCountsAsAccepted: v.literal(true), diff --git a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx index bfd5e48f1..b93409c2d 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx @@ -69,7 +69,7 @@ function ServiceActionRow({ const presentation = serviceActionPresentations[action.id]; const active = action.activeRun !== undefined; const disabled = - active || + (active && !recoveryPending) || globalBusy || (action.availability === "unavailable" && !recoveryPending); return ( @@ -225,7 +225,7 @@ export function OverviewServiceActionsCard({ selectedAction === undefined || (selectedAction.availability === "unavailable" && !selectedRecoveryPending) || - selectedAction.activeRun !== undefined + (selectedAction.activeRun !== undefined && !selectedRecoveryPending) } confirmLabel={ selectedRecoveryPending diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx index 8020549ab..71cc59dd1 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.test.tsx @@ -135,10 +135,14 @@ const allAvailableStatus = Object.freeze({ observedAtMs: timestampMs + 3000, } satisfies GetServiceActionsStatusResult); -const unavailableCleanupStatus = Object.freeze({ +const unavailableActiveCleanupStatus = Object.freeze({ actions: allAvailableStatus.actions.map((action) => action.id === "openclaw-cleanup" - ? { ...action, availability: "unavailable" as const } + ? { + ...action, + activeRun: queuedRun, + availability: "unavailable" as const, + } : action ), observedAtMs: timestampMs + 4000, @@ -450,7 +454,7 @@ describe("OverviewServiceActionsSection", () => { const firstIndex = harnesses.indexOf(first); if (firstIndex !== -1) harnesses.splice(firstIndex, 1); const second = renderSection( - [unavailableCleanupStatus, unavailableCleanupStatus], + [unavailableActiveCleanupStatus, unavailableActiveCleanupStatus], [queuedResult] ); const recoveryButton = await screen.findByRole("button", { diff --git a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts index e40a2421c..95a0a8203 100644 --- a/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts +++ b/greenfield/src/server/platform/gateway/persistentGatewayProtocol.test.ts @@ -604,6 +604,15 @@ describe("persistent Gateway protocol-v4 boundary", () => { sentinel: {}, }) ).toEqual({ method: "update.run", status: "failed" }); + expect(() => + parsePersistentGatewayOpenClawServiceActionResponse("update.run", { + handoff: { status: "joined" }, + ok: false, + restart: null, + result: { status: "skipped" }, + sentinel: {}, + }) + ).toThrow(TypeError); expect( parsePersistentGatewayOpenClawServiceActionResponse("update.run", { ok: true, diff --git a/greenfield/src/test/integration/openclaw/sourceAudit.test.ts b/greenfield/src/test/integration/openclaw/sourceAudit.test.ts index d8908d215..d2e256957 100644 --- a/greenfield/src/test/integration/openclaw/sourceAudit.test.ts +++ b/greenfield/src/test/integration/openclaw/sourceAudit.test.ts @@ -3861,6 +3861,8 @@ describe("explicit OpenClaw source audit", () => { }); expect(audit.operations.updateRun).toMatchObject({ managedHandoff: { + internalJoinedStatusCrossesRpc: false, + nonOwningWireStatus: "already-running", readyMarkerTimeoutMs: 30_000, sensitiveTemporaryFilesRemoved: true, startedHandoffCountsAsAccepted: true, From 9e3ab16254a9bd73a71f70d9baf4c407ad7d996e Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 18:53:17 +0200 Subject: [PATCH 11/13] feat(greenfield): complete fixed service actions --- .../application-architecture.md | 98 +-- .../greenfield-rewrite/data-and-security.md | 123 +-- .../greenfield-rewrite/implementation-plan.md | 19 +- .../greenfield-rewrite/progress.md | 66 +- .../runtime-and-delivery.md | 74 +- ...erviceActions.getStatus.output.schema.json | 6 +- .../serviceActions.request.input.schema.json | 46 ++ .../serviceActions.request.output.schema.json | 2 + .../migration.sql | 47 ++ .../snapshot.json | 175 +++++ .../scripts/delivery/buildRelease.test.ts | 5 + .../hostOperationsProvisioningPolicy.test.ts | 106 +++ .../hostOperationsProvisioningPolicy.ts | 7 + .../installHostOperationsProvisioning.test.ts | 524 +++++++++++++ .../productionReleasePublication.test.ts | 5 + .../60-mira-dashboard-host-operations.rules | 19 + .../provisioning/host-operations/README.md | 68 ++ .../hostOperationsProvisioningFilesystem.ts | 702 ++++++++++++++++++ .../installHostOperationsProvisioning.ts | 699 +++++++++++++++++ .../mira-dashboard-deferred-reboot.service | 23 + .../mira-dashboard-deferred-reboot.timer | 7 + .../mira-dashboard-host-operation | 50 ++ ...mira-dashboard-host-system-cleanup.service | 14 + ...mira-dashboard-host-system-restart.service | 30 + .../mira-dashboard-host-system-update.service | 15 + .../provisioning/host-operations/policy.ts | 69 ++ .../scripts/delivery/releaseIdentity.test.ts | 15 + .../scripts/delivery/releaseIdentity.ts | 8 +- greenfield/scripts/delivery/releaseStaging.ts | 7 + .../scripts/documentation/jsonSchema.test.ts | 4 +- .../scripts/documentation/jsonSchema.ts | 2 +- .../scripts/sourceBoundaries/policy.test.ts | 2 +- .../testSupport/productionDeliveryFixture.ts | 5 + greenfield/src/app/dashboardServer.test.ts | 1 + greenfield/src/app/dashboardServer.ts | 7 + greenfield/src/app/developmentWorker.ts | 4 +- greenfield/src/app/worker.test.ts | 8 +- greenfield/src/app/worker.ts | 15 +- .../src/browser/jobs/JobsRoute.test.tsx | 132 +++- greenfield/src/browser/jobs/JobsRoute.tsx | 4 +- .../browser/overview/OverviewRoute.test.tsx | 2 + .../overview/OverviewServiceActionsCard.tsx | 29 +- .../OverviewServiceActionsSection.test.tsx | 40 +- .../OverviewServiceActionsSection.tsx | 9 +- .../overview/serviceActionsOperations.test.ts | 30 +- .../overview/serviceActionsOperations.ts | 36 + .../src/contracts/serviceActions.test.ts | 11 +- greenfield/src/contracts/serviceActions.ts | 18 + .../database/migrations/jobsSchema.test.ts | 31 +- .../migrations/migrationGraph.test.ts | 4 + .../src/server/database/schema/checks.ts | 9 + .../server/database/schema/drizzleSchema.ts | 1 + .../database/schema/hostRestartClaimFence.ts | 49 ++ .../src/server/database/schema/jobRuns.ts | 10 + .../validation/hostRestartClaimFence.ts | 46 ++ .../src/server/database/validation/jobRuns.ts | 26 +- .../database/validation/rowSchemas.test.ts | 1 + .../server/domains/cache/repository.test.ts | 2 + .../server/domains/files/jobScheduler.test.ts | 1 + .../domains/jobs/actionExecutors.test.ts | 69 +- .../server/domains/jobs/actionExecutors.ts | 37 +- .../domains/jobs/actionRegistry.test.ts | 16 +- .../src/server/domains/jobs/actionRegistry.ts | 22 +- .../server/domains/jobs/coordinator.test.ts | 8 + .../src/server/domains/jobs/coordinator.ts | 49 ++ .../domains/jobs/logMaintenanceQueue.test.ts | 1 + greenfield/src/server/domains/jobs/records.ts | 4 + .../server/domains/jobs/repository.test.ts | 442 +++++++++++ .../src/server/domains/jobs/repository.ts | 219 +++++- .../src/server/domains/jobs/service.test.ts | 1 + .../domains/jobs/serviceActionQueue.test.ts | 29 + .../server/domains/jobs/serviceActionQueue.ts | 21 +- .../server/domains/jobs/workerRuntime.test.ts | 22 +- .../src/server/domains/jobs/workerRuntime.ts | 7 + .../server/domains/jobs/workerSystem.test.ts | 2 + .../openClawSettings/restartQueue.test.ts | 4 + .../domains/serviceActions/procedures.test.ts | 2 + .../domains/serviceActions/service.test.ts | 8 + .../serviceActions/statusReader.test.ts | 6 + .../src/shared/databaseMigrationManifest.ts | 4 +- greenfield/src/shared/hostOperations.ts | 17 + greenfield/src/shared/linuxBootIdentity.ts | 14 + .../parity/fixtures/legacy-endpoints.json | 4 +- .../src/test/parity/parityInventory.test.ts | 16 +- .../system/fixedHostOperationsBroker.test.ts | 184 +++++ .../system/fixedHostOperationsBroker.ts | 209 ++++++ .../src/worker/system/linuxBootIdentity.ts | 27 + .../systemHostOperationsProvisioning.test.ts | 134 ++++ 88 files changed, 4912 insertions(+), 234 deletions(-) create mode 100644 greenfield/scripts/delivery/hostOperationsProvisioningPolicy.test.ts create mode 100644 greenfield/scripts/delivery/hostOperationsProvisioningPolicy.ts create mode 100644 greenfield/scripts/delivery/installHostOperationsProvisioning.test.ts create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/README.md create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service create mode 100644 greenfield/scripts/delivery/provisioning/host-operations/policy.ts create mode 100644 greenfield/src/server/database/schema/hostRestartClaimFence.ts create mode 100644 greenfield/src/server/database/validation/hostRestartClaimFence.ts create mode 100644 greenfield/src/shared/hostOperations.ts create mode 100644 greenfield/src/shared/linuxBootIdentity.ts create mode 100644 greenfield/src/worker/system/fixedHostOperationsBroker.test.ts create mode 100644 greenfield/src/worker/system/fixedHostOperationsBroker.ts create mode 100644 greenfield/src/worker/system/linuxBootIdentity.ts create mode 100644 greenfield/src/worker/system/systemHostOperationsProvisioning.test.ts diff --git a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md index 66498227c..911fd09ee 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md +++ b/greenfield/docs/architecture/greenfield-rewrite/application-architecture.md @@ -464,41 +464,51 @@ is exclusive, caller-idempotent, single-attempt, non-retry-safe, and non-cancell worker owns its fixed no-shell lifecycle command. Ambiguous enqueue or terminal settlement is reconciled by durable run identity and never blindly dispatches a second restart. -### Purpose-built Service Actions partially replace generic exec consumers +### Purpose-built Service Actions replace consumed generic exec behavior -The Overview exposes exactly four fixed Service Actions through -`serviceActions.getStatus` and `serviceActions.request`: OpenClaw session cleanup, OpenClaw -installation update, host restart, and host update. The browser submits only a fixed action ID and -a caller-owned idempotency key. The web process commits a sanitized attempt audit, checks a fresh -exact-release worker advertisement, and revalidates the browser session plus recent MFA at the -durable enqueue handoff. It returns a job-run ID rather than waiting for a privileged effect and -links all progress and terminal state to the existing Jobs surface. +The Overview exposes exactly six fixed Service Actions through +`serviceActions.getStatus` and `serviceActions.request`: OpenClaw session cleanup, OpenClaw Gateway +restart, OpenClaw installation update, bounded host cleanup, host restart, and host update. The browser submits only +a fixed action ID and a caller-owned idempotency key. The web process commits a sanitized attempt +audit, checks a fresh exact-release worker advertisement, and revalidates the browser session plus +recent MFA at the durable enqueue handoff. It returns a job-run ID rather than waiting for a +privileged effect and links all progress and terminal state to the existing Jobs surface. OpenClaw cleanup and update are implemented worker-only through the hash-pinned `sessions.cleanup` and `update.run` Gateway methods. Their providers accept no browser parameters, persist only bounded schema-validated summaries, never return raw Gateway results, and never blindly replay a post-dispatch unknown outcome. Cleanup deliberately uses OpenClaw's source-owned -session/artifact maintenance instead of reproducing legacy recursive deletion. These safe -replacements do not yet close `POST /api/exec/start`: the legacy broad `system_cleanup` behavior -crosses separate ownership domains and remains planned as three explicit capabilities. Docker -prune belongs to the Docker slice, apt cleanup belongs to host/package maintenance, and journald -vacuum belongs to log maintenance. The row stays open until all three are delivered or separately -reviewed without removing their operator-visible behavior. - -The contract and Overview retain fixed rows for host restart and host update, but production marks -both unavailable. The current web and worker processes share one Unix identity, so a group- or -shared-user polkit rule would also give a compromised web process the worker's root authority. No -such broker, polkit rule, root helper, or host-operation unit ships in this slice. Enabling either -host action later requires a distinct worker OS identity, a root-owned immutable worker boundary, -and separately reviewed provisioning and rollback before the worker may advertise the action key. +session/artifact maintenance instead of reproducing legacy recursive deletion. + +OpenClaw restart reuses the same existing `openclaw.gateway.restart` definition, executor, and +provider as Settings. Service Actions add only a second fixed admission surface and durable run +projection; they do not introduce another lifecycle command or remove the Settings control. + +System cleanup is one fixed root-brokered operation, not a caller-composed command. It attempts +package autoremove and cache cleanup, rotates journald and enforces both a 14-day and 1 GiB +retention bound, then prunes only unused Docker content older than 168 hours. It never passes +`--volumes`, continues through every fixed phase so partial cleanup is not silently skipped, and +fails the durable job if any phase fails. Its result contains only the validated terminal status. +The action owns `host.mutation` and `host.logs`, so it cannot overlap other privileged host or log +maintenance work. + +Host cleanup, restart, and update have a fixed `/usr/bin/systemctl` worker broker and exact +root-owned service units. No command, argument, path, environment, secret, or raw process output +crosses the port. The release ships manifest-verified provisioning and explicit rollback assets, +but production does not compose or install that authority while web and worker share one Unix +identity. A later separately approved topology change must give the worker a distinct OS principal, +keep the web principal outside its host-operation group, install and reload the reviewed assets, +and only then compose the broker. Until then all three rows remain fail-closed `unavailable`. The interactive PTY remains the sole terminal boundary. Shell `cd` and completion are owned by the connected shell/readline protocol, termination uses the bounded terminal session control, and no new generic command, cwd, or completion API is introduced. The unused synchronous `POST /api/exec` endpoint is a reviewed removal because no current browser or scoped automation consumer depends on it. Implemented long-running exec consumers map to either the PTY or the fixed durable Service -Actions queue, while the inventory keeps `POST /api/exec/start` planned for the outstanding -cleanup decomposition. +Actions queue. The fixed cleanup foundation and bounded PTY cover the consumed behavior without +restoring a generic command surface, but `POST /api/exec/start` remains planned until the distinct +worker topology and separately approved provisioning make `system-cleanup` executable in +production. ### Current-protocol Control UI projections @@ -933,27 +943,27 @@ suppress the next session's transport request. The rewrite may replace every component, store, hook, and API call, but it is incomplete until the following behavior is covered by automated tests and a manual parity checklist. -| Surface | Required behavior after rewrite | -| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Global shell | Authentication boundary, responsive navigation, theme/layout behavior, notification bell/modal, connection status, errors, and route recovery. | -| `/` | Health, agent, task, job, notification, Docker, Git, database, quota, weather, and operational overview cards retain cached values through transient refresh errors. | -| `/tasks` | Kanban/list behavior, create/edit/delete, status and assignee movement, labels, updates, automation configuration, full current search/filter semantics, and live deltas. | -| `/agents` | Agent state, metadata, current task, history, status transitions, and live updates. | -| `/sessions` | Gateway session listing, filtering, metadata, actions, refresh, and live state. | -| `/chat` | All streaming, thinking/tool display, cancel/retry/steer/concurrent send, history, attachment, settings, session, unread/follow/scroll, compaction, and restart/reconnect behavior described above. | -| `/logs` | Named-source selection, redacted bounded tail/search, custom reviewed app/container rotation, fixed system-logrotate host policies, and non-blocking errors. | -| `/jobs` | Dashboard schedules, OpenClaw cron jobs, enable/disable intent and expiry, run history, manual run/cancel, worker state, output, and aggregate counts. | -| `/reports` | Daily briefs, summaries, heartbeats, custom reports, filters, pagination/detail linking, Markdown display, cached refresh behavior, and incident links. | -| Notifications | Read/unread behavior, source links, filtering, badges, and exactly-once notification per active incident generation. | -| `/delivery` | PR review queues, trusted PR development, previews, release records, deploy/rollback actions, progress, blocking reasons, and retention. | -| `/files` | Safe workspace browsing, edit/save, upload/download/preview, Markdown/code rendering, path policy, and conflict/error handling. | -| `/docker` | Inventory, independently refreshed live stats, managed update policy, checks/actions, history, console commands, and duplicate-submit prevention. | -| `/database` | PostgreSQL/PgBouncer and Dashboard SQLite views, source picker, metrics, maintenance assessment/actions, cached fallback, and balanced layout. | -| `/moltbook` | Cached/API data, refresh behavior, status and error presentation, and existing actions. | -| `/settings` | Persistent OpenClaw/Dashboard tab, OpenClaw configuration, password, WebAuthn/passkeys, TOTP, recovery codes, browser sessions, secret handling, and restart actions. | -| `/terminal` | Real PTY input/output, ANSI/UTF-8, resize, signals, bounded reconnect replay, cancellation, backpressure, and narrow-screen interaction. The selected workspace root is a starting location, not a filesystem sandbox. | -| Media/STT/TTS | Existing upload constraints, MIME normalization, preview/download, transcription, speech generation, and scoped errors. | -| New `/docs` | Generated procedure, raw HTTP, realtime, database, configuration, runtime, package, and route references, searchable without exposing secrets. | +| Surface | Required behavior after rewrite | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Global shell | Authentication boundary, responsive navigation, theme/layout behavior, notification bell/modal, connection status, errors, and route recovery. | +| `/` | Health, agent, task, job, notification, Docker, Git, database, quota, weather, and operational overview cards retain cached values through transient refresh errors. | +| `/tasks` | Kanban/list behavior, create/edit/delete, status and assignee movement, labels, updates, automation configuration, full current search/filter semantics, and live deltas. | +| `/agents` | Agent state, metadata, current task, history, status transitions, and live updates. | +| `/sessions` | Gateway session listing, filtering, metadata, actions, refresh, and live state. | +| `/chat` | All streaming, thinking/tool display, cancel/retry/steer/concurrent send, history, attachment, settings, session, unread/follow/scroll, compaction, and restart/reconnect behavior described above. | +| `/logs` | Named-source selection, redacted bounded tail/search, custom reviewed app/container rotation, fixed system-logrotate host policies, and non-blocking errors. | +| `/jobs` | Dashboard schedules, OpenClaw cron jobs, the full six-item fixed Service Action inventory even before its first run, exact run-ID detail links, enable/disable intent and expiry, run history, manual run/cancel, worker state, output, and aggregate counts. | +| `/reports` | Daily briefs, summaries, heartbeats, custom reports, filters, pagination/detail linking, Markdown display, cached refresh behavior, and incident links. | +| Notifications | Read/unread behavior, source links, filtering, badges, and exactly-once notification per active incident generation. | +| `/delivery` | PR review queues, trusted PR development, previews, release records, deploy/rollback actions, progress, blocking reasons, and retention. | +| `/files` | Safe workspace browsing, edit/save, upload/download/preview, Markdown/code rendering, path policy, and conflict/error handling. | +| `/docker` | Inventory, independently refreshed live stats, managed update policy, checks/actions, history, console commands, and duplicate-submit prevention. | +| `/database` | PostgreSQL/PgBouncer and Dashboard SQLite views, source picker, metrics, maintenance assessment/actions, cached fallback, and balanced layout. | +| `/moltbook` | Cached/API data, refresh behavior, status and error presentation, and existing actions. | +| `/settings` | Persistent OpenClaw/Dashboard tab, OpenClaw configuration, password, WebAuthn/passkeys, TOTP, recovery codes, browser sessions, secret handling, and restart actions. | +| `/terminal` | Real PTY input/output, ANSI/UTF-8, resize, signals, bounded reconnect replay, cancellation, backpressure, and narrow-screen interaction. The selected workspace root is a starting location, not a filesystem sandbox. | +| Media/STT/TTS | Existing upload constraints, MIME normalization, preview/download, transcription, speech generation, and scoped errors. | +| New `/docs` | Generated procedure, raw HTTP, realtime, database, configuration, runtime, package, and route references, searchable without exposing secrets. | The existing API endpoint list is an input to the parity inventory, not a contract to preserve. Each old endpoint must map to a new procedure, a raw protocol route, or an explicit removal diff --git a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md index 61eb23c84..83c9623a8 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md +++ b/greenfield/docs/architecture/greenfield-rewrite/data-and-security.md @@ -179,38 +179,39 @@ queryable lifecycle. ### Index plan -| Query shape | Index or constraint | -| ----------------------------- | ----------------------------------------------------------------------------------------- | -| Session lookup | unique `auth_sessions(validator_hash)` | -| Session expiry cleanup | `auth_sessions(expires_at_ms)` | -| User credentials | `user_webauthn_credentials(user_id, created_at, id)` and unique credential ID | -| WebAuthn challenge | unique partial binding/purpose indexes plus `(expires_at, id)` cleanup | -| Automation principal history | `automation_principals_created_id_idx` plus `automation_principals_active_created_id_idx` | -| Automation credential history | `automation_credentials_principal_created_idx` | -| Active automation credentials | partial `automation_credentials_active_principal_created_idx` while unrevoked | -| Staged credential rotation | full `automation_credentials_replacement_idx` plus a unique partial replacement index | -| Task board | `tasks(status, priority, updated_at_ms DESC)` | -| Task label filter | `task_labels(label, task_id)` | -| Task timeline | `task_updates(task_id, created_at_ms, id)` and equivalent event index | -| Agent task history | unique active-agent partial index plus `(agent_id, started_at_ms, id)` | -| Latest reports | `reports(kind, occurred_at_ms DESC, id DESC)` | -| Heartbeat stream | `reports(source, source_job_id, occurred_at_ms DESC, id DESC)` | -| Active incidents | partial `incidents(monitor_key, last_seen_at_ms DESC) WHERE state = 'active'` | -| Incident identity | unique `incidents(monitor_key, fingerprint)` | -| Unread notifications | partial `notifications(occurred_at_ms DESC) WHERE read_at_ms IS NULL` | -| Incident notification | unique `(incident_id, incident_generation, channel)` when incident is non-null | -| Queue claim | partial `job_runs(available_at, priority DESC, queued_at, id) WHERE state = 'queued'` | -| One active scheduled run | unique partial `job_runs(scheduled_job_id) WHERE state IN ('queued', 'running')` | -| Active action status | partial `job_runs_action_active_idx`; exact predicate below | -| Terminal maintenance status | partial `job_runs_action_payload_terminal_idx`; exact predicate below | -| Worker expiry | `worker_instances(heartbeat_at, id)` | -| Job timeline | `job_run_events(job_run_id, sequence)` | -| Realtime catch-up | `realtime_events(topic, id)` | -| Chat replay | unique `chat_run_events(chat_run_id, sequence)` | -| Deployment history | `deployments(state, updated_at_ms DESC)` | -| Docker history | `docker_update_events(managed_service_id, created_at_ms DESC)` | -| Cache refresh/expiry | `cache_entries(last_attempt_status, expires_at_ms, key)` | -| Audit cursor | `audit_events(occurred_at_ms DESC, id DESC)` plus request/target indexes | +| Query shape | Index or constraint | +| ------------------------------ | ----------------------------------------------------------------------------------------- | +| Session lookup | unique `auth_sessions(validator_hash)` | +| Session expiry cleanup | `auth_sessions(expires_at_ms)` | +| User credentials | `user_webauthn_credentials(user_id, created_at, id)` and unique credential ID | +| WebAuthn challenge | unique partial binding/purpose indexes plus `(expires_at, id)` cleanup | +| Automation principal history | `automation_principals_created_id_idx` plus `automation_principals_active_created_id_idx` | +| Automation credential history | `automation_credentials_principal_created_idx` | +| Active automation credentials | partial `automation_credentials_active_principal_created_idx` while unrevoked | +| Staged credential rotation | full `automation_credentials_replacement_idx` plus a unique partial replacement index | +| Task board | `tasks(status, priority, updated_at_ms DESC)` | +| Task label filter | `task_labels(label, task_id)` | +| Task timeline | `task_updates(task_id, created_at_ms, id)` and equivalent event index | +| Agent task history | unique active-agent partial index plus `(agent_id, started_at_ms, id)` | +| Latest reports | `reports(kind, occurred_at_ms DESC, id DESC)` | +| Heartbeat stream | `reports(source, source_job_id, occurred_at_ms DESC, id DESC)` | +| Active incidents | partial `incidents(monitor_key, last_seen_at_ms DESC) WHERE state = 'active'` | +| Incident identity | unique `incidents(monitor_key, fingerprint)` | +| Unread notifications | partial `notifications(occurred_at_ms DESC) WHERE read_at_ms IS NULL` | +| Incident notification | unique `(incident_id, incident_generation, channel)` when incident is non-null | +| Queue claim | partial `job_runs(available_at, priority DESC, queued_at, id) WHERE state = 'queued'` | +| One active scheduled run | unique partial `job_runs(scheduled_job_id) WHERE state IN ('queued', 'running')` | +| Active action status | partial `job_runs_action_active_idx`; exact predicate below | +| Terminal maintenance status | partial `job_runs_action_payload_terminal_idx`; exact predicate below | +| Terminal Service Action status | partial `job_runs_service_action_terminal_idx`; exact predicate below | +| Worker expiry | `worker_instances(heartbeat_at, id)` | +| Job timeline | `job_run_events(job_run_id, sequence)` | +| Realtime catch-up | `realtime_events(topic, id)` | +| Chat replay | unique `chat_run_events(chat_run_id, sequence)` | +| Deployment history | `deployments(state, updated_at_ms DESC)` | +| Docker history | `docker_update_events(managed_service_id, created_at_ms DESC)` | +| Cache refresh/expiry | `cache_entries(last_attempt_status, expires_at_ms, key)` | +| Audit cursor | `audit_events(occurred_at_ms DESC, id DESC)` plus request/target indexes | The action-status indexes intentionally mirror the repository's literal predicates: @@ -221,6 +222,10 @@ The action-status indexes intentionally mirror the repository's literal predicat `(action_key, payload_json, queued_at DESC, id DESC)` where `action_key = 'maintenance.rotate-logs'`, `length(CAST(payload_json AS BLOB)) <= 128`, and `state IN ('cancelled', 'failed', 'succeeded', 'timed-out')`. +- `job_runs_service_action_terminal_idx` indexes + `(action_key, queued_at DESC, id DESC)` where `action_key` is one of the six fixed + Service Action keys, `payload_json = '{}'`, and + `state IN ('cancelled', 'failed', 'succeeded', 'timed-out')`. Primary keys and unique constraints already create indexes; the schema does not add redundant copies. Partial-index predicates must match query predicates exactly enough for SQLite to use @@ -282,27 +287,38 @@ restarts, or unbounded shell commands. Those operations become durable `job_runs the worker. Service Actions are a separate fixed-intent boundary, not a generic exec facade. The contract -contains exactly `openclaw-cleanup`, `openclaw-update`, `system-restart`, and `system-update`; a -caller can supply only one of those IDs plus an actor-bound idempotency key. Reads and requests are -session-only under dedicated capabilities, requests require recent MFA, and audit attempt must -commit before the durable enqueue handoff. That handoff rechecks exact-release worker -availability, the current browser session, and recent MFA. Enqueue uncertainty is reconciled by -the same principal/idempotency intent, and post-dispatch uncertainty never authorizes a replay. +contains exactly `openclaw-cleanup`, `openclaw-restart`, `openclaw-update`, `system-cleanup`, +`system-restart`, and `system-update`; a caller can supply only one of those IDs plus an actor-bound +idempotency key. +Reads and requests are session-only under dedicated capabilities, requests require recent MFA, and +audit attempt must commit before the durable enqueue handoff. That handoff rechecks exact-release +worker availability, the current browser session, and recent MFA. Enqueue uncertainty is +reconciled by the same principal/idempotency intent, and post-dispatch uncertainty never authorizes +a replay. The production worker advertises only actions for which its composition owns an exact executor. OpenClaw cleanup and update are worker-only, fixed-parameter Gateway operations with bounded, -sanitized results. Host restart and host update remain canonical contract/UI rows but are -unavailable in production because the web and worker currently share one Unix identity. A shared -group or polkit grant would therefore collapse the web/worker trust boundary. This rewrite ships -no shared-user host broker, polkit rule, root helper, or host-operation systemd unit. Future host -enablement requires a distinct worker OS identity, root-owned immutable worker execution, exact -subject and operation policy, and reviewed install/rollback evidence before either action key can -be advertised. - -This boundary is a partial secure replacement for `POST /api/exec/start`, not a feature-removal -claim. The legacy `system_cleanup` intent remains planned as three separately authorized effects: -Docker prune in the Docker slice, apt cleanup in host/package maintenance, and journald vacuum in -log maintenance. None may be smuggled back through a generic shell or shared-user privilege grant. +sanitized results. OpenClaw restart reuses the existing fixed `openclaw.gateway.restart` worker +definition, executor, and provider already used by Settings; its `host.mutation` plus +`openclaw.gateway` resource locks serialize it with both host maintenance and other Gateway +mutations. Host cleanup, restart, and update have one separately provisioned fixed broker +whose only input is the reviewed operation ID and whose only output is an accepted/completed +status. The root-owned units use fixed paths, fixed arguments, bounded output and deadlines, and no +shell or caller-controlled environment. Manifest-bound provisioning validates no-follow file +identity, ownership, modes, and content integrity and retains explicit rollback to the previous +immutable release. Production does not compose that broker while web and worker share one Unix +identity: the host actions stay unavailable until a distinct worker OS principal exists and +separately approved provisioning binds only that principal, excludes the web process, reloads the +reviewed policy and units, and composes the broker. + +`system-cleanup` attempts all fixed phases and fails if any phase fails: package autoremove, +package-cache cleanup, journald rotation plus 14-day/1 GiB retention, and Docker system prune for +unused content older than 168 hours. It never prunes volumes. The durable definition is exclusive, +single-attempt, non-cancellable, non-retry-safe, and reserves both `host.mutation` and `host.logs`. +Together with the bounded PTY, this defines the narrow replacement for the consumed +`POST /api/exec/start` behavior without reintroducing a generic shell, command, path, or +shared-user privilege grant. The parity row remains planned until the distinct-worker production +topology and separately approved provisioning make this host action executable. The `cache:read` automation heartbeat is a separate sanitized projection, not a shortcut around session, task, job, or cron detail authorization. It reads process-local validated Gateway @@ -322,6 +338,13 @@ Queue behavior is explicit: - a strict singleton `job_worker_control` row persists cross-process claim pause state and versioned operator changes; its absence is an integrity failure, never an implicit resume; +- the separate singleton `host_restart_claim_fence` is armed atomically only when its exact + `host.system.restart` lease is the sole globally running run; every worker refuses new claims + while that fence is unexpired for the kernel-owned Linux boot identity; +- after dispatch begins, an error or lost response cannot prove that `systemctl --no-block` failed + before accepting the reboot timer, so both accepted and ambiguous outcomes retain the fence; a + changed boot identity removes it, and bounded same-boot expiry restores admission if reboot + never occurs; - one immediate transaction considers at most 32 totally ordered candidates, skips candidates with occupied resources, and atomically assigns the first eligible run plus every required resource lease; diff --git a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md index 3e8d046ee..c0c5d444f 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md +++ b/greenfield/docs/architecture/greenfield-rewrite/implementation-plan.md @@ -103,14 +103,19 @@ including restart during streaming. filesystem isolation requires a separate mount, namespace, or container sandbox. - keep shell `cd`, completion, and termination inside the implemented bounded PTY. Replace consumed legacy exec behavior only with purpose-built durable Service Actions; do not restore a generic - command, shell, or cwd API for the unused synchronous exec route. Keep `POST /api/exec/start` - planned until `system_cleanup` is decomposed without feature loss: Docker prune in the Docker - slice, apt cleanup in host/package maintenance, and journald vacuum in log maintenance. -- expose the four fixed Service Action intents in contract/UI, but advertise only exact executors + command, shell, or cwd API for the unused synchronous exec route. Stage the replacement for the + consumed `POST /api/exec/start` behavior as that PTY plus one fixed `system-cleanup` operation + that preserves package cleanup, bounded journald retention, and age-filtered Docker pruning + without deleting volumes. Keep the parity row planned until the host operation is executable in + the approved production topology. +- expose the six fixed Service Action intents in contract/UI, but advertise only exact executors owned by a fresh worker on the current release. OpenClaw cleanup/update use reviewed worker-only - Gateway methods. Host restart/update remain unavailable until web and worker have distinct OS - identities and a root-owned immutable worker boundary with reviewed provisioning and rollback; - a shared-user/group polkit grant is forbidden. + Gateway methods, while OpenClaw restart reuses the existing fixed restart executor/provider also + exposed in Settings. Host cleanup/restart/update use only exact root-owned systemd units through the + fixed worker broker. Production must not compose that broker until the worker has a distinct OS + principal. Separately approved provisioning must bind only that principal, exclude the web + principal, validate immutable artifacts, and preserve explicit rollback; a shared-user/group + grant is forbidden. **Exit gate:** capability, step-up, audit, cancellation, resource-limit, and failure-recovery tests pass for every privileged operation. diff --git a/greenfield/docs/architecture/greenfield-rewrite/progress.md b/greenfield/docs/architecture/greenfield-rewrite/progress.md index ddb03b4b9..86720dd69 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/progress.md +++ b/greenfield/docs/architecture/greenfield-rewrite/progress.md @@ -7,15 +7,15 @@ This matrix is the living phase status. Update it in the same change that materially advances or closes a phase; dated entries below provide the evidence, not a second status source. -| Phase | Status | Current evidence and remaining gate | -| ----------------------------------- | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | -| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | -| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | -| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | -| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | -| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Overview Service Actions partially replace consumed legacy exec flows with four fixed intents; OpenClaw cleanup/update are worker-owned, host restart/update remain explicitly unavailable pending a distinct worker OS identity, and `POST /api/exec/start` stays planned until system cleanup is decomposed across Docker, host/package, and log-maintenance authorities. Docker control, database, GitHub, deployment, database backup/restore, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | -| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | +| Phase | Status | Current evidence and remaining gate | +| ----------------------------------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0 — Evidence and qualification | Complete | All eight mandatory spikes pass on exact Bun revision `17d6843606d76620cb55d31424d7fb0aed51c367`: build, transport, cross-process SQLite/outbox, Drizzle/Bun SQLite, browser data, chat batching, shutdown, and capped resources. Source-derived parity and the OpenClaw source audit pass as additional evidence. | +| 1 — Foundation | Complete | The self-contained future root builds immutable browser/web/worker artifacts, protects project-local production state, installs exact Bun and systemd artifacts, migrates a database copy, atomically promotes the release/database pair, serves readiness/browser assets, writes project-local logs, and proves crash-safe rollback and shutdown in a disposable lifecycle. | +| 2 — Trust and transport | Complete for the stated server scope | Authentication, MFA, WebAuthn, automation credentials, audit, authenticated renewable SSE, one-shot native Gateway bootstrap verification, and the consolidated [threat model](../../security/greenfield-phase-two-threat-model.md) have executable evidence. Browser UI and production cutover remain later gates. | +| 3 — Core operator domains | Started | Task and agent-directory parity are implemented with durable history, realtime invalidation, and browser workflows. Monitoring ingestion plus report, incident, and notification server/browser parity are implemented. Dashboard-local durable schedules/jobs, real worker execution, their `/jobs` operator UI, the first claim-fenced `system.host` cache provider, its cache browser, and bounded system metrics are implemented. Root composition covers every implemented Phase 3 operator domain, and Phase 4A now supplies the OpenClaw-cron half of `/jobs`. Full root parity and privileged/external providers remain later gates, so the Phase 3 exit stays open. | +| 4 — Gateway and chat | Started | The current installed OpenClaw source is hash-pinned for the persistent sessions, cron, chat, companion, task, and media surfaces. Process-owned Gateway lifecycle, durable realtime invalidation, sessions and agent availability, OpenClaw cron/tasks, the compact heartbeat, the durable chat journal/runtime, bounded history and reconciliation, managed and descriptor-rooted local-history media through one transcript-authorized proxy, and the `/chat` frontend are implemented. Recorded contract, protocol, service, browser, restart, load-boundary, and security tests cover the slice; live Gateway smoke/restart evidence and the Phase 4 exit gate remain open. | +| 5 — Privileged and external domains | Started | Files and Logs have closed reviewed `/files` and `/logs` parity. Moltbook now has a fixed-host worker-only provider, one claim-fenced durable last-known-good snapshot, four session-only read procedures, and the reviewed `/moltbook` browser workflow. Terminal is a worker-owned interactive PTY over a hardened WebSocket with bounded reconnect replay. `/settings` exposes bounded, redacted OpenClaw configuration and skill controls plus a one-shot exact configuration export and a durable worker-owned Gateway restart. The legacy local-media path API is securely narrowed to opaque transcript-bound Chat media references without a new browser route. Overview Service Actions expose six fixed intents; OpenClaw cleanup/restart/update are worker-owned, while bounded system cleanup preserves the reviewed package/journal/Docker effects behind a foundation that remains unavailable until the distinct-worker topology and separately approved root provisioning exist. `POST /api/exec/start` therefore stays planned. Docker control, database, GitHub, deployment, database backup/restore, production host provisioning, and the remaining privileged adapters stay open; the Phase 5 exit gate is not claimed. | +| 6 — Parity, hardening, and cutover | Not started | Full UI parity, generated `/docs`, load/resource/restore evidence, cutover rehearsal, fresh production database, and legacy removal remain open. | ### 2026-08-03 — Phase 0 started @@ -1555,33 +1555,49 @@ full-browser parity, production rehearsal, cutover, and legacy deletion remain o ### 2026-08-12 — Purpose-built Service Actions narrow consumed exec authority -- The Overview now exposes exactly four fixed Service Actions through session-only +- The Overview now exposes exactly six fixed Service Actions through session-only `serviceActions.getStatus` and recent-MFA `serviceActions.request`: OpenClaw cleanup, OpenClaw - update, host restart, and host update. Requests carry a caller-owned idempotency key, commit a - fail-closed attempted audit record, recheck fresh exact-release worker availability, and - revalidate session and recent MFA at durable enqueue. The browser receives only a durable run ID - and follows progress through `/jobs`; no stdout, command, environment, or provider response + restart, OpenClaw update, system cleanup, host restart, and host update. Requests carry a caller-owned idempotency + key, commit a fail-closed attempted audit record, recheck fresh exact-release worker availability, + and revalidate session and recent MFA at durable enqueue. The browser receives only a durable run + ID and follows progress through `/jobs`; no stdout, command, environment, or provider response crosses the contract. +- `/jobs` always renders the complete fixed six-action inventory, including actions with no prior + run, alongside Dashboard and OpenClaw cron jobs. Each observed run ID links to that exact run + detail instead of implying that only Overview owns the history. - OpenClaw cleanup and update are implemented as exact worker-only, hash-pinned `sessions.cleanup` and `update.run` calls. Cleanup uses OpenClaw's own bounded maintenance policy instead of reintroducing legacy recursive deletion, and update preserves the managed handoff. Both actions are single-attempt, non-retry-safe, non-cancellable, resource-locked jobs with sanitized results and explicit unknown-outcome handling. -- Host restart and host update remain fixed contract/UI rows but production reports both - `unavailable`. Web and worker currently share one Unix identity, so a shared-user or group-based - polkit broker would collapse the intended privilege boundary. No unsafe host broker, root helper, - polkit rule, or operation unit ships. Future enablement requires a distinct worker OS identity, - root-owned immutable worker code/configuration, exact-principal authorization, and reviewed - provisioning plus rollback before those worker action keys may be advertised. +- OpenClaw restart reuses the fixed `openclaw.gateway.restart` definition, executor, and provider + already used by Settings; its shared `host.mutation` and `openclaw.gateway` resource locks + serialize it with host maintenance and Gateway operations. Service Actions add no second + lifecycle command and preserve the Settings restart surface. +- System cleanup is defined as an exclusive, single-attempt fixed host operation. It attempts package + autoremove/cache cleanup, journald rotation with 14-day/1 GiB retention, and unused Docker content + pruning only after 168 hours, never volumes. Host cleanup, restart, and update use an exact + `/usr/bin/systemctl` broker and root-owned fixed units with bounded deadlines and output. The + release ships manifest-verified provisioning and rollback artifacts, but production reports the + host rows `unavailable`. Because web and worker currently share one Unix identity, activation + first requires a distinct worker OS principal; only then may separately approved root + provisioning bind that principal, exclude the web process, reload the reviewed units and policy, + and compose the broker. +- Host restart additionally has a durable database-global admission fence. The exact restart + claim may arm it only when no other run is globally running; every worker then blocks new claims. + Acceptance and every ambiguous broker rejection, abort, timeout, or lost response retain it + because `systemctl --no-block` may already have accepted the reboot timer. A changed Linux boot + identity reconciles it, and bounded same-boot expiry recovers when no reboot occurs. This does + not reuse operator pause. - The interactive PTY already owns shell `cd`, completion, and bounded termination. Legacy long-running exec consumers map to either that PTY or the purpose-built durable Service Actions queue. The unused synchronous `POST /api/exec` route is a reviewed removal with no current browser or scoped automation consumer; no generic shell/command replacement was added. - `POST /api/exec/start` remains planned because the broad legacy `system_cleanup` consumer is only - partially replaced: Docker prune belongs to the Docker slice, apt cleanup to host/package - maintenance, and journald vacuum to log maintenance. Keeping the row open preserves all three - operator capabilities without restoring their unsafe shared shell boundary. -- The living inventory is now **112 implemented, 42 planned, and three reviewed removals** out of + The bounded PTY plus fixed `system-cleanup` foundation preserve the narrow replacement for all + three consumed cleanup effects without restoring their unsafe shared shell boundary. Because + the host executor remains unavailable in production, `POST /api/exec/start` stays planned until + the distinct-worker topology and separately approved root provisioning are complete. +- The living inventory remains **112 implemented, 42 planned, and three reviewed removals** out of 157 legacy endpoints. Browser routes remain **12 implemented and four planned**. This advances Phase 5 without claiming complete exec, host-operation, Docker/database/delivery parity, or the aggregate Phase 5 exit gate. diff --git a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md index 9601db765..794c2cae1 100644 --- a/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md +++ b/greenfield/docs/architecture/greenfield-rewrite/runtime-and-delivery.md @@ -423,27 +423,59 @@ target-directory write access for its private stage file, `renameat2` exchange, at runtime. Descriptor validation, per-file bounds, CAS, and the fixed worker manifest are the write boundary. -The fixed Service Actions contract and Overview include host restart and host update, but the -production release does not install or compose authority for them. Web and worker still run as the -same Unix user, so granting that identity a root helper, polkit action, or root-owned operation unit -would also grant the web process the same authority. Consequently both host actions remain -`unavailable`, the worker does not advertise their action keys, and enqueue rechecks fail closed. -No host-operation helper, polkit rule, or operation unit is included in release staging. - -Future host-action enablement is a delivery/topology change rather than an application toggle. It -must introduce a distinct worker OS identity, keep the web principal outside that identity and its -groups, execute only root-owned immutable worker code/configuration, constrain authorization to -the exact worker principal and fixed operation, and ship manifest-verified provisioning plus -explicit rollback. Only after that boundary has executable installation, identity, availability, -and rollback evidence may production compose a host broker and advertise either host action. -OpenClaw cleanup and update do not use this deferred host authority: their exact worker-only -Gateway operations are already implemented and remain available only when a fresh exact-release -worker advertises them. - -Those fixed operations do not close the legacy `POST /api/exec/start` row. Its `system_cleanup` -consumer remains planned as Docker prune in the Docker slice, apt cleanup in host/package -maintenance, and journald vacuum in log maintenance. Delivery must preserve each capability behind -its own reviewed authority rather than recreate the old shared shell boundary. +The fixed Service Actions contract and Overview include host cleanup, host restart, and host +update. A fixed `/usr/bin/systemctl` broker and root-owned helper, policy, service/timer, and +manifest-verified installer artifacts ship in the release, but the current production composition +does not instantiate the broker or mutate root-owned host state. Web and worker still share one +Unix identity, so group-authorizing that identity would also authorize the internet-facing web +process. Consequently all host actions remain `unavailable` and enqueue rechecks fail closed. + +Before the worker can request the fixed restart unit, it atomically arms the database-global +restart claim fence using the validated Linux `/proc/sys/kernel/random/boot_id` identity. Arming +requires that exact owned restart lease to be the only globally running job; once armed, all +worker processes stop claiming new jobs. The current broker has no proof that a rejection, abort, +timeout, or lost response occurred before `systemctl start --no-block` accepted the reboot timer; +therefore every outcome after dispatch begins retains the fence. A new boot identity reconciles +it, while five-minute same-boot expiry restores admission if no reboot occurs. This safety fence +is independent of the operator-controlled queue pause. + +Host-action enablement is a reviewed delivery/topology operation rather than an application +toggle. It must first move the worker to a distinct OS principal, keep the web principal outside +that identity and its groups, execute only root-owned immutable code/configuration, constrain +authorization to the three exact units, verify no-follow identity/ownership/mode/content, retain +explicit rollback to the previous immutable release, and then compose the broker. Only that +boundary may make the worker advertise a host action. +The root installer must never consume the application-owned release tree directly. A reviewed +handoff first transfers the exact release into a dedicated root-owned immutable staging path. The +release root and every traversed source directory must be `root:root 0500`; the release identity, +manifest, and every admitted helper/unit/policy artifact must be `root:root 0400`. Source hashes +from an application-owned release do not establish authority, even when internally consistent. +The handoff also provisions one exact root-owned Bun runtime at +`/var/lib/mira-dashboard-host-provisioning/runtime/bun` with mode `0555`; every ancestor is +root-owned and not group/other-writable beneath the root-owned, non-group/other-writable +`/var/lib/mira-dashboard-host-provisioning` trust root. It invokes the root-owned staged installer +by absolute path, never a package script or application-checkout module. This pre-execution boundary is +mandatory because Bun loads the entrypoint and its local dependencies before their in-process +runtime/source checks can execute. The installer then validates its exact `process.execPath` and +ancestor ownership/modes again before admitting release bytes. +Before ownership transfer or launch, change control independently verifies the candidate against +the reviewed Git commit/tree and supplies the exact release-manifest SHA-256 out of band. The root +command must not derive that trust anchor from the application checkout. The installer compares +the supplied digest to the held root-owned manifest bytes before parsing any artifact digest, so an +internally consistent app-forged release and manifest are insufficient. +OpenClaw cleanup, restart, and update do not use this deferred host authority: their exact +worker-only Gateway operations are already implemented and remain available only when a fresh +exact-release worker advertises them. Restart reuses the same fixed action provider as the Settings +control rather than adding a second lifecycle executor. + +The fixed `system-cleanup` unit preserves the consumed cleanup behavior behind one reviewed +authority. It attempts package autoremove and cache cleanup, journald rotate plus 14-day/1 GiB +vacuum bounds, and Docker system prune for unused content older than 168 hours; it never passes +`--volumes`. Each phase is attempted, any failure fails the unit, output is discarded, and the +worker receives only a completed status. Together with the bounded PTY this defines the narrow +replacement for the legacy `POST /api/exec/start` behavior without recreating the old shared shell +boundary. That parity row remains planned until the distinct worker identity and separately +approved root provisioning make `system-cleanup` executable in production. The web process also derives the fixed `/media` descriptor boundary from that same reviewed root. It exposes no configurable media directory, recursive listing, or diff --git a/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json b/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json index b5aa0e085..b3bd2f76a 100644 --- a/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json +++ b/greenfield/docs/generated/schemas/serviceActions.getStatus.output.schema.json @@ -224,7 +224,9 @@ "id": { "enum": [ "openclaw-cleanup", + "openclaw-restart", "openclaw-update", + "system-cleanup", "system-restart", "system-update" ], @@ -444,8 +446,8 @@ ], "additionalProperties": false }, - "maxItems": 4, - "$comment": "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered." + "maxItems": 6, + "$comment": "Live Valibot validation additionally requires the six fixed service-action rows to be complete, unique, and canonically ordered." }, "observedAtMs": { "type": "integer", diff --git a/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json b/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json index e25b87cfc..f2986875b 100644 --- a/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json +++ b/greenfield/docs/generated/schemas/serviceActions.request.input.schema.json @@ -24,6 +24,29 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "actionId": { + "const": "openclaw-restart" + }, + "confirmation": { + "const": "restart-openclaw" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -47,6 +70,29 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "actionId": { + "const": "system-cleanup" + }, + "confirmation": { + "const": "cleanup-system" + }, + "idempotencyKey": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^(?:[A-Za-z0-9_-]{4})*(?:[A-Za-z0-9_-][AQgw]|[A-Za-z0-9_-]{2}[AEIMQUYcgkosw048])?$" + } + }, + "required": [ + "actionId", + "confirmation", + "idempotencyKey" + ], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json b/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json index c9688b937..0df0cd18f 100644 --- a/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json +++ b/greenfield/docs/generated/schemas/serviceActions.request.output.schema.json @@ -5,7 +5,9 @@ "actionId": { "enum": [ "openclaw-cleanup", + "openclaw-restart", "openclaw-update", + "system-cleanup", "system-restart", "system-update" ], diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql index 2888300a7..e63a17b09 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql +++ b/greenfield/migrations/20260804022252_dashboard-foundation/migration.sql @@ -963,6 +963,7 @@ CREATE TABLE `job_runs` ( `queued_at` integer NOT NULL, `requested_by_id` text NOT NULL, `requested_by_kind` text NOT NULL, + `required_worker_release_id` text, `resource_class` text NOT NULL, `resource_keys_json` text NOT NULL, `result_json` text, @@ -993,6 +994,7 @@ CREATE TABLE `job_runs` ( CONSTRAINT "job_runs_payload_json_check" CHECK(length(CAST("payload_json" AS BLOB)) <= 65536 AND CASE WHEN json_valid("payload_json") THEN json_type("payload_json") = 'object' ELSE 0 END), CONSTRAINT "job_runs_priority_check" CHECK("priority" BETWEEN -100 AND 100), CONSTRAINT "job_runs_requested_actor_check" CHECK((("requested_by_kind" = 'user' AND length("requested_by_id") = 36 AND instr("requested_by_id", char(0)) = 0 AND length(replace("requested_by_id", '-', '')) = 32 AND replace("requested_by_id", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr("requested_by_id", 9, 1) = '-' AND substr("requested_by_id", 14, 1) = '-' AND substr("requested_by_id", 15, 1) = '7' AND substr("requested_by_id", 19, 1) = '-' AND substr("requested_by_id", 20, 1) GLOB '[89ab]' AND substr("requested_by_id", 24, 1) = '-') OR ("requested_by_kind" = 'automation' AND length("requested_by_id") BETWEEN 1 AND 64 AND instr("requested_by_id", char(0)) = 0 AND "requested_by_id" = lower("requested_by_id") AND substr("requested_by_id", 1, 1) GLOB '[a-z0-9]' AND "requested_by_id" NOT GLOB '*[^a-z0-9._-]*') OR ("requested_by_kind" = 'system' AND length("requested_by_id") BETWEEN 1 AND 128 AND instr("requested_by_id", char(0)) = 0 AND "requested_by_id" = lower("requested_by_id") AND substr("requested_by_id", 1, 1) GLOB '[a-z0-9]' AND "requested_by_id" NOT GLOB '*[^a-z0-9._-]*'))), + CONSTRAINT "job_runs_required_worker_release_id_check" CHECK("required_worker_release_id" IS NULL OR length("required_worker_release_id") = 40 AND instr("required_worker_release_id", char(0)) = 0 AND "required_worker_release_id" NOT GLOB '*[^0-9a-f]*'), CONSTRAINT "job_runs_resource_class_check" CHECK("resource_class" IN ('exclusive', 'host-heavy', 'interactive', 'light', 'network')), CONSTRAINT "job_runs_resource_keys_json_check" CHECK(length(CAST("resource_keys_json" AS BLOB)) <= 4096 AND CASE WHEN json_valid("resource_keys_json") THEN json_type("resource_keys_json") = 'array' ELSE 0 END), CONSTRAINT "job_runs_result_json_check" CHECK("result_json" IS NULL OR (length(CAST("result_json" AS BLOB)) <= 65536 AND CASE WHEN json_valid("result_json") THEN json_type("result_json") = 'object' ELSE 0 END)), @@ -1103,6 +1105,22 @@ CREATE TABLE `worker_instances` ( CONSTRAINT "worker_instances_time_check" CHECK("started_at" BETWEEN 0 AND 8640000000000000 AND "heartbeat_at" BETWEEN 0 AND 8640000000000000 AND "heartbeat_at" >= "started_at" AND ("draining_at" IS NULL OR ("draining_at" BETWEEN 0 AND 8640000000000000 AND "draining_at" >= "started_at")) AND ("stopped_at" IS NULL OR ("stopped_at" BETWEEN 0 AND 8640000000000000 AND "stopped_at" >= "draining_at"))) ) STRICT, WITHOUT ROWID; --> statement-breakpoint +CREATE TABLE `host_restart_claim_fence` ( + `armed_at` integer NOT NULL, + `boot_identity` text NOT NULL, + `expires_at` integer NOT NULL, + `id` integer PRIMARY KEY NOT NULL, + `job_run_id` text NOT NULL, + `lease_token` text NOT NULL, + `worker_instance_id` text NOT NULL, + CONSTRAINT `fk_host_restart_claim_fence_job_run_id_job_runs_id_fk` FOREIGN KEY (`job_run_id`) REFERENCES `job_runs`(`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, + CONSTRAINT `fk_host_restart_claim_fence_worker_instance_id_worker_instances_id_fk` FOREIGN KEY (`worker_instance_id`) REFERENCES `worker_instances`(`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, + CONSTRAINT "host_restart_claim_fence_boot_identity_check" CHECK(length("boot_identity") = 36 AND instr("boot_identity", char(0)) = 0 AND length(replace("boot_identity", '-', '')) = 32 AND replace("boot_identity", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr("boot_identity", 9, 1) = '-' AND substr("boot_identity", 14, 1) = '-' AND substr("boot_identity", 19, 1) = '-' AND substr("boot_identity", 24, 1) = '-'), + CONSTRAINT "host_restart_claim_fence_id_check" CHECK("id" = 1), + CONSTRAINT "host_restart_claim_fence_lease_token_check" CHECK(length("lease_token") = 36 AND instr("lease_token", char(0)) = 0 AND length(replace("lease_token", '-', '')) = 32 AND replace("lease_token", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr("lease_token", 9, 1) = '-' AND substr("lease_token", 14, 1) = '-' AND substr("lease_token", 15, 1) = '7' AND substr("lease_token", 19, 1) = '-' AND substr("lease_token", 20, 1) GLOB '[89ab]' AND substr("lease_token", 24, 1) = '-'), + CONSTRAINT "host_restart_claim_fence_time_check" CHECK("armed_at" BETWEEN 0 AND 8640000000000000 AND "expires_at" BETWEEN 0 AND 8640000000000000 AND "expires_at" > "armed_at") +) STRICT, WITHOUT ROWID; +--> statement-breakpoint CREATE UNIQUE INDEX `job_disable_intents_active_schedule_unique` ON `job_disable_intents` (`scheduled_job_id`) WHERE "job_disable_intents"."scheduled_job_id" IS NOT NULL AND "job_disable_intents"."ended_at" IS NULL;--> statement-breakpoint CREATE UNIQUE INDEX `job_disable_intents_active_external_unique` ON `job_disable_intents` (`external_provider`,`external_job_id`) WHERE "job_disable_intents"."external_job_id" IS NOT NULL AND "job_disable_intents"."ended_at" IS NULL;--> statement-breakpoint CREATE INDEX `job_disable_intents_active_expiry_idx` ON `job_disable_intents` (`expires_at`,`id`) WHERE "job_disable_intents"."expires_at" IS NOT NULL AND "job_disable_intents"."ended_at" IS NULL;--> statement-breakpoint @@ -1114,6 +1132,7 @@ CREATE INDEX `job_runs_claim_idx` ON `job_runs` ("available_at" asc,"priority" d CREATE UNIQUE INDEX `job_runs_one_active_schedule_idx` ON `job_runs` (`scheduled_job_id`) WHERE "job_runs"."scheduled_job_id" IS NOT NULL AND "job_runs"."state" IN ('queued', 'running');--> statement-breakpoint CREATE INDEX `job_runs_action_active_idx` ON `job_runs` (`action_key`,"state" desc,"queued_at" desc,"id" desc) WHERE "job_runs"."state" IN ('queued', 'running');--> statement-breakpoint CREATE INDEX `job_runs_action_payload_terminal_idx` ON `job_runs` (`action_key`,`payload_json`,"queued_at" desc,"id" desc) WHERE "job_runs"."action_key" = 'maintenance.rotate-logs' AND length(CAST("job_runs"."payload_json" AS BLOB)) <= 128 AND "job_runs"."state" IN ('cancelled', 'failed', 'succeeded', 'timed-out');--> statement-breakpoint +CREATE INDEX `job_runs_service_action_terminal_idx` ON `job_runs` (`action_key`,"queued_at" desc,"id" desc) WHERE "job_runs"."action_key" IN ('openclaw.sessions.cleanup', 'openclaw.gateway.restart', 'openclaw.installation.update', 'host.system.cleanup', 'host.system.restart', 'host.system.update') AND "job_runs"."payload_json" = '{}' AND "job_runs"."state" IN ('cancelled', 'failed', 'succeeded', 'timed-out');--> statement-breakpoint CREATE INDEX `job_runs_queued_id_idx` ON `job_runs` (`queued_at`,`id`);--> statement-breakpoint CREATE INDEX `job_runs_schedule_queued_id_idx` ON `job_runs` (`scheduled_job_id`,`queued_at`,`id`);--> statement-breakpoint CREATE INDEX `job_runs_running_lease_idx` ON `job_runs` (`lease_expires_at`,`id`) WHERE "job_runs"."state" = 'running';--> statement-breakpoint @@ -2547,3 +2566,31 @@ BEFORE DELETE ON task_events BEGIN SELECT RAISE(ABORT, 'task_events is append-only'); END; +--> statement-breakpoint +CREATE TRIGGER host_restart_claim_fence_validate_insert +BEFORE INSERT ON host_restart_claim_fence +WHEN NOT EXISTS ( + SELECT 1 + FROM job_runs + WHERE id = NEW.job_run_id + AND action_key = 'host.system.restart' + AND payload_json = '{}' + AND state = 'running' + AND lease_owner_id = NEW.worker_instance_id + AND lease_token = NEW.lease_token + AND lease_expires_at > NEW.armed_at +) + OR EXISTS ( + SELECT 1 + FROM job_runs + WHERE state = 'running' AND id <> NEW.job_run_id + ) +BEGIN + SELECT RAISE(ABORT, 'host restart fence requires the only running exact restart claim'); +END; +--> statement-breakpoint +CREATE TRIGGER host_restart_claim_fence_reject_update +BEFORE UPDATE ON host_restart_claim_fence +BEGIN + SELECT RAISE(ABORT, 'host restart claim fence is immutable'); +END; diff --git a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json index fb044f48f..4e8383d07 100644 --- a/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json +++ b/greenfield/migrations/20260804022252_dashboard-foundation/snapshot.json @@ -62,6 +62,10 @@ "name": "chat_transcript_generations", "entityType": "tables" }, + { + "name": "host_restart_claim_fence", + "entityType": "tables" + }, { "name": "incident_observations", "entityType": "tables" @@ -1728,6 +1732,76 @@ "entityType": "columns", "table": "chat_transcript_generations" }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "armed_at", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "boot_identity", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "expires_at", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "job_run_id", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lease_token", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "worker_instance_id", + "entityType": "columns", + "table": "host_restart_claim_fence" + }, { "type": "text", "notNull": true, @@ -2438,6 +2512,16 @@ "entityType": "columns", "table": "job_runs" }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "required_worker_release_id", + "entityType": "columns", + "table": "job_runs" + }, { "type": "text", "notNull": true, @@ -4403,6 +4487,36 @@ "entityType": "fks", "table": "chat_runtime_snapshots" }, + { + "columns": [ + "job_run_id" + ], + "tableTo": "job_runs", + "columnsTo": [ + "id" + ], + "onUpdate": "RESTRICT", + "onDelete": "RESTRICT", + "nameExplicit": false, + "name": "fk_host_restart_claim_fence_job_run_id_job_runs_id_fk", + "entityType": "fks", + "table": "host_restart_claim_fence" + }, + { + "columns": [ + "worker_instance_id" + ], + "tableTo": "worker_instances", + "columnsTo": [ + "id" + ], + "onUpdate": "RESTRICT", + "onDelete": "RESTRICT", + "nameExplicit": false, + "name": "fk_host_restart_claim_fence_worker_instance_id_worker_instances_id_fk", + "entityType": "fks", + "table": "host_restart_claim_fence" + }, { "columns": [ "incident_id" @@ -4836,6 +4950,15 @@ "table": "chat_runtime_snapshots", "entityType": "pks" }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "host_restart_claim_fence_pk", + "table": "host_restart_claim_fence", + "entityType": "pks" + }, { "columns": [ "id" @@ -6092,6 +6215,28 @@ "entityType": "indexes", "table": "job_runs" }, + { + "columns": [ + { + "value": "action_key", + "isExpression": false + }, + { + "value": "\"queued_at\" desc", + "isExpression": true + }, + { + "value": "\"id\" desc", + "isExpression": true + } + ], + "isUnique": false, + "where": "\"job_runs\".\"action_key\" IN ('openclaw.sessions.cleanup', 'openclaw.gateway.restart', 'openclaw.installation.update', 'host.system.cleanup', 'host.system.restart', 'host.system.update') AND \"job_runs\".\"payload_json\" = '{}' AND \"job_runs\".\"state\" IN ('cancelled', 'failed', 'succeeded', 'timed-out')", + "origin": "manual", + "name": "job_runs_service_action_terminal_idx", + "entityType": "indexes", + "table": "job_runs" + }, { "columns": [ { @@ -7460,6 +7605,30 @@ "entityType": "checks", "table": "chat_transcript_generations" }, + { + "value": "length(\"boot_identity\") = 36 AND instr(\"boot_identity\", char(0)) = 0 AND length(replace(\"boot_identity\", '-', '')) = 32 AND replace(\"boot_identity\", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr(\"boot_identity\", 9, 1) = '-' AND substr(\"boot_identity\", 14, 1) = '-' AND substr(\"boot_identity\", 19, 1) = '-' AND substr(\"boot_identity\", 24, 1) = '-'", + "name": "host_restart_claim_fence_boot_identity_check", + "entityType": "checks", + "table": "host_restart_claim_fence" + }, + { + "value": "\"id\" = 1", + "name": "host_restart_claim_fence_id_check", + "entityType": "checks", + "table": "host_restart_claim_fence" + }, + { + "value": "length(\"lease_token\") = 36 AND instr(\"lease_token\", char(0)) = 0 AND length(replace(\"lease_token\", '-', '')) = 32 AND replace(\"lease_token\", '-', '') NOT GLOB '*[^0-9a-f]*' AND substr(\"lease_token\", 9, 1) = '-' AND substr(\"lease_token\", 14, 1) = '-' AND substr(\"lease_token\", 15, 1) = '7' AND substr(\"lease_token\", 19, 1) = '-' AND substr(\"lease_token\", 20, 1) GLOB '[89ab]' AND substr(\"lease_token\", 24, 1) = '-'", + "name": "host_restart_claim_fence_lease_token_check", + "entityType": "checks", + "table": "host_restart_claim_fence" + }, + { + "value": "\"armed_at\" BETWEEN 0 AND 8640000000000000 AND \"expires_at\" BETWEEN 0 AND 8640000000000000 AND \"expires_at\" > \"armed_at\"", + "name": "host_restart_claim_fence_time_check", + "entityType": "checks", + "table": "host_restart_claim_fence" + }, { "value": "CASE WHEN json_valid(\"details_json\") THEN json_type(\"details_json\") = 'object' ELSE 0 END", "name": "incident_observations_details_json_check", @@ -7700,6 +7869,12 @@ "entityType": "checks", "table": "job_runs" }, + { + "value": "\"required_worker_release_id\" IS NULL OR length(\"required_worker_release_id\") = 40 AND instr(\"required_worker_release_id\", char(0)) = 0 AND \"required_worker_release_id\" NOT GLOB '*[^0-9a-f]*'", + "name": "job_runs_required_worker_release_id_check", + "entityType": "checks", + "table": "job_runs" + }, { "value": "\"resource_class\" IN ('exclusive', 'host-heavy', 'interactive', 'light', 'network')", "name": "job_runs_resource_class_check", diff --git a/greenfield/scripts/delivery/buildRelease.test.ts b/greenfield/scripts/delivery/buildRelease.test.ts index 4b7f7fa57..dcba0d2e4 100644 --- a/greenfield/scripts/delivery/buildRelease.test.ts +++ b/greenfield/scripts/delivery/buildRelease.test.ts @@ -67,6 +67,11 @@ async function repositoryFixture(): Promise { path.join(repositoryRoot, "systemd"), { recursive: true } ), + cp( + path.join(sourceProjectRoot, "scripts/delivery/provisioning/host-operations"), + path.join(repositoryRoot, "scripts/delivery/provisioning/host-operations"), + { recursive: true } + ), cp( path.join(sourceProjectRoot, "scripts/delivery/provisioning/log-maintenance"), path.join(repositoryRoot, "scripts/delivery/provisioning/log-maintenance"), diff --git a/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.test.ts b/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.test.ts new file mode 100644 index 000000000..2f15da736 --- /dev/null +++ b/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "bun:test"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; + +import { + hostOperationsProvisioningArtifacts, + hostOperationsProvisioningReleaseArtifactPaths, +} from "./hostOperationsProvisioningPolicy.ts"; + +const sourceRoot = path.join(import.meta.dir, "provisioning/host-operations"); + +describe("host-operations provisioning artifact policy", () => { + test("inventories the seven exact root-owned artifacts and all installer support", async () => { + expect(hostOperationsProvisioningArtifacts).toEqual([ + { + artifactPath: + "scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules", + destinationPath: + "/etc/polkit-1/rules.d/60-mira-dashboard-host-operations.rules", + mode: 0o644, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation", + destinationPath: "/usr/local/libexec/mira-dashboard-host-operation", + mode: 0o755, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service", + destinationPath: + "/etc/systemd/system/mira-dashboard-deferred-reboot.service", + mode: 0o644, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer", + destinationPath: + "/etc/systemd/system/mira-dashboard-deferred-reboot.timer", + mode: 0o644, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service", + destinationPath: + "/etc/systemd/system/mira-dashboard-host-system-cleanup.service", + mode: 0o644, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service", + destinationPath: + "/etc/systemd/system/mira-dashboard-host-system-restart.service", + mode: 0o644, + }, + { + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service", + destinationPath: + "/etc/systemd/system/mira-dashboard-host-system-update.service", + mode: 0o644, + }, + ]); + const sourceFiles = await readdir(sourceRoot); + const sourceEntries = sourceFiles.toSorted(); + expect(sourceEntries).toEqual([ + "60-mira-dashboard-host-operations.rules", + "README.md", + "hostOperationsProvisioningFilesystem.ts", + "installHostOperationsProvisioning.ts", + "mira-dashboard-deferred-reboot.service", + "mira-dashboard-deferred-reboot.timer", + "mira-dashboard-host-operation", + "mira-dashboard-host-system-cleanup.service", + "mira-dashboard-host-system-restart.service", + "mira-dashboard-host-system-update.service", + "policy.ts", + ]); + expect(hostOperationsProvisioningReleaseArtifactPaths).toEqual( + sourceEntries.map( + (fileName) => `scripts/delivery/provisioning/host-operations/${fileName}` + ) + ); + for (const artifact of hostOperationsProvisioningArtifacts) { + const source = path.join( + path.resolve(import.meta.dir, "../.."), + artifact.artifactPath + ); + const status = await lstat(source, { bigint: true }); + expect(status.isFile()).toBe(true); + expect(status.isSymbolicLink()).toBe(false); + expect(status.nlink).toBe(1n); + expect(status.size).toBeGreaterThan(0n); + } + }); + + test("does not expose the root installer through an application-owned package script", async () => { + const packageJson = JSON.parse( + await readFile(path.resolve(import.meta.dir, "../../package.json"), "utf8") + ) as { readonly scripts?: Readonly> }; + + expect(packageJson.scripts).not.toHaveProperty( + "delivery:install-host-operations" + ); + }); +}); diff --git a/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.ts b/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.ts new file mode 100644 index 000000000..dadf27a16 --- /dev/null +++ b/greenfield/scripts/delivery/hostOperationsProvisioningPolicy.ts @@ -0,0 +1,7 @@ +/** Exact root-owned host-operation artifacts admitted into an immutable release. */ +export { + hostOperationsProvisioningArtifacts, + hostOperationsProvisioningReleaseArtifactPaths, + hostOperationsProvisioningSupportArtifactPaths, + type HostOperationsProvisioningArtifactPolicy, +} from "./provisioning/host-operations/policy.ts"; diff --git a/greenfield/scripts/delivery/installHostOperationsProvisioning.test.ts b/greenfield/scripts/delivery/installHostOperationsProvisioning.test.ts new file mode 100644 index 000000000..18eb54bb3 --- /dev/null +++ b/greenfield/scripts/delivery/installHostOperationsProvisioning.test.ts @@ -0,0 +1,524 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + chmod, + chown, + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { rejectionError } from "../testSupport/rejection.ts"; +import { + parseInstallHostOperationsProvisioningArguments, + runInstallHostOperationsProvisioningCli, +} from "./provisioning/host-operations/installHostOperationsProvisioning.ts"; +import { + hostOperationsProvisioningArtifacts, + hostOperationsProvisioningReleaseArtifactPaths, +} from "./provisioning/host-operations/policy.ts"; + +const releaseId = "a".repeat(40); +const sourceProjectRoot = path.resolve(import.meta.dir, "../.."); +const temporaryRoots: string[] = []; +const currentUserId = typeof process.getuid === "function" ? process.getuid() : -1; +const currentGroupId = typeof process.getgid === "function" ? process.getgid() : -1; +const fixtureSourceIdentity = Object.freeze({ + groupId: BigInt(currentGroupId), + userId: BigInt(currentUserId), +}); + +afterEach(async () => { + for (const temporaryRoot of temporaryRoots.splice(0)) { + await makeWritable(temporaryRoot); + await rm(temporaryRoot, { force: true, recursive: true }); + } +}); + +function sha256(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} + +async function makeWritable(entryPath: string): Promise { + const status = await lstat(entryPath); + if (!status.isDirectory() || status.isSymbolicLink()) return; + await chmod(entryPath, 0o700); + for (const entry of await readdir(entryPath, { withFileTypes: true })) { + if (entry.isDirectory() && !entry.isSymbolicLink()) { + await makeWritable(path.join(entryPath, entry.name)); + } + } +} + +async function changeOwnerRecursively( + entryPath: string, + userId: number, + groupId: number +): Promise { + const status = await lstat(entryPath); + if (status.isDirectory() && !status.isSymbolicLink()) { + for (const entry of await readdir(entryPath, { withFileTypes: true })) { + await changeOwnerRecursively( + path.join(entryPath, entry.name), + userId, + groupId + ); + } + } + await chown(entryPath, userId, groupId); +} + +async function releaseFixture( + options: Readonly<{ + readonly owner?: Readonly<{ readonly groupId: number; readonly userId: number }>; + }> = {} +): Promise { + const temporaryRoot = await mkdtemp( + path.join(tmpdir(), "mira-host-operations-release-") + ); + temporaryRoots.push(temporaryRoot); + const releaseRoot = path.join(temporaryRoot, releaseId); + const source = path.join( + sourceProjectRoot, + "scripts/delivery/provisioning/host-operations" + ); + const destination = path.join( + releaseRoot, + "scripts/delivery/provisioning/host-operations" + ); + await mkdir(releaseRoot, { recursive: true, mode: 0o700 }); + await cp(source, destination, { recursive: true }); + const artifacts = []; + for (const artifactPath of hostOperationsProvisioningReleaseArtifactPaths) { + const bytes = await readFile(path.join(releaseRoot, artifactPath)); + artifacts.push({ + bytes: bytes.byteLength, + path: artifactPath, + sha256: sha256(bytes), + }); + await chmod(path.join(releaseRoot, artifactPath), 0o400); + } + await writeFile( + path.join(releaseRoot, "release-manifest.json"), + `${JSON.stringify({ + artifacts, + formatVersion: 1, + source: { commitSha: releaseId, treeState: "clean" }, + })}\n`, + { mode: 0o400 } + ); + for (const directory of [ + "scripts/delivery/provisioning/host-operations", + "scripts/delivery/provisioning", + "scripts/delivery", + "scripts", + "", + ]) { + await chmod(path.join(releaseRoot, directory), 0o500); + } + if ( + options.owner !== undefined && + (options.owner.userId !== currentUserId || + options.owner.groupId !== currentGroupId) + ) { + await changeOwnerRecursively( + releaseRoot, + options.owner.userId, + options.owner.groupId + ); + } + return releaseRoot; +} + +async function runtimeBoundaryFixture(releaseRoot: string) { + const trustRoot = await mkdtemp(path.join(tmpdir(), "mira-host-operations-runtime-")); + temporaryRoots.push(trustRoot); + const runtimeDirectory = path.join(trustRoot, "runtime"); + const executablePath = path.join(runtimeDirectory, "bun"); + await mkdir(runtimeDirectory, { mode: 0o700 }); + await writeFile(executablePath, "fixed test runtime", { mode: 0o555 }); + await chmod(executablePath, 0o555); + await chmod(runtimeDirectory, 0o500); + await chmod(trustRoot, 0o500); + const entrypointPath = path.join( + releaseRoot, + "scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts" + ); + return Object.freeze({ + actualExecutablePath: executablePath, + actualEntrypointPath: entrypointPath, + expectedEntrypointPath: entrypointPath, + expectedExecutablePath: executablePath, + ...fixtureSourceIdentity, + trustRoot, + }); +} + +async function installerTestHooks( + releaseRoot: string, + destinationRoot?: string, + options: Readonly<{ readonly admitFixtureSource?: boolean }> = {} +) { + return { + ...(destinationRoot === undefined ? {} : { destinationRoot }), + ...(options.admitFixtureSource === false + ? {} + : { expectedSourceIdentity: fixtureSourceIdentity }), + requireRoot: () => {}, + runtimeBoundary: await runtimeBoundaryFixture(releaseRoot), + }; +} + +async function destinationFixture( + options: Readonly<{ readonly includeLibexec?: boolean }> = {} +): Promise { + const destinationRoot = await mkdtemp( + path.join(tmpdir(), "mira-host-operations-destination-") + ); + temporaryRoots.push(destinationRoot); + await chmod(destinationRoot, 0o700); + const directories = ["etc/polkit-1/rules.d", "etc/systemd/system"]; + if (options.includeLibexec === false) directories.push("usr/local"); + else directories.push("usr/local/libexec"); + for (const directory of directories) { + await mkdir(path.join(destinationRoot, directory), { + mode: 0o755, + recursive: true, + }); + } + return destinationRoot; +} + +async function argumentsFor(releaseRoot: string): Promise { + const manifestBytes = await readFile(path.join(releaseRoot, "release-manifest.json")); + return [ + `--release-root=${releaseRoot}`, + `--release-id=${releaseId}`, + `--release-manifest-sha256=${sha256(manifestBytes)}`, + ]; +} + +describe("root host-operations provisioning installer", () => { + test("installs exact manifest bytes atomically without activating host policy", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + + expect( + await runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ).toEqual({ releaseId, status: "INSTALLED" }); + await runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ); + + for (const artifact of hostOperationsProvisioningArtifacts) { + const installedPath = path.join( + destinationRoot, + artifact.destinationPath.slice(1) + ); + expect(await readFile(installedPath)).toEqual( + await readFile(path.join(releaseRoot, artifact.artifactPath)) + ); + const status = await lstat(installedPath); + expect(status.isFile()).toBe(true); + expect(status.isSymbolicLink()).toBe(false); + expect(status.nlink).toBe(1); + expect(status.mode & 0o7777).toBe(artifact.mode); + expect(status.uid).toBe( + typeof process.getuid === "function" ? process.getuid() : -1 + ); + expect(status.gid).toBe( + typeof process.getgid === "function" ? process.getgid() : -1 + ); + } + }); + + test("creates and validates the reviewed libexec target on a fresh host", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture({ includeLibexec: false }); + const libexec = path.join(destinationRoot, "usr/local/libexec"); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + + expect( + await runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ).toEqual({ releaseId, status: "INSTALLED" }); + + const status = await lstat(libexec); + expect(status.isDirectory()).toBeTrue(); + expect(status.isSymbolicLink()).toBeFalse(); + expect(status.mode & 0o7777).toBe(0o755); + expect( + await readFile(path.join(libexec, "mira-dashboard-host-operation")) + ).toEqual( + await readFile( + path.join( + releaseRoot, + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation" + ) + ) + ); + }); + + test("restores exact immutable release bytes during an explicit rollback reinstall", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + const arguments_ = await argumentsFor(releaseRoot); + await runInstallHostOperationsProvisioningCli(arguments_, hooks); + + const target = hostOperationsProvisioningArtifacts[1]; + const installedPath = path.join(destinationRoot, target.destinationPath.slice(1)); + await writeFile(installedPath, "superseded release bytes", { + mode: target.mode, + }); + + expect(await runInstallHostOperationsProvisioningCli(arguments_, hooks)).toEqual({ + releaseId, + status: "INSTALLED", + }); + expect(await readFile(installedPath)).toEqual( + await readFile(path.join(releaseRoot, target.artifactPath)) + ); + }); + + test("preflights every existing destination before creating libexec", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture({ includeLibexec: false }); + const libexec = path.join(destinationRoot, "usr/local/libexec"); + await chmod(path.join(destinationRoot, "etc/systemd/system"), 0o777); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + + const failure = await rejectionError( + runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ); + expect(failure.message).toBe("Host operations provisioning installation failed"); + const missing = await rejectionError(lstat(libexec)); + expect((missing as NodeJS.ErrnoException).code).toBe("ENOENT"); + }); + + test("preflights a later existing target file before creating libexec", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture({ includeLibexec: false }); + const libexec = path.join(destinationRoot, "usr/local/libexec"); + const systemdTarget = path.join( + destinationRoot, + "etc/systemd/system/mira-dashboard-host-system-update.service" + ); + await writeFile(systemdTarget, "unsafe", { mode: 0o600 }); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + + const failure = await rejectionError( + runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ); + expect(failure.message).toBe("Host operations provisioning installation failed"); + const missing = await rejectionError(lstat(libexec)); + expect((missing as NodeJS.ErrnoException).code).toBe("ENOENT"); + }); + + test("fails before replacement for untrusted destinations and target swaps", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const arguments_ = await argumentsFor(releaseRoot); + const baseHooks = await installerTestHooks(releaseRoot, destinationRoot); + await runInstallHostOperationsProvisioningCli(arguments_, baseHooks); + + const first = hostOperationsProvisioningArtifacts[0]; + const firstTarget = path.join(destinationRoot, first.destinationPath.slice(1)); + const displaced = `${firstTarget}.displaced`; + const swapFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(arguments_, { + ...baseHooks, + filesystem: { + async beforeRename(destinationPath) { + if (destinationPath !== first.destinationPath) return; + await rename(firstTarget, displaced); + await symlink(displaced, firstTarget); + }, + }, + }) + ); + expect(swapFailure.message).toBe( + "Host operations provisioning installation failed" + ); + + await rm(firstTarget); + await rename(displaced, firstTarget); + await chmod(path.dirname(firstTarget), 0o777); + const permissionFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(arguments_, baseHooks) + ); + expect(permissionFailure.message).toBe( + "Host operations provisioning installation failed" + ); + }); + + test("rejects changed release bytes, non-root execution, and extra arguments", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const arguments_ = await argumentsFor(releaseRoot); + const sourceArtifact = path.join( + releaseRoot, + hostOperationsProvisioningArtifacts[0].artifactPath + ); + await chmod(sourceArtifact, 0o600); + await writeFile(sourceArtifact, "changed"); + await chmod(sourceArtifact, 0o400); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + + const releaseFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(arguments_, hooks) + ); + expect(releaseFailure.message).toBe( + "Host operations provisioning installation failed" + ); + + const rootFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(arguments_, { + ...(await installerTestHooks(releaseRoot)), + requireRoot: () => { + throw new Error("not root"); + }, + }) + ); + expect(rootFailure.message).toBe( + "Host operations provisioning installation failed" + ); + expect(() => + parseInstallHostOperationsProvisioningArguments([ + ...arguments_, + "--destination-root=/tmp", + ]) + ).toThrow( + "Usage: bun installHostOperationsProvisioning.ts --release-root=/absolute/release/<40-hex> --release-id=<40-hex> --release-manifest-sha256=<64-hex>" + ); + }); + + test("requires one externally supplied exact manifest digest", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + const arguments_ = await argumentsFor(releaseRoot); + const digestIndex = arguments_.findIndex((argument) => + argument.startsWith("--release-manifest-sha256=") + ); + expect(digestIndex).toBeGreaterThanOrEqual(0); + + const wrongDigestArguments = [...arguments_]; + wrongDigestArguments[digestIndex] = `--release-manifest-sha256=${"f".repeat(64)}`; + const failure = await rejectionError( + runInstallHostOperationsProvisioningCli(wrongDigestArguments, hooks) + ); + expect(failure.message).toBe("Host operations provisioning installation failed"); + + expect(() => + parseInstallHostOperationsProvisioningArguments( + arguments_.filter( + (argument) => !argument.startsWith("--release-manifest-sha256=") + ) + ) + ).toThrow( + "Usage: bun installHostOperationsProvisioning.ts --release-root=/absolute/release/<40-hex> --release-id=<40-hex> --release-manifest-sha256=<64-hex>" + ); + }); + + test("rejects an internally consistent release not owned by root", async () => { + const nonRootOwner = + currentUserId === 0 && currentGroupId === 0 + ? { groupId: 65_534, userId: 65_534 } + : { groupId: currentGroupId, userId: currentUserId }; + const releaseRoot = await releaseFixture({ owner: nonRootOwner }); + const destinationRoot = await destinationFixture(); + const releaseStatus = await lstat(releaseRoot); + const hooks = await installerTestHooks(releaseRoot, destinationRoot, { + admitFixtureSource: false, + }); + expect([releaseStatus.uid, releaseStatus.gid]).not.toEqual([0, 0]); + + const failure = await rejectionError( + runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ); + expect(failure.message).toBe("Host operations provisioning installation failed"); + }); + + test("rejects a writable or unexpected provisioning runtime and entrypoint", async () => { + const releaseRoot = await releaseFixture(); + const destinationRoot = await destinationFixture(); + const hooks = await installerTestHooks(releaseRoot, destinationRoot); + await chmod(hooks.runtimeBoundary.expectedExecutablePath, 0o755); + + const writableFailure = await rejectionError( + runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ); + expect(writableFailure.message).toBe( + "Host operations provisioning installation failed" + ); + + await chmod(hooks.runtimeBoundary.expectedExecutablePath, 0o555); + const unexpectedFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(await argumentsFor(releaseRoot), { + ...hooks, + runtimeBoundary: { + ...hooks.runtimeBoundary, + actualExecutablePath: `${hooks.runtimeBoundary.actualExecutablePath}.other`, + }, + }) + ); + expect(unexpectedFailure.message).toBe( + "Host operations provisioning installation failed" + ); + + const entrypointFailure = await rejectionError( + runInstallHostOperationsProvisioningCli(await argumentsFor(releaseRoot), { + ...hooks, + runtimeBoundary: { + ...hooks.runtimeBoundary, + actualEntrypointPath: path.join( + releaseRoot, + "scripts/delivery/provisioning/host-operations/policy.ts" + ), + }, + }) + ); + expect(entrypointFailure.message).toBe( + "Host operations provisioning installation failed" + ); + + await chmod(hooks.runtimeBoundary.trustRoot, 0o520); + const ancestorFailure = await rejectionError( + runInstallHostOperationsProvisioningCli( + await argumentsFor(releaseRoot), + hooks + ) + ); + expect(ancestorFailure.message).toBe( + "Host operations provisioning installation failed" + ); + }); +}); diff --git a/greenfield/scripts/delivery/productionReleasePublication.test.ts b/greenfield/scripts/delivery/productionReleasePublication.test.ts index 95233ade3..c497b7669 100644 --- a/greenfield/scripts/delivery/productionReleasePublication.test.ts +++ b/greenfield/scripts/delivery/productionReleasePublication.test.ts @@ -97,6 +97,11 @@ async function repositoryFixture(): Promise { path.join(repositoryRoot, "systemd"), { recursive: true } ), + cp( + path.join(sourceProjectRoot, "scripts/delivery/provisioning/host-operations"), + path.join(repositoryRoot, "scripts/delivery/provisioning/host-operations"), + { recursive: true } + ), cp( path.join(sourceProjectRoot, "scripts/delivery/provisioning/log-maintenance"), path.join(repositoryRoot, "scripts/delivery/provisioning/log-maintenance"), diff --git a/greenfield/scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules b/greenfield/scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules new file mode 100644 index 000000000..ee0059b85 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules @@ -0,0 +1,19 @@ +polkit.addRule(function (action, subject) { + if ( + action.id !== "org.freedesktop.systemd1.manage-units" || + action.lookup("verb") !== "start" || + !subject.isInGroup("mira-dashboard-host-operations") + ) { + return polkit.Result.NOT_HANDLED; + } + + var unit = action.lookup("unit"); + var allowed = [ + "mira-dashboard-host-system-cleanup.service", + "mira-dashboard-host-system-restart.service", + "mira-dashboard-host-system-update.service" + ]; + return allowed.indexOf(unit) === -1 + ? polkit.Result.NOT_HANDLED + : polkit.Result.YES; +}); diff --git a/greenfield/scripts/delivery/provisioning/host-operations/README.md b/greenfield/scripts/delivery/provisioning/host-operations/README.md new file mode 100644 index 000000000..6750e0b17 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/README.md @@ -0,0 +1,68 @@ +# Host operations root provisioning + +This subtree contains the complete reviewed root boundary for three fixed operations: +`system-restart`, `system-update`, and `system-cleanup`. It does not expose a shell, +command, path, unit, environment, or output parameter. + +| Artifact | Destination | Owner/mode | +| -------------------------------------------- | ---------------------------------------------------------------- | ---------------- | +| `mira-dashboard-host-operation` | `/usr/local/libexec/mira-dashboard-host-operation` | `root:root 0755` | +| `mira-dashboard-host-system-restart.service` | `/etc/systemd/system/mira-dashboard-host-system-restart.service` | `root:root 0644` | +| `mira-dashboard-host-system-update.service` | `/etc/systemd/system/mira-dashboard-host-system-update.service` | `root:root 0644` | +| `mira-dashboard-host-system-cleanup.service` | `/etc/systemd/system/mira-dashboard-host-system-cleanup.service` | `root:root 0644` | +| `mira-dashboard-deferred-reboot.service` | `/etc/systemd/system/mira-dashboard-deferred-reboot.service` | `root:root 0644` | +| `mira-dashboard-deferred-reboot.timer` | `/etc/systemd/system/mira-dashboard-deferred-reboot.timer` | `root:root 0644` | +| `60-mira-dashboard-host-operations.rules` | `/etc/polkit-1/rules.d/60-mira-dashboard-host-operations.rules` | `root:root 0644` | + +Do not invoke the root installer against the application-owned production release tree. First +transfer the exact release into a dedicated root-owned immutable staging tree. The staged release +root and every directory traversed below it must be `root:root 0500`; `release-manifest.json` and +every admitted provisioning artifact must be `root:root 0400`. Preserve the exact release ID as the +staged root's basename. The installer rejects an internally consistent manifest when any admitted +source object is owned by the application user or group. + +Provision the exact Bun executable separately at +`/var/lib/mira-dashboard-host-provisioning/runtime/bun` as `root:root 0555`. Every ancestor of that +runtime path below the root-owned, non-group/other-writable +`/var/lib/mira-dashboard-host-provisioning` trust root must also be root-owned and not writable by +group or others. The root handoff must verify +the runtime and complete staged module tree before launch: Bun loads the entrypoint and its local +dependencies before in-process validation can run, so an application-owned interpreter or script +cannot establish its own authority. + +Before transfer or ownership change, independently verify the candidate against the reviewed Git +commit/tree or approved release record and obtain the exact `release-manifest.json` SHA-256 through +trusted change control. Do not compute the value from the application checkout as part of the root +install command: that would merely trust the same potentially forged source twice. The installer +requires this out-of-band digest and compares it to the held root-owned manifest bytes before +parsing any artifact hash. + +Install exact manifest-bound bytes only by invoking the staged script with that absolute runtime: + +```sh +/var/lib/mira-dashboard-host-provisioning/runtime/bun \ + /var/lib/mira-dashboard-host-provisioning/releases//scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts \ + --release-root=/var/lib/mira-dashboard-host-provisioning/releases/ \ + --release-id=<40-hex-commit> \ + --release-manifest-sha256= +``` + +There is deliberately no package-manager script for this root operation. Do not run the installer +from the application checkout, with the application user's Bun executable, or through a relative +module path. + +The installer does not reload systemd or polkit, create groups, enable timers, start an operation, +or compose the worker broker. The current web and worker user services share one Unix identity, so +that identity must never be added to `mira-dashboard-host-operations`: doing so would authorize the +web process too. A separately reviewed topology change must first move the worker to a distinct OS +principal. Only that principal may then join the fixed group before systemd/polkit reload and broker +composition. Until every step is complete, all three host operations remain unavailable. + +Rollback reinstalls the same seven files from the previous immutable release and reloads +systemd/polkit. The deferred timer is never enabled; it is started only by the root-owned +restart helper after systemd accepts the reviewed restart unit. + +Cleanup removes orphaned packages and stale package cache entries, rotates then vacuums +journald to fixed 14-day and 1 GiB limits, and prunes only unused Docker content older +than seven days. It never prunes Docker volumes. Every phase runs even when an earlier +phase fails, and the operation reports failure if any phase did not complete. diff --git a/greenfield/scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts b/greenfield/scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts new file mode 100644 index 000000000..7d367f3cc --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts @@ -0,0 +1,702 @@ +import { constants, type BigIntStats } from "node:fs"; +import { + lstat, + mkdir, + open, + realpath, + rename, + unlink, + type FileHandle, +} from "node:fs/promises"; +import path from "node:path"; + +import { + hostOperationsProvisioningArtifacts, + hostOperationsProvisioningCreatedDirectories, + type HostOperationsProvisioningArtifactPolicy, +} from "./policy.ts"; + +const installationFailureMessage = "Host operations provisioning installation failed"; +const directoryFlags = + constants.O_RDONLY | + constants.O_DIRECTORY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK; +const fileReadFlags = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK; +const temporaryFileFlags = + constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW | constants.O_RDWR; +const temporaryFileMode = 0o600; +const maximumArtifactBytes = 64 * 1024; + +/** Manifest-bound bytes for one exact root provisioning target. */ +export type VerifiedHostOperationsProvisioningFile = + HostOperationsProvisioningArtifactPolicy & { + readonly bytes: Uint8Array; + readonly sha256: string; + }; + +/** Deterministic race boundaries used only by adversarial filesystem tests. */ +export interface HostOperationsProvisioningFilesystemTestHooks { + readonly beforeRename?: (destinationPath: string) => Promise | void; +} + +interface OpenedDirectory { + readonly device: bigint; + readonly groupId: number; + readonly handle: FileHandle; + readonly inode: bigint; + readonly path: string; + readonly userId: number; +} + +interface PendingDirectoryCreation { + readonly expectedPath: string; + readonly mode: number; + readonly parent: OpenedDirectory; +} + +interface ExistingFileSnapshot { + readonly changeTimeNs: bigint; + readonly device: bigint; + readonly groupId: bigint; + readonly inode: bigint; + readonly mode: bigint; + readonly modifiedTimeNs: bigint; + readonly size: bigint; + readonly userId: bigint; +} + +interface StagedArtifact { + readonly destination: string; + readonly directory: OpenedDirectory; + readonly existing: ExistingFileSnapshot | undefined; + readonly file: VerifiedHostOperationsProvisioningFile; + readonly temporary: string; + renamed: boolean; +} + +function installationFailure(): Error { + return new Error(installationFailureMessage); +} + +function errorCode(error: unknown): string | undefined { + return error instanceof Error && "code" in error + ? (error as NodeJS.ErrnoException).code + : undefined; +} + +function sha256(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} + +function sameDirectoryIdentity(status: BigIntStats, directory: OpenedDirectory): boolean { + return status.dev === directory.device && status.ino === directory.inode; +} + +function validOwnedDirectory( + status: BigIntStats, + userId: number, + groupId: number +): boolean { + return ( + status.isDirectory() && + !status.isSymbolicLink() && + status.uid === BigInt(userId) && + status.gid === BigInt(groupId) && + (status.mode & 0o022n) === 0n + ); +} + +async function closeHandle(handle: FileHandle | undefined): Promise { + if (!handle) return true; + try { + await handle.close(); + return true; + } catch { + return false; + } +} + +async function openOwnedDirectory( + openPath: string, + expectedPath: string, + userId: number, + groupId: number +): Promise { + let handle: FileHandle | undefined; + let directory: OpenedDirectory | undefined; + try { + handle = await open(openPath, directoryFlags); + const [held, atPath, canonical] = await Promise.all([ + handle.stat({ bigint: true }), + lstat(expectedPath, { bigint: true }), + realpath(`/proc/self/fd/${handle.fd}`), + ]); + if ( + canonical !== expectedPath || + !validOwnedDirectory(held, userId, groupId) || + !validOwnedDirectory(atPath, userId, groupId) || + atPath.dev !== held.dev || + atPath.ino !== held.ino + ) { + throw installationFailure(); + } + directory = Object.freeze({ + device: held.dev, + groupId, + handle, + inode: held.ino, + path: expectedPath, + userId, + }); + } catch { + await closeHandle(handle); + throw installationFailure(); + } + return directory; +} + +async function validateOpenedDirectory(directory: OpenedDirectory): Promise { + const [held, atPath, canonical] = await Promise.all([ + directory.handle.stat({ bigint: true }), + lstat(directory.path, { bigint: true }), + realpath(`/proc/self/fd/${directory.handle.fd}`), + ]); + if ( + canonical !== directory.path || + !sameDirectoryIdentity(held, directory) || + !sameDirectoryIdentity(atPath, directory) || + !validOwnedDirectory(held, directory.userId, directory.groupId) || + !validOwnedDirectory(atPath, directory.userId, directory.groupId) + ) { + throw installationFailure(); + } +} + +async function openExistingOwnedDirectory( + parent: OpenedDirectory, + expectedPath: string, + userId: number, + groupId: number +): Promise { + const segment = path.basename(expectedPath); + const anchoredPath = path.join(`/proc/self/fd/${parent.handle.fd}`, segment); + try { + await lstat(anchoredPath, { bigint: true }); + } catch (error) { + if (errorCode(error) === "ENOENT") return undefined; + throw installationFailure(); + } + return openOwnedDirectory(anchoredPath, expectedPath, userId, groupId); +} + +async function openOrCreateOwnedDirectory( + parent: OpenedDirectory, + expectedPath: string, + userId: number, + groupId: number, + creationMode: number +): Promise { + const segment = path.basename(expectedPath); + const anchoredPath = path.join(`/proc/self/fd/${parent.handle.fd}`, segment); + const existing = await openExistingOwnedDirectory( + parent, + expectedPath, + userId, + groupId + ); + if (existing !== undefined) return existing; + + let created = false; + try { + await mkdir(anchoredPath, { mode: creationMode }); + created = true; + await parent.handle.sync(); + } catch (error) { + if (errorCode(error) !== "EEXIST") throw installationFailure(); + } + const directory = await openOwnedDirectory( + anchoredPath, + expectedPath, + userId, + groupId + ); + if (!created) return directory; + try { + const before = await directory.handle.stat({ bigint: true }); + await directory.handle.chmod(creationMode); + await directory.handle.sync(); + const after = await directory.handle.stat({ bigint: true }); + if ( + !sameDirectoryIdentity(before, directory) || + !sameDirectoryIdentity(after, directory) || + (after.mode & 0o7777n) !== BigInt(creationMode) + ) { + throw installationFailure(); + } + await validateOpenedDirectory(directory); + return directory; + } catch { + await closeHandle(directory.handle); + throw installationFailure(); + } +} + +function destinationBelowRoot(destinationRoot: string, destinationPath: string): string { + const relative = path.relative(path.parse(destinationPath).root, destinationPath); + if ( + !path.isAbsolute(destinationPath) || + path.resolve(destinationPath) !== destinationPath || + relative.length === 0 || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw installationFailure(); + } + return path.join(destinationRoot, relative); +} + +function validateFiles(files: readonly VerifiedHostOperationsProvisioningFile[]): void { + if ( + files.length !== hostOperationsProvisioningArtifacts.length || + files.some((file, index) => { + const expected = hostOperationsProvisioningArtifacts[index]; + return ( + expected === undefined || + file.artifactPath !== expected.artifactPath || + file.destinationPath !== expected.destinationPath || + file.mode !== expected.mode || + file.bytes.byteLength < 1 || + file.bytes.byteLength > maximumArtifactBytes || + !/^[a-f\d]{64}$/u.test(file.sha256) || + sha256(file.bytes) !== file.sha256 + ); + }) + ) { + throw installationFailure(); + } +} + +function snapshotExistingFile( + status: BigIntStats, + directory: OpenedDirectory, + expectedMode: number +): ExistingFileSnapshot { + if ( + !status.isFile() || + status.isSymbolicLink() || + status.nlink !== 1n || + status.uid !== BigInt(directory.userId) || + status.gid !== BigInt(directory.groupId) || + status.dev !== directory.device || + (status.mode & 0o7777n) !== BigInt(expectedMode) + ) { + throw installationFailure(); + } + return Object.freeze({ + changeTimeNs: status.ctimeNs, + device: status.dev, + groupId: status.gid, + inode: status.ino, + mode: status.mode, + modifiedTimeNs: status.mtimeNs, + size: status.size, + userId: status.uid, + }); +} + +async function existingFileSnapshot( + anchoredPath: string, + directory: OpenedDirectory, + expectedMode: number +): Promise { + try { + return snapshotExistingFile( + await lstat(anchoredPath, { bigint: true }), + directory, + expectedMode + ); + } catch (error) { + if (errorCode(error) === "ENOENT") return undefined; + throw installationFailure(); + } +} + +function sameExistingFile( + left: ExistingFileSnapshot | undefined, + right: ExistingFileSnapshot | undefined +): boolean { + if (!left || !right) return left === right; + return ( + left.changeTimeNs === right.changeTimeNs && + left.device === right.device && + left.groupId === right.groupId && + left.inode === right.inode && + left.mode === right.mode && + left.modifiedTimeNs === right.modifiedTimeNs && + left.size === right.size && + left.userId === right.userId + ); +} + +async function writeAll(handle: FileHandle, bytes: Uint8Array): Promise { + let offset = 0; + while (offset < bytes.byteLength) { + const written = await handle.write( + bytes, + offset, + bytes.byteLength - offset, + offset + ); + if (written.bytesWritten < 1) throw installationFailure(); + offset += written.bytesWritten; + } +} + +async function readExactFile( + anchoredPath: string, + expected: VerifiedHostOperationsProvisioningFile, + directory: OpenedDirectory +): Promise { + let handle: FileHandle | undefined; + let failed = false; + try { + handle = await open(anchoredPath, fileReadFlags); + const held = await handle.stat({ bigint: true }); + if ( + !held.isFile() || + held.isSymbolicLink() || + held.nlink !== 1n || + held.uid !== BigInt(directory.userId) || + held.gid !== BigInt(directory.groupId) || + held.dev !== directory.device || + held.size !== BigInt(expected.bytes.byteLength) || + (held.mode & 0o7777n) !== BigInt(expected.mode) + ) { + throw installationFailure(); + } + const contents = Buffer.alloc(expected.bytes.byteLength + 1); + let offset = 0; + while (offset < contents.byteLength) { + const read = await handle.read( + contents, + offset, + contents.byteLength - offset, + offset + ); + if (read.bytesRead === 0) break; + offset += read.bytesRead; + } + const [heldAfter, atPath] = await Promise.all([ + handle.stat({ bigint: true }), + lstat(anchoredPath, { bigint: true }), + ]); + if ( + offset !== expected.bytes.byteLength || + heldAfter.dev !== held.dev || + heldAfter.ino !== held.ino || + heldAfter.ctimeNs !== held.ctimeNs || + heldAfter.mtimeNs !== held.mtimeNs || + heldAfter.size !== held.size || + atPath.dev !== held.dev || + atPath.ino !== held.ino || + sha256(contents.subarray(0, offset)) !== expected.sha256 + ) { + throw installationFailure(); + } + } catch { + failed = true; + } + if (!(await closeHandle(handle))) failed = true; + if (failed) throw installationFailure(); +} + +async function stageArtifact( + directory: OpenedDirectory, + destination: string, + file: VerifiedHostOperationsProvisioningFile +): Promise { + const descriptorRoot = `/proc/self/fd/${directory.handle.fd}`; + const anchoredDestination = path.join(descriptorRoot, path.basename(destination)); + const temporary = path.join( + descriptorRoot, + `.mira-host-operations.${Bun.randomUUIDv7()}.tmp` + ); + const existing = await existingFileSnapshot( + anchoredDestination, + directory, + file.mode + ); + let handle: FileHandle | undefined; + let failed = false; + try { + handle = await open(temporary, temporaryFileFlags, temporaryFileMode); + await handle.chmod(file.mode); + await writeAll(handle, file.bytes); + await handle.sync(); + const status = await handle.stat({ bigint: true }); + if ( + !status.isFile() || + status.nlink !== 1n || + status.uid !== BigInt(directory.userId) || + status.gid !== BigInt(directory.groupId) || + status.dev !== directory.device || + status.size !== BigInt(file.bytes.byteLength) || + (status.mode & 0o7777n) !== BigInt(file.mode) + ) { + throw installationFailure(); + } + } catch { + failed = true; + } + if (!(await closeHandle(handle))) failed = true; + if (failed) { + try { + await unlink(temporary); + } catch { + // The operation is already failing; no temporary path is ever reused. + } + throw installationFailure(); + } + try { + await readExactFile(temporary, file, directory); + } catch { + try { + await unlink(temporary); + } catch { + // The operation is already failing; no temporary path is ever reused. + } + throw installationFailure(); + } + return { + destination: anchoredDestination, + directory, + existing, + file, + renamed: false, + temporary, + }; +} + +async function openDestinationDirectories( + destinationRoot: string, + files: readonly VerifiedHostOperationsProvisioningFile[], + userId: number, + groupId: number +): Promise<{ + readonly directories: readonly OpenedDirectory[]; + readonly targetDirectories: ReadonlyMap; +}> { + const directories: OpenedDirectory[] = []; + const byPath = new Map(); + const pendingCreations = new Map(); + const root = await openOwnedDirectory( + destinationRoot, + destinationRoot, + userId, + groupId + ); + directories.push(root); + byPath.set(destinationRoot, root); + const creatableDirectories = new Map( + hostOperationsProvisioningCreatedDirectories.map((directory) => [ + destinationBelowRoot(destinationRoot, directory.destinationPath), + directory.mode, + ]) + ); + try { + for (const file of files) { + const targetDirectory = path.dirname( + destinationBelowRoot(destinationRoot, file.destinationPath) + ); + const relative = path.relative(destinationRoot, targetDirectory); + if ( + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw installationFailure(); + } + let current = root; + let currentPath = destinationRoot; + const segments = relative.split(path.sep).filter(Boolean); + for (const [index, segment] of segments.entries()) { + currentPath = path.join(currentPath, segment); + const existing = byPath.get(currentPath); + if (existing) { + current = existing; + continue; + } + const pending = pendingCreations.get(currentPath); + if (pending !== undefined) { + if (index !== segments.length - 1) throw installationFailure(); + continue; + } + const opened = await openExistingOwnedDirectory( + current, + currentPath, + userId, + groupId + ); + if (opened !== undefined) { + current = opened; + directories.push(current); + byPath.set(currentPath, current); + continue; + } + const mode = creatableDirectories.get(currentPath); + if (mode === undefined || index !== segments.length - 1) { + throw installationFailure(); + } + pendingCreations.set( + currentPath, + Object.freeze({ expectedPath: currentPath, mode, parent: current }) + ); + } + } + for (const directory of directories) { + await validateOpenedDirectory(directory); + } + for (const file of files) { + const destination = destinationBelowRoot( + destinationRoot, + file.destinationPath + ); + const targetDirectory = path.dirname(destination); + const directory = byPath.get(targetDirectory); + if (directory === undefined) { + if (!pendingCreations.has(targetDirectory)) { + throw installationFailure(); + } + continue; + } + await existingFileSnapshot( + path.join( + `/proc/self/fd/${directory.handle.fd}`, + path.basename(destination) + ), + directory, + file.mode + ); + } + for (const creation of pendingCreations.values()) { + await validateOpenedDirectory(creation.parent); + const created = await openOrCreateOwnedDirectory( + creation.parent, + creation.expectedPath, + userId, + groupId, + creation.mode + ); + directories.push(created); + byPath.set(creation.expectedPath, created); + } + return Object.freeze({ + directories: Object.freeze(directories), + targetDirectories: byPath, + }); + } catch { + for (const directory of directories.toReversed()) { + await closeHandle(directory.handle); + } + throw installationFailure(); + } +} + +/** + * Installs all seven manifest-bound files through held destination descriptors. + * Source verification and a non-mutating preflight of every existing destination + * directory and target file complete before the one reviewed support directory may + * be created. Every file is replaced atomically; no service or policy daemon is activated. + * @param destinationRoot `/` in production or one explicit test-only filesystem root. + * @param files Exact ordered source bytes and manifest hashes. + * @param testHooks Deterministic mutation boundaries for adversarial tests. + */ +export async function installHostOperationsProvisioningFiles( + destinationRoot: string, + files: readonly VerifiedHostOperationsProvisioningFile[], + testHooks: HostOperationsProvisioningFilesystemTestHooks = {} +): Promise { + if ( + process.platform !== "linux" || + typeof process.getuid !== "function" || + typeof process.getgid !== "function" || + !path.isAbsolute(destinationRoot) || + destinationRoot.includes("\0") || + destinationRoot.length > 4096 || + path.resolve(destinationRoot) !== destinationRoot + ) { + throw installationFailure(); + } + validateFiles(files); + const opened = await openDestinationDirectories( + destinationRoot, + files, + process.getuid(), + process.getgid() + ); + const staged: StagedArtifact[] = []; + let failed = false; + try { + for (const file of files) { + const destination = destinationBelowRoot( + destinationRoot, + file.destinationPath + ); + const directory = opened.targetDirectories.get(path.dirname(destination)); + if (!directory) throw installationFailure(); + staged.push(await stageArtifact(directory, destination, file)); + } + + for (const directory of opened.directories) { + await validateOpenedDirectory(directory); + } + for (const artifact of staged) { + const current = await existingFileSnapshot( + artifact.destination, + artifact.directory, + artifact.file.mode + ); + if (!sameExistingFile(artifact.existing, current)) { + throw installationFailure(); + } + await readExactFile(artifact.temporary, artifact.file, artifact.directory); + } + + for (const artifact of staged) { + await testHooks.beforeRename?.(artifact.file.destinationPath); + await validateOpenedDirectory(artifact.directory); + const current = await existingFileSnapshot( + artifact.destination, + artifact.directory, + artifact.file.mode + ); + if (!sameExistingFile(artifact.existing, current)) { + throw installationFailure(); + } + await rename(artifact.temporary, artifact.destination); + artifact.renamed = true; + await artifact.directory.handle.sync(); + await readExactFile(artifact.destination, artifact.file, artifact.directory); + } + + for (const directory of opened.directories) { + await validateOpenedDirectory(directory); + } + } catch { + failed = true; + } + for (const artifact of staged) { + if (artifact.renamed) continue; + try { + await unlink(artifact.temporary); + } catch (error) { + if (errorCode(error) !== "ENOENT") failed = true; + } + } + for (const directory of opened.directories.toReversed()) { + if (!(await closeHandle(directory.handle))) failed = true; + } + if (failed) throw installationFailure(); +} diff --git a/greenfield/scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts b/greenfield/scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts new file mode 100644 index 000000000..6b8341d6b --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts @@ -0,0 +1,699 @@ +import { constants, type BigIntStats } from "node:fs"; +import { lstat, open, realpath, type FileHandle } from "node:fs/promises"; +import path from "node:path"; + +import { + installHostOperationsProvisioningFiles, + type HostOperationsProvisioningFilesystemTestHooks, + type VerifiedHostOperationsProvisioningFile, +} from "./hostOperationsProvisioningFilesystem.ts"; +import { + hostOperationsProvisioningArtifacts, + hostOperationsProvisioningReleaseArtifactPaths, +} from "./policy.ts"; + +const installationFailureMessage = "Host operations provisioning installation failed"; +const installationUsage = + "Usage: bun installHostOperationsProvisioning.ts --release-root=/absolute/release/<40-hex> --release-id=<40-hex> --release-manifest-sha256=<64-hex>"; +const directoryFlags = + constants.O_RDONLY | + constants.O_DIRECTORY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK; +const fileFlags = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK; +const immutableDirectoryMode = 0o500n; +const immutableFileMode = 0o400n; +const maximumManifestBytes = 4 * 1024 * 1024; +const maximumProvisioningArtifactBytes = 64 * 1024; +const maximumArtifactCount = 4096; +const commitShaPattern = /^[a-f\d]{40}$/u; +const artifactShaPattern = /^[a-f\d]{64}$/u; +const artifactSegmentPattern = /^[A-Za-z0-9.@_+-]+$/u; +const provisioningPrefix = "scripts/delivery/provisioning/host-operations/"; +const productionSourceIdentity = Object.freeze({ groupId: 0n, userId: 0n }); +const productionRuntimeExecutablePath = + "/var/lib/mira-dashboard-host-provisioning/runtime/bun"; +const installerRelativePath = + "scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts"; + +interface ReleaseArtifactRecord { + readonly bytes: number; + readonly path: string; + readonly sha256: string; +} + +interface ReleaseDirectory { + readonly device: bigint; + readonly groupId: bigint; + readonly handle: FileHandle; + readonly inode: bigint; + readonly path: string; + readonly userId: bigint; +} + +interface LoadedProvisioningRelease { + readonly files: readonly VerifiedHostOperationsProvisioningFile[]; + readonly identity: string; +} + +interface HostOperationsProvisioningSourceIdentity { + readonly groupId: bigint; + readonly userId: bigint; +} + +interface HostOperationsProvisioningRuntimeBoundary extends HostOperationsProvisioningSourceIdentity { + readonly actualExecutablePath: string; + readonly actualEntrypointPath: string; + readonly expectedEntrypointPath: string; + readonly expectedExecutablePath: string; + readonly trustRoot: string; +} + +const productionRuntimeBoundary = Object.freeze({ + actualExecutablePath: process.execPath, + actualEntrypointPath: import.meta.path, + expectedEntrypointPath: "", + expectedExecutablePath: productionRuntimeExecutablePath, + ...productionSourceIdentity, + trustRoot: "/var/lib/mira-dashboard-host-provisioning", +}); + +/** Deterministic non-production boundaries used only by focused installer tests. */ +export interface InstallHostOperationsProvisioningTestHooks { + readonly destinationRoot?: string; + readonly expectedSourceIdentity?: HostOperationsProvisioningSourceIdentity; + readonly filesystem?: HostOperationsProvisioningFilesystemTestHooks; + readonly requireRoot?: () => void; + readonly runtimeBoundary?: HostOperationsProvisioningRuntimeBoundary; +} + +/** Exact root installer CLI inputs. */ +export interface InstallHostOperationsProvisioningArguments { + readonly releaseId: string; + readonly releaseManifestSha256: string; + readonly releaseRoot: string; +} + +/** Redacted root installer result; no host paths or artifact details are exposed. */ +export interface InstallHostOperationsProvisioningResult { + readonly releaseId: string; + readonly status: "INSTALLED"; +} + +function installationFailure(): Error { + return new Error(installationFailureMessage); +} + +function sha256(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} + +async function closeHandle(handle: FileHandle | undefined): Promise { + if (!handle) return true; + try { + await handle.close(); + return true; + } catch { + return false; + } +} + +function validReleaseDirectory( + status: BigIntStats, + identity: HostOperationsProvisioningSourceIdentity, + device?: bigint +): boolean { + return ( + status.isDirectory() && + !status.isSymbolicLink() && + (status.mode & 0o7777n) === immutableDirectoryMode && + status.gid === identity.groupId && + status.uid === identity.userId && + (device === undefined || status.dev === device) + ); +} + +async function openReleaseDirectory( + openPath: string, + expectedPath: string, + identity: HostOperationsProvisioningSourceIdentity, + device?: bigint +): Promise { + let handle: FileHandle | undefined; + try { + handle = await open(openPath, directoryFlags); + const [held, atPath, canonical] = await Promise.all([ + handle.stat({ bigint: true }), + lstat(expectedPath, { bigint: true }), + realpath(`/proc/self/fd/${handle.fd}`), + ]); + if ( + canonical !== expectedPath || + !validReleaseDirectory(held, identity, device) || + !validReleaseDirectory(atPath, identity, device) || + atPath.dev !== held.dev || + atPath.ino !== held.ino + ) { + throw installationFailure(); + } + return Object.freeze({ + device: held.dev, + groupId: held.gid, + handle, + inode: held.ino, + path: expectedPath, + userId: held.uid, + }); + } catch { + await closeHandle(handle); + throw installationFailure(); + } +} + +function validArtifactPath(artifactPath: string): boolean { + const segments = artifactPath.split("/"); + return ( + artifactPath.length > 0 && + artifactPath.length <= 4096 && + !path.isAbsolute(artifactPath) && + segments.every( + (segment) => + segment.length > 0 && + segment !== "." && + segment !== ".." && + artifactSegmentPattern.test(segment) + ) + ); +} + +function pathIsWithin(parent: string, child: string): boolean { + const relative = path.relative(parent, child); + return ( + relative.length > 0 && !relative.startsWith(`..${path.sep}`) && relative !== ".." + ); +} + +async function validateRuntimeBoundary( + boundary: HostOperationsProvisioningRuntimeBoundary, + releaseRoot: string +): Promise { + const { + actualEntrypointPath, + actualExecutablePath, + expectedEntrypointPath, + expectedExecutablePath, + trustRoot, + } = boundary; + const releaseEntrypointPath = path.join(releaseRoot, installerRelativePath); + if ( + !path.isAbsolute(actualEntrypointPath) || + !path.isAbsolute(actualExecutablePath) || + !path.isAbsolute(expectedEntrypointPath) || + !path.isAbsolute(expectedExecutablePath) || + !path.isAbsolute(trustRoot) || + path.resolve(actualEntrypointPath) !== actualEntrypointPath || + path.resolve(actualExecutablePath) !== actualExecutablePath || + path.resolve(expectedEntrypointPath) !== expectedEntrypointPath || + path.resolve(expectedExecutablePath) !== expectedExecutablePath || + path.resolve(trustRoot) !== trustRoot || + actualEntrypointPath !== expectedEntrypointPath || + expectedEntrypointPath !== releaseEntrypointPath || + actualExecutablePath !== expectedExecutablePath || + !pathIsWithin(trustRoot, expectedExecutablePath) + ) { + throw installationFailure(); + } + + const relativeExecutablePath = path.relative(trustRoot, expectedExecutablePath); + const segments = relativeExecutablePath.split(path.sep); + const executableName = segments.pop(); + if (!executableName || segments.some((segment) => !segment)) { + throw installationFailure(); + } + + let currentPath = trustRoot; + for (const segment of ["", ...segments]) { + if (segment) currentPath = path.join(currentPath, segment); + const [status, canonical] = await Promise.all([ + lstat(currentPath, { bigint: true }), + realpath(currentPath), + ]); + if ( + canonical !== currentPath || + !status.isDirectory() || + status.isSymbolicLink() || + status.uid !== boundary.userId || + status.gid !== boundary.groupId || + (status.mode & 0o022n) !== 0n + ) { + throw installationFailure(); + } + } + + const [executable, canonicalExecutable] = await Promise.all([ + lstat(expectedExecutablePath, { bigint: true }), + realpath(expectedExecutablePath), + ]); + if ( + canonicalExecutable !== expectedExecutablePath || + !executable.isFile() || + executable.isSymbolicLink() || + executable.nlink !== 1n || + executable.uid !== boundary.userId || + executable.gid !== boundary.groupId || + (executable.mode & 0o7777n) !== 0o555n + ) { + throw installationFailure(); + } + + const [entrypoint, canonicalEntrypoint] = await Promise.all([ + lstat(expectedEntrypointPath, { bigint: true }), + realpath(expectedEntrypointPath), + ]); + if ( + canonicalEntrypoint !== expectedEntrypointPath || + !entrypoint.isFile() || + entrypoint.isSymbolicLink() || + entrypoint.nlink !== 1n || + entrypoint.uid !== boundary.userId || + entrypoint.gid !== boundary.groupId || + (entrypoint.mode & 0o7777n) !== immutableFileMode + ) { + throw installationFailure(); + } +} + +function recordValue(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function exactKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).toSorted(); + return ( + actual.length === expected.length && + expected.every((key, index) => actual[index] === key) + ); +} + +function parseManifestArtifacts( + manifestBytes: Uint8Array, + releaseId: string +): readonly ReleaseArtifactRecord[] { + let parsed: unknown; + try { + parsed = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(manifestBytes) + ); + } catch { + throw installationFailure(); + } + const manifest = recordValue(parsed); + const source = recordValue(manifest?.source); + if ( + !manifest || + !source || + manifest.formatVersion !== 1 || + source.commitSha !== releaseId || + source.treeState !== "clean" || + !Array.isArray(manifest.artifacts) || + manifest.artifacts.length === 0 || + manifest.artifacts.length > maximumArtifactCount + ) { + throw installationFailure(); + } + const records: ReleaseArtifactRecord[] = []; + for (const value of manifest.artifacts) { + const record = recordValue(value); + if ( + !record || + !exactKeys(record, ["bytes", "path", "sha256"]) || + typeof record.bytes !== "number" || + !Number.isSafeInteger(record.bytes) || + record.bytes < 1 || + typeof record.path !== "string" || + !validArtifactPath(record.path) || + typeof record.sha256 !== "string" || + !artifactShaPattern.test(record.sha256) + ) { + throw installationFailure(); + } + records.push( + Object.freeze({ + bytes: record.bytes, + path: record.path, + sha256: record.sha256, + }) + ); + } + if ( + records.some( + (record, index) => + index > 0 && record.path <= (records[index - 1]?.path ?? "") + ) + ) { + throw installationFailure(); + } + const provisioningPaths = records + .filter(({ path: artifactPath }) => artifactPath.startsWith(provisioningPrefix)) + .map(({ path: artifactPath }) => artifactPath); + if ( + provisioningPaths.length !== + hostOperationsProvisioningReleaseArtifactPaths.length || + hostOperationsProvisioningReleaseArtifactPaths.some( + (expected, index) => provisioningPaths[index] !== expected + ) + ) { + throw installationFailure(); + } + return Object.freeze(records); +} + +async function readHeldFile( + directory: ReleaseDirectory, + fileName: string, + maximumBytes: number, + expected?: ReleaseArtifactRecord +): Promise { + const anchoredPath = path.join(`/proc/self/fd/${directory.handle.fd}`, fileName); + const expectedPath = path.join(directory.path, fileName); + let handle: FileHandle | undefined; + let output: Uint8Array | undefined; + try { + handle = await open(anchoredPath, fileFlags); + const [held, atPath, canonical] = await Promise.all([ + handle.stat({ bigint: true }), + lstat(expectedPath, { bigint: true }), + realpath(`/proc/self/fd/${handle.fd}`), + ]); + const expectedBytes = expected?.bytes; + if ( + canonical !== expectedPath || + !held.isFile() || + held.isSymbolicLink() || + held.nlink !== 1n || + held.uid !== directory.userId || + held.gid !== directory.groupId || + held.dev !== directory.device || + held.size < 1n || + held.size > BigInt(maximumBytes) || + (held.mode & 0o7777n) !== immutableFileMode || + atPath.dev !== held.dev || + atPath.ino !== held.ino || + (expectedBytes !== undefined && held.size !== BigInt(expectedBytes)) + ) { + throw installationFailure(); + } + const bytes = Buffer.alloc(Number(held.size) + 1); + let offset = 0; + while (offset < bytes.byteLength) { + const read = await handle.read( + bytes, + offset, + bytes.byteLength - offset, + offset + ); + if (read.bytesRead === 0) break; + offset += read.bytesRead; + } + const [heldAfter, atPathAfter] = await Promise.all([ + handle.stat({ bigint: true }), + lstat(expectedPath, { bigint: true }), + ]); + if ( + offset !== Number(held.size) || + heldAfter.dev !== held.dev || + heldAfter.ino !== held.ino || + heldAfter.size !== held.size || + heldAfter.ctimeNs !== held.ctimeNs || + heldAfter.mtimeNs !== held.mtimeNs || + atPathAfter.dev !== held.dev || + atPathAfter.ino !== held.ino + ) { + throw installationFailure(); + } + output = bytes.subarray(0, offset); + if (expected && sha256(output) !== expected.sha256) { + throw installationFailure(); + } + } catch { + await closeHandle(handle); + throw installationFailure(); + } + if (!(await closeHandle(handle)) || !output) throw installationFailure(); + return output; +} + +async function openReleaseDirectories( + releaseRoot: string, + relativeDirectories: readonly string[], + expectedSourceIdentity: HostOperationsProvisioningSourceIdentity +): Promise> { + const opened = new Map(); + const root = await openReleaseDirectory( + releaseRoot, + releaseRoot, + expectedSourceIdentity + ); + opened.set("", root); + try { + for (const relativeDirectory of relativeDirectories) { + let current = root; + let currentRelative = ""; + for (const segment of relativeDirectory.split("/").filter(Boolean)) { + currentRelative = currentRelative + ? `${currentRelative}/${segment}` + : segment; + const existing = opened.get(currentRelative); + if (existing) { + current = existing; + continue; + } + const expectedPath = path.join(releaseRoot, currentRelative); + current = await openReleaseDirectory( + path.join(`/proc/self/fd/${current.handle.fd}`, segment), + expectedPath, + root, + root.device + ); + opened.set(currentRelative, current); + } + } + return opened; + } catch { + for (const directory of [...opened.values()].toReversed()) { + await closeHandle(directory.handle); + } + throw installationFailure(); + } +} + +async function loadProvisioningRelease( + releaseRoot: string, + releaseId: string, + releaseManifestSha256: string, + expectedSourceIdentity: HostOperationsProvisioningSourceIdentity +): Promise { + const provisioningDirectory = provisioningPrefix.slice(0, -1); + const directories = await openReleaseDirectories( + releaseRoot, + [provisioningDirectory], + expectedSourceIdentity + ); + let loaded: LoadedProvisioningRelease | undefined; + let failed = false; + try { + const root = directories.get(""); + const source = directories.get(provisioningDirectory); + if (!root || !source) throw installationFailure(); + const manifestBytes = await readHeldFile( + root, + "release-manifest.json", + maximumManifestBytes + ); + if (sha256(manifestBytes) !== releaseManifestSha256) { + throw installationFailure(); + } + const artifacts = parseManifestArtifacts(manifestBytes, releaseId); + const byPath = new Map(artifacts.map((artifact) => [artifact.path, artifact])); + const sourceBytes = new Map(); + for (const artifactPath of hostOperationsProvisioningReleaseArtifactPaths) { + const record = byPath.get(artifactPath); + if (!record || record.bytes > maximumProvisioningArtifactBytes) { + throw installationFailure(); + } + sourceBytes.set( + artifactPath, + await readHeldFile( + source, + artifactPath.slice(provisioningPrefix.length), + maximumProvisioningArtifactBytes, + record + ) + ); + } + const files = hostOperationsProvisioningArtifacts.map((policy) => { + const record = byPath.get(policy.artifactPath); + const bytes = sourceBytes.get(policy.artifactPath); + if (!record || !bytes) throw installationFailure(); + return Object.freeze({ + ...policy, + bytes, + sha256: record.sha256, + }); + }); + loaded = Object.freeze({ + files: Object.freeze(files), + identity: sha256(manifestBytes), + }); + } catch { + failed = true; + } + for (const directory of [...directories.values()].toReversed()) { + if (!(await closeHandle(directory.handle))) failed = true; + } + if (failed || !loaded) throw installationFailure(); + return loaded; +} + +function sameRelease( + left: LoadedProvisioningRelease, + right: LoadedProvisioningRelease +): boolean { + return ( + left.identity === right.identity && + left.files.length === right.files.length && + left.files.every( + (file, index) => + file.artifactPath === right.files[index]?.artifactPath && + file.sha256 === right.files[index]?.sha256 && + file.bytes.byteLength === right.files[index]?.bytes.byteLength + ) + ); +} + +function requireRoot(): void { + if ( + process.platform !== "linux" || + typeof process.getuid !== "function" || + typeof process.getgid !== "function" || + process.getuid() !== 0 || + process.getgid() !== 0 + ) { + throw installationFailure(); + } +} + +function readNamedArguments(arguments_: readonly string[]): Record { + const values = Object.create(null) as Record; + for (const argument of arguments_) { + const separator = argument.indexOf("="); + if (separator <= 2 || !argument.startsWith("--")) { + throw new TypeError(installationUsage); + } + const name = argument.slice(2, separator); + const value = argument.slice(separator + 1); + if (!value || Object.hasOwn(values, name)) { + throw new TypeError(installationUsage); + } + values[name] = value; + } + return values; +} + +/** + * Parses exactly one immutable release root and its commit identity. + * @param arguments_ Exact named CLI arguments after the Bun entrypoint. + * @returns Frozen, canonical release identity arguments. + */ +export function parseInstallHostOperationsProvisioningArguments( + arguments_: readonly string[] +): InstallHostOperationsProvisioningArguments { + if (arguments_.length !== 3) throw new TypeError(installationUsage); + const values = readNamedArguments(arguments_); + const releaseId = values["release-id"]; + const releaseManifestSha256 = values["release-manifest-sha256"]; + const releaseRoot = values["release-root"]; + if ( + !releaseId || + !commitShaPattern.test(releaseId) || + !releaseManifestSha256 || + !artifactShaPattern.test(releaseManifestSha256) || + !releaseRoot || + !path.isAbsolute(releaseRoot) || + releaseRoot.includes("\0") || + releaseRoot.length > 4096 || + path.resolve(releaseRoot) !== releaseRoot || + path.parse(releaseRoot).root === releaseRoot || + path.basename(releaseRoot) !== releaseId || + Object.keys(values).length !== 3 + ) { + throw new TypeError(installationUsage); + } + return Object.freeze({ releaseId, releaseManifestSha256, releaseRoot }); +} + +/** + * Verifies one frozen release before and after atomically installing exact root files. + * This deliberately performs no daemon reload, group mutation, enablement, or service start. + * @param arguments_ Exact release-root and release-id CLI arguments. + * @param testHooks Deterministic non-production filesystem and identity boundaries. + * @returns A redacted installed release identity. + */ +export async function runInstallHostOperationsProvisioningCli( + arguments_: readonly string[], + testHooks: InstallHostOperationsProvisioningTestHooks = {} +): Promise { + const parsed = parseInstallHostOperationsProvisioningArguments(arguments_); + try { + (testHooks.requireRoot ?? requireRoot)(); + const runtimeBoundary = testHooks.runtimeBoundary ?? { + ...productionRuntimeBoundary, + expectedEntrypointPath: path.join(parsed.releaseRoot, installerRelativePath), + }; + await validateRuntimeBoundary(runtimeBoundary, parsed.releaseRoot); + const expectedSourceIdentity = + testHooks.expectedSourceIdentity ?? productionSourceIdentity; + const first = await loadProvisioningRelease( + parsed.releaseRoot, + parsed.releaseId, + parsed.releaseManifestSha256, + expectedSourceIdentity + ); + const preflight = await loadProvisioningRelease( + parsed.releaseRoot, + parsed.releaseId, + parsed.releaseManifestSha256, + expectedSourceIdentity + ); + if (!sameRelease(first, preflight)) throw installationFailure(); + await installHostOperationsProvisioningFiles( + testHooks.destinationRoot ?? "/", + first.files, + testHooks.filesystem + ); + const after = await loadProvisioningRelease( + parsed.releaseRoot, + parsed.releaseId, + parsed.releaseManifestSha256, + expectedSourceIdentity + ); + if (!sameRelease(first, after)) throw installationFailure(); + return Object.freeze({ releaseId: parsed.releaseId, status: "INSTALLED" }); + } catch { + throw installationFailure(); + } +} + +if (import.meta.main) { + try { + const result = await runInstallHostOperationsProvisioningCli(Bun.argv.slice(2)); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + const message = + error instanceof TypeError ? error.message : installationFailureMessage; + process.stderr.write(`${message}\n`); + process.exitCode = 1; + } +} diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service new file mode 100644 index 000000000..0d70d62c1 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service @@ -0,0 +1,23 @@ +[Unit] +Description=Mira Dashboard deferred host reboot + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/usr/bin/systemctl reboot +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +RestrictAddressFamilies=AF_UNIX +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=true +RemoveIPC=true +SystemCallArchitectures=native +StandardOutput=null +StandardError=null +TimeoutStartSec=30s diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer new file mode 100644 index 000000000..170a45b36 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer @@ -0,0 +1,7 @@ +[Unit] +Description=Mira Dashboard deferred host reboot timer + +[Timer] +OnActiveSec=10s +AccuracySec=1s +Unit=mira-dashboard-deferred-reboot.service diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation new file mode 100644 index 000000000..bd67836ee --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation @@ -0,0 +1,50 @@ +#!/bin/sh +set -u + +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +umask 077 + +fail() { + exit 64 +} + +[ "$#" -eq 1 ] || fail + +case "$1" in + system-restart) + exec /usr/bin/systemctl start --no-block mira-dashboard-deferred-reboot.timer >/dev/null 2>&1 + ;; + system-update) + update_status=0 + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update >/dev/null 2>&1 || update_status=$? + + upgrade_status=0 + if [ "$update_status" -eq 0 ]; then + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get full-upgrade -y >/dev/null 2>&1 || upgrade_status=$? + else + upgrade_status=1 + fi + + configure_status=0 + DEBIAN_FRONTEND=noninteractive /usr/bin/dpkg --configure -a >/dev/null 2>&1 || configure_status=$? + + [ "$update_status" -eq 0 ] || exit 1 + [ "$upgrade_status" -eq 0 ] || exit 1 + [ "$configure_status" -eq 0 ] || exit 1 + ;; + system-cleanup) + cleanup_status=0 + + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get autoremove -y >/dev/null 2>&1 || cleanup_status=1 + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get autoclean -y >/dev/null 2>&1 || cleanup_status=1 + /usr/bin/journalctl --rotate >/dev/null 2>&1 || cleanup_status=1 + /usr/bin/journalctl --vacuum-time=14d --vacuum-size=1G >/dev/null 2>&1 || cleanup_status=1 + /usr/bin/docker system prune --all --force --filter until=168h >/dev/null 2>&1 || cleanup_status=1 + + [ "$cleanup_status" -eq 0 ] || exit 1 + ;; + *) + fail + ;; +esac diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service new file mode 100644 index 000000000..b72fd622b --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service @@ -0,0 +1,14 @@ +[Unit] +Description=Mira Dashboard reviewed host cleanup +After=network-online.target docker.service systemd-journald.service +Wants=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +UMask=0022 +ExecStart=/usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LANG=C LC_ALL=C DEBIAN_FRONTEND=noninteractive HOME=/root USER=root LOGNAME=root SHELL=/bin/sh /usr/local/libexec/mira-dashboard-host-operation system-cleanup +StandardOutput=null +StandardError=null +TimeoutStartSec=30min diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service new file mode 100644 index 000000000..15292866f --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service @@ -0,0 +1,30 @@ +[Unit] +Description=Mira Dashboard reviewed host restart request +After=local-fs.target + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LANG=C LC_ALL=C /usr/local/libexec/mira-dashboard-host-operation system-restart +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectClock=true +ProtectControlGroups=true +ProtectHome=true +ProtectHostname=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +RestrictAddressFamilies=AF_UNIX +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=true +RemoveIPC=true +SystemCallArchitectures=native +StandardOutput=null +StandardError=null +TimeoutStartSec=30s diff --git a/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service new file mode 100644 index 000000000..799619376 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service @@ -0,0 +1,15 @@ +[Unit] +Description=Mira Dashboard reviewed host package update +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +UMask=0022 +ExecStart=/usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LANG=C LC_ALL=C DEBIAN_FRONTEND=noninteractive HOME=/root USER=root LOGNAME=root SHELL=/bin/sh /usr/local/libexec/mira-dashboard-host-operation system-update +ExecStopPost=/usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin LANG=C LC_ALL=C DEBIAN_FRONTEND=noninteractive HOME=/root USER=root LOGNAME=root SHELL=/bin/sh /usr/bin/dpkg --configure -a +StandardOutput=null +StandardError=null +TimeoutStartSec=115min diff --git a/greenfield/scripts/delivery/provisioning/host-operations/policy.ts b/greenfield/scripts/delivery/provisioning/host-operations/policy.ts new file mode 100644 index 000000000..c24474615 --- /dev/null +++ b/greenfield/scripts/delivery/provisioning/host-operations/policy.ts @@ -0,0 +1,69 @@ +/** Exact release-relative and host destination policy for root host operations. */ +export const hostOperationsProvisioningArtifacts = Object.freeze([ + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules", + destinationPath: "/etc/polkit-1/rules.d/60-mira-dashboard-host-operations.rules", + mode: 0o644, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation", + destinationPath: "/usr/local/libexec/mira-dashboard-host-operation", + mode: 0o755, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service", + destinationPath: "/etc/systemd/system/mira-dashboard-deferred-reboot.service", + mode: 0o644, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer", + destinationPath: "/etc/systemd/system/mira-dashboard-deferred-reboot.timer", + mode: 0o644, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service", + destinationPath: "/etc/systemd/system/mira-dashboard-host-system-cleanup.service", + mode: 0o644, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service", + destinationPath: "/etc/systemd/system/mira-dashboard-host-system-restart.service", + mode: 0o644, + }), + Object.freeze({ + artifactPath: + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service", + destinationPath: "/etc/systemd/system/mira-dashboard-host-system-update.service", + mode: 0o644, + }), +] as const); + +export type HostOperationsProvisioningArtifactPolicy = + (typeof hostOperationsProvisioningArtifacts)[number]; + +/** Exact host directory the installer may create when absent on a fresh host. */ +export const hostOperationsProvisioningCreatedDirectories = Object.freeze([ + Object.freeze({ destinationPath: "/usr/local/libexec", mode: 0o755 }), +] as const); + +/** Reviewed non-installed support files shipped with the root installer. */ +export const hostOperationsProvisioningSupportArtifactPaths = Object.freeze([ + "scripts/delivery/provisioning/host-operations/README.md", + "scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts", + "scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts", + "scripts/delivery/provisioning/host-operations/policy.ts", +] as const); + +/** Complete exact provisioning subtree admitted into an immutable release. */ +export const hostOperationsProvisioningReleaseArtifactPaths = Object.freeze( + [ + ...hostOperationsProvisioningArtifacts.map(({ artifactPath }) => artifactPath), + ...hostOperationsProvisioningSupportArtifactPaths, + ].toSorted() +); diff --git a/greenfield/scripts/delivery/releaseIdentity.test.ts b/greenfield/scripts/delivery/releaseIdentity.test.ts index 85ba1b35e..444fcc767 100644 --- a/greenfield/scripts/delivery/releaseIdentity.test.ts +++ b/greenfield/scripts/delivery/releaseIdentity.test.ts @@ -116,6 +116,10 @@ async function releaseFixture(): Promise<{ path.join(sourceProjectRoot, "migrations"), path.join(releaseRoot, "migrations") ), + copyDirectory( + path.join(sourceProjectRoot, "scripts/delivery/provisioning/host-operations"), + path.join(releaseRoot, "scripts/delivery/provisioning/host-operations") + ), copyDirectory( path.join(sourceProjectRoot, "scripts/delivery/provisioning/log-maintenance"), path.join(releaseRoot, "scripts/delivery/provisioning/log-maintenance") @@ -171,6 +175,17 @@ describe("release identity", () => { .filter(({ path: artifactPath }) => artifactPath.startsWith("scripts/")) .map(({ path: artifactPath }) => artifactPath) ).toEqual([ + "scripts/delivery/provisioning/host-operations/60-mira-dashboard-host-operations.rules", + "scripts/delivery/provisioning/host-operations/README.md", + "scripts/delivery/provisioning/host-operations/hostOperationsProvisioningFilesystem.ts", + "scripts/delivery/provisioning/host-operations/installHostOperationsProvisioning.ts", + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.service", + "scripts/delivery/provisioning/host-operations/mira-dashboard-deferred-reboot.timer", + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-operation", + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-cleanup.service", + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-restart.service", + "scripts/delivery/provisioning/host-operations/mira-dashboard-host-system-update.service", + "scripts/delivery/provisioning/host-operations/policy.ts", "scripts/delivery/provisioning/log-maintenance/60-mira-dashboard-log-maintenance.rules", "scripts/delivery/provisioning/log-maintenance/README.md", "scripts/delivery/provisioning/log-maintenance/installLogMaintenanceProvisioning.ts", diff --git a/greenfield/scripts/delivery/releaseIdentity.ts b/greenfield/scripts/delivery/releaseIdentity.ts index 97af2c281..09b176d44 100644 --- a/greenfield/scripts/delivery/releaseIdentity.ts +++ b/greenfield/scripts/delivery/releaseIdentity.ts @@ -18,6 +18,7 @@ import { } from "../buildSourceIdentity.ts"; import { readBoundedUtf8RegularFile } from "../files/boundedFile.ts"; import { resolveDirectPackageVersions } from "../packageIdentity.ts"; +import { hostOperationsProvisioningReleaseArtifactPaths } from "./hostOperationsProvisioningPolicy.ts"; import { logMaintenanceProvisioningReleaseArtifactPaths } from "./logMaintenanceProvisioningPolicy.ts"; import { productionSystemdUnits } from "./productionSystemdUnitPolicy.ts"; import { @@ -54,7 +55,12 @@ const exactMetadataPaths = Object.freeze([ const exactSystemdPaths = Object.freeze( productionSystemdUnits.map(({ artifactPath }) => artifactPath) ); -const exactScriptPaths = logMaintenanceProvisioningReleaseArtifactPaths; +const exactScriptPaths = Object.freeze( + [ + ...hostOperationsProvisioningReleaseArtifactPaths, + ...logMaintenanceProvisioningReleaseArtifactPaths, + ].toSorted() +); /** Bun identity observed by release creation and activation verification. */ export interface ReleaseRuntimeIdentity { diff --git a/greenfield/scripts/delivery/releaseStaging.ts b/greenfield/scripts/delivery/releaseStaging.ts index 606b6b020..088941893 100644 --- a/greenfield/scripts/delivery/releaseStaging.ts +++ b/greenfield/scripts/delivery/releaseStaging.ts @@ -236,6 +236,13 @@ export async function stageReleaseArtifacts( ), path.join(stagingRoot, "scripts/delivery/provisioning/log-maintenance") ), + copyArtifactTree( + path.join( + sources.repositoryRoot, + "scripts/delivery/provisioning/host-operations" + ), + path.join(stagingRoot, "scripts/delivery/provisioning/host-operations") + ), copyMetadataFile( path.join(sources.repositoryRoot, ".bun-version"), sources.repositoryRoot, diff --git a/greenfield/scripts/documentation/jsonSchema.test.ts b/greenfield/scripts/documentation/jsonSchema.test.ts index 583038f8b..caa57409e 100644 --- a/greenfield/scripts/documentation/jsonSchema.test.ts +++ b/greenfield/scripts/documentation/jsonSchema.test.ts @@ -85,8 +85,8 @@ describe("contract JSON Schema conversion", () => { properties: { actions: { $comment: - "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered.", - maxItems: 4, + "Live Valibot validation additionally requires the six fixed service-action rows to be complete, unique, and canonically ordered.", + maxItems: 6, }, }, }); diff --git a/greenfield/scripts/documentation/jsonSchema.ts b/greenfield/scripts/documentation/jsonSchema.ts index f2b5acd6b..569277dc3 100644 --- a/greenfield/scripts/documentation/jsonSchema.ts +++ b/greenfield/scripts/documentation/jsonSchema.ts @@ -282,7 +282,7 @@ const noNulJsonSchemaPattern = String.raw`^[^\u0000]*$`; const runtimeCheckComments = new Map([ [ serviceActionStatusesAreCanonical, - "Live Valibot validation additionally requires the four fixed service-action rows to be complete, unique, and canonically ordered.", + "Live Valibot validation additionally requires the six fixed service-action rows to be complete, unique, and canonically ordered.", ], [ workspaceFileContentTicketIsConsistent, diff --git a/greenfield/scripts/sourceBoundaries/policy.test.ts b/greenfield/scripts/sourceBoundaries/policy.test.ts index eb5a6a3c0..810f9322a 100644 --- a/greenfield/scripts/sourceBoundaries/policy.test.ts +++ b/greenfield/scripts/sourceBoundaries/policy.test.ts @@ -756,5 +756,5 @@ describe("source-boundary policy", () => { const projectRootUrl = new URL("../..", import.meta.url); const violations = await checkSourceBoundaries(fileURLToPath(projectRootUrl)); expect(violations).toEqual([]); - }, 30_000); + }, 60_000); }); diff --git a/greenfield/scripts/testSupport/productionDeliveryFixture.ts b/greenfield/scripts/testSupport/productionDeliveryFixture.ts index 8366878f0..20fade71c 100644 --- a/greenfield/scripts/testSupport/productionDeliveryFixture.ts +++ b/greenfield/scripts/testSupport/productionDeliveryFixture.ts @@ -126,6 +126,11 @@ export async function createLocalReleaseFixture( path.join(repositoryRoot, "systemd"), { recursive: true } ), + cp( + path.join(sourceProjectRoot, "scripts/delivery/provisioning/host-operations"), + path.join(repositoryRoot, "scripts/delivery/provisioning/host-operations"), + { recursive: true } + ), cp( path.join(sourceProjectRoot, "scripts/delivery/provisioning/log-maintenance"), path.join(repositoryRoot, "scripts/delivery/provisioning/log-maintenance"), diff --git a/greenfield/src/app/dashboardServer.test.ts b/greenfield/src/app/dashboardServer.test.ts index 2e5ad6ee8..c383cc3d2 100644 --- a/greenfield/src/app/dashboardServer.test.ts +++ b/greenfield/src/app/dashboardServer.test.ts @@ -1286,6 +1286,7 @@ describe("Dashboard OpenClaw operations composition", () => { const claimAt = new Date(authenticationTestNow.getTime() + 1); const claim = await jobRepository.claimNextRun({ at: claimAt, + bootIdentity: "00000000-0000-0000-0000-000000000001", leaseExpiresAt: new Date(authenticationTestNow.getTime() + 30_000), leaseToken, minimumHeartbeatAt: new Date(authenticationTestNow.getTime() - 1), diff --git a/greenfield/src/app/dashboardServer.ts b/greenfield/src/app/dashboardServer.ts index d5226eafd..1a141ed93 100644 --- a/greenfield/src/app/dashboardServer.ts +++ b/greenfield/src/app/dashboardServer.ts @@ -46,8 +46,10 @@ import { type GatewaySessionsService, } from "../server/domains/gatewaySessions/service.ts"; import { + hostSystemCleanupJobActionDefinition, hostSystemRestartJobActionDefinition, hostSystemUpdateJobActionDefinition, + openClawGatewayRestartJobActionDefinition, openClawInstallationUpdateJobActionDefinition, openClawSessionsCleanupJobActionDefinition, } from "../server/domains/jobs/actionRegistry.ts"; @@ -763,7 +765,9 @@ export async function createDashboardServer( }); const serviceActionDefinitions = Object.freeze({ "openclaw-cleanup": openClawSessionsCleanupJobActionDefinition, + "openclaw-restart": openClawGatewayRestartJobActionDefinition, "openclaw-update": openClawInstallationUpdateJobActionDefinition, + "system-cleanup": hostSystemCleanupJobActionDefinition, "system-restart": hostSystemRestartJobActionDefinition, "system-update": hostSystemUpdateJobActionDefinition, }); @@ -792,6 +796,9 @@ export async function createDashboardServer( ? {} : { nowMs: () => domainNow().getTime() }), repository: jobRepository, + ...(options.verifiedReleaseId === undefined + ? {} + : { requiredWorkerReleaseId: options.verifiedReleaseId }), wakeEventPump, }), statusReader: diff --git a/greenfield/src/app/developmentWorker.ts b/greenfield/src/app/developmentWorker.ts index 8bddb5044..46039d4d7 100644 --- a/greenfield/src/app/developmentWorker.ts +++ b/greenfield/src/app/developmentWorker.ts @@ -51,13 +51,15 @@ export async function runDevelopmentWorkerProcess( openClawRoot, logMaintenance, moltbook, - hostOperations + hostOperations, + bootIdentity ) => { const writer = createDescriptorWorkspaceFileStructuralWriter({ roots: [workspaceRoot, openClawRoot], spoolRoot: layout.production.state.workspaceFileUploads, }); return createDashboardWorkerRuntime({ + bootIdentity, database: { migrationsDirectory: path.join(source.releaseRoot, "migrations"), releaseId: source.manifest.source.commitSha, diff --git a/greenfield/src/app/worker.test.ts b/greenfield/src/app/worker.test.ts index 96bb185c4..fe7e675c7 100644 --- a/greenfield/src/app/worker.test.ts +++ b/greenfield/src/app/worker.test.ts @@ -28,6 +28,7 @@ const workspaceRoot = "/srv/mira-workspace"; const releaseId = "b".repeat(40); const revision = "a".repeat(40); const checksum = "c".repeat(64); +const bootIdentity = "00000000-0000-0000-0000-000000000001"; const layout = deriveDashboardProjectLayout(projectRoot); const release: RuntimeRelease = Object.freeze({ manifest: parseReleaseManifest({ @@ -212,7 +213,8 @@ function processFixture( observedOpenClawRoot, observedLogMaintenance, _observedMoltbook, - observedHostOperations + observedHostOperations, + observedBootIdentity ) { expect(observedLayout).toBe(layout); expect(observedRelease).toBe(release); @@ -244,6 +246,7 @@ function processFixture( }); expect(observedLogMaintenance).toBe(logMaintenance); expect(observedHostOperations).toBeUndefined(); + expect(observedBootIdentity).toBe(bootIdentity); expect(Object.keys(observedGatewayTransport).toSorted()).toEqual([ "requestOpenClawServiceAction", "start", @@ -308,6 +311,9 @@ function processFixture( }) ); }, + readBootIdentity() { + return Promise.resolve(bootIdentity); + }, startLogMaintenanceAvailability(options) { expect(options.availablePolicies).toBe(logMaintenance.availablePolicies); expect(options.logMaintenanceRoot).toBe( diff --git a/greenfield/src/app/worker.ts b/greenfield/src/app/worker.ts index bb7e5e9a4..ba136af96 100644 --- a/greenfield/src/app/worker.ts +++ b/greenfield/src/app/worker.ts @@ -40,6 +40,7 @@ import { createProcessTerminationController, type ProcessTerminationController, } from "../server/platform/runtime/processSignals.ts"; +import type { LinuxBootIdentity } from "../shared/linuxBootIdentity.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../shared/openClawGatewayLifecycle.ts"; import type { OpenClawServiceActionsExecutionPort } from "../shared/openClawServiceActions.ts"; import { @@ -70,6 +71,7 @@ import { } from "../worker/logs/managedLogRotation.ts"; import { createFixedOpenClawGatewayLifecycle } from "../worker/openClaw/gatewayLifecycle.ts"; import { type DashboardWorkerRuntime } from "../worker/runtime.ts"; +import { readLinuxBootIdentity } from "../worker/system/linuxBootIdentity.ts"; import { taskNotificationWorkerLoop } from "../worker/taskNotifications.ts"; import { startWorkerTerminalBrokerLifecycle, @@ -86,6 +88,7 @@ export interface DashboardWorkerProcessOptions { /** Injectable process boundaries used by deterministic composition tests. */ export interface DashboardWorkerProcessDependencies { + readonly readBootIdentity: () => Promise; readonly createGatewayTransport: ( options: PersistentGatewayTransportOptions ) => PersistentGatewayTaskNotificationTransport; @@ -114,7 +117,8 @@ export interface DashboardWorkerProcessDependencies { openClawRoot: WorkerWorkspaceFileRootConfiguration, logMaintenance: LogMaintenanceExecutor, moltbook: MoltbookDashboardCollector, - hostOperations: FixedHostOperationsExecutionPort | undefined + hostOperations: FixedHostOperationsExecutionPort | undefined, + bootIdentity: LinuxBootIdentity ) => DashboardWorkerRuntime; readonly createTerminationController: () => ProcessTerminationController; readonly loadRelease: ( @@ -201,13 +205,15 @@ const defaultDependencies = Object.freeze({ openClawRoot, logMaintenance, moltbook, - hostOperations + hostOperations, + bootIdentity ) => { const writer = createDescriptorWorkspaceFileStructuralWriter({ roots: [workspaceRoot, openClawRoot], spoolRoot: layout.production.state.workspaceFileUploads, }); return createDashboardWorkerRuntime({ + bootIdentity, database: { migrationsDirectory: path.join(release.releaseRoot, "migrations"), releaseId: release.manifest.source.commitSha, @@ -234,6 +240,7 @@ const defaultDependencies = Object.freeze({ resolveProjectLayout: resolveDashboardProjectLayout, resolveOpenClawFileRoot: resolveReviewedWorkerOpenClawFileRoot, resolveWorkspaceFileRoot: resolveReviewedWorkerWorkspaceFileRoot, + readBootIdentity: readLinuxBootIdentity, startLogMaintenanceAvailability: startLogMaintenanceAvailabilityPublisher, startTerminalBroker: startWorkerTerminalBrokerLifecycle, } satisfies DashboardWorkerProcessDependencies); @@ -330,6 +337,7 @@ export async function runDashboardWorkerProcess( const openClawServiceActions = dependencies.createOpenClawServiceActions(gatewayTransport); const hostOperations = dependencies.createHostOperations?.(); + const bootIdentity = await dependencies.readBootIdentity(); const moltbook = createMoltbookDashboardCollector({ agentName: configuration.moltbookAgentName, apiKey: configuration.moltbookApiKey, @@ -345,7 +353,8 @@ export async function runDashboardWorkerProcess( openClawRoot, logMaintenance, moltbook, - hostOperations + hostOperations, + bootIdentity ); const runtimeCompletion = runtime.completion.then( () => ({ kind: "stopped" as const }), diff --git a/greenfield/src/browser/jobs/JobsRoute.test.tsx b/greenfield/src/browser/jobs/JobsRoute.test.tsx index 99a6aedcd..a6ab0b332 100644 --- a/greenfield/src/browser/jobs/JobsRoute.test.tsx +++ b/greenfield/src/browser/jobs/JobsRoute.test.tsx @@ -26,6 +26,10 @@ import type { RunScheduleInput, UpdateScheduleInput, } from "../../contracts/schedules.ts"; +import { + type GetServiceActionsStatusResult, + serviceActionIds, +} from "../../contracts/serviceActions.ts"; import { createDashboardQueryClient } from "../api/queryClient.ts"; import type { DashboardRealtimeClient } from "../api/realtimeClient.ts"; import { @@ -132,6 +136,37 @@ function runDetail(run: JobRunSummary): JobRunDetail { }; } +function serviceActionRun(): JobRunSummary { + return { + ...queuedRun({ + displayName: "OpenClaw session cleanup", + id: runId, + }), + actionKey: "openclaw.sessions.cleanup", + attemptLimit: 1, + cancellationPolicy: "never", + priority: 20, + resourceClass: "exclusive", + resourceKeys: ["host.mutation"], + retrySafe: false, + timeoutMs: 600_000, + triggerType: "manual", + }; +} + +function serviceActionsStatus(activeRun?: JobRunSummary): GetServiceActionsStatusResult { + return { + actions: serviceActionIds.map((id) => ({ + ...(id === "openclaw-cleanup" && activeRun !== undefined + ? { activeRun } + : {}), + availability: "available" as const, + id, + })), + observedAtMs: timestampMs, + }; +} + function scheduleSummary( id = scheduleId, overrides: Partial = {} @@ -229,6 +264,7 @@ class JobsRouteTransport implements DashboardTrpcTransport { readonly scheduleDetails = new Map(); scheduleRuns: JobRunSummary[] = []; schedules: ScheduleSummary[] = []; + serviceActionsStatus = serviceActionsStatus(); addRunDetail(run: JobRunSummary): void { this.runDetails.set(run.id, runDetail(run)); @@ -478,6 +514,9 @@ class JobsRouteTransport implements DashboardTrpcTransport { total: 0, }); } + case "serviceActions.getStatus": { + return Promise.resolve(this.serviceActionsStatus); + } case "schedules.list": { if (this.failScheduleList) { return Promise.reject(new TypeError("Schedule list unavailable")); @@ -525,7 +564,7 @@ const queryClients: ReturnType[] = []; const collectionRegistries: DashboardBrowserCollections[] = []; const mountedViews: ReturnType[] = []; -function renderJobsRoute( +async function renderJobsRoute( path: string, transport: JobsRouteTransport, realtimeClient: DashboardRealtimeClient = noOpDashboardRealtimeClient @@ -555,6 +594,9 @@ function renderJobsRoute( /> ) ); + if (transport.authStatus.state === "authenticated") { + await screen.findByRole("heading", { level: 3, name: "OpenClaw cleanup" }); + } return { queryClient, router }; } @@ -567,6 +609,56 @@ afterEach(async () => { }); describe("Dashboard jobs route", () => { + test("shows fixed Service Actions and exposes their durable run history and detail", async () => { + const transport = new JobsRouteTransport(); + const run = serviceActionRun(); + transport.runs = [run]; + transport.addRunDetail(run); + transport.serviceActionsStatus = serviceActionsStatus(run); + await renderJobsRoute("/jobs", transport); + const user = userEvent.setup(); + + expect( + await screen.findByRole("heading", { level: 2, name: "Service actions" }) + ).toBeTruthy(); + const serviceActions = screen.getByRole("region", { + name: "Service actions", + }); + expect(within(serviceActions).getAllByRole("heading", { level: 3 })).toHaveLength( + serviceActionIds.length + ); + expect( + within(serviceActions).getByRole("heading", { name: "OpenClaw cleanup" }) + ).toBeTruthy(); + expect( + within(serviceActions).getByRole("heading", { name: "OpenClaw restart" }) + ).toBeTruthy(); + expect( + within(serviceActions).getByRole("heading", { name: "System cleanup" }) + ).toBeTruthy(); + expect( + within(serviceActions).getByRole("heading", { name: "System restart" }) + ).toBeTruthy(); + expect(screen.queryByRole("link", { name: "View Dashboard jobs" })).toBeNull(); + expect(transport.callsFor("serviceActions.getStatus")).toEqual([ + { input: {}, kind: "query", path: "serviceActions.getStatus" }, + ]); + + const openRun = await screen.findByRole("button", { + name: `Open run ${run.displayName}; action ${run.actionKey}; id ${run.id}`, + }); + await user.click(openRun); + const detailHeading = await screen.findByRole("heading", { + level: 2, + name: run.displayName, + }); + await waitFor(() => expect(detailHeading).toHaveFocus()); + expect(transport.callsFor("jobs.getRun").at(-1)?.input).toEqual({ + eventLimit: 100, + id: run.id, + }); + }); + test("loads independent exact deep links and wires navigation and realtime refresh", async () => { const transport = new JobsRouteTransport(); const run = queuedRun({ @@ -578,7 +670,7 @@ describe("Dashboard jobs route", () => { transport.addRunDetail(run); transport.addScheduleDetail(schedule); const realtimeClient = new ControlledDashboardRealtimeClient(); - const { queryClient } = renderJobsRoute( + const { queryClient } = await renderJobsRoute( `/jobs?runId=${runId}&scheduleId=${scheduleId}`, transport, realtimeClient @@ -682,7 +774,7 @@ describe("Dashboard jobs route", () => { transport.runDetails.set(runId, eventPage(initialRun, 202, 100, 103)); transport.addRunEventDetail(runId, 103, eventPage(initialRun, 102, 100, 3)); const realtimeClient = new ControlledDashboardRealtimeClient(); - const { queryClient } = renderJobsRoute( + const { queryClient } = await renderJobsRoute( `/jobs?runId=${runId}`, transport, realtimeClient @@ -797,7 +889,7 @@ describe("Dashboard jobs route", () => { test("drops malformed selections without issuing exact-detail calls", async () => { const transport = new JobsRouteTransport(); - const { queryClient } = renderJobsRoute( + const { queryClient } = await renderJobsRoute( "/jobs?runId=not-a-run&scheduleId=Bad%20Schedule", transport ); @@ -827,7 +919,7 @@ describe("Dashboard jobs route", () => { }); transport.addRunDetail(run); transport.addScheduleDetail(schedule); - renderJobsRoute(`/jobs?runId=${runId}&scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?runId=${runId}&scheduleId=${scheduleId}`, transport); expect( await screen.findByRole("heading", { @@ -865,7 +957,7 @@ describe("Dashboard jobs route", () => { transport.schedules = [schedule]; transport.addRunDetail(run); transport.addScheduleDetail(schedule); - const { queryClient } = renderJobsRoute( + const { queryClient } = await renderJobsRoute( `/jobs?runId=${runId}&scheduleId=${scheduleId}`, transport ); @@ -950,7 +1042,7 @@ describe("Dashboard jobs route", () => { test("clears a schedule filter error while the draft is corrected", async () => { const transport = new JobsRouteTransport(); - const { queryClient } = renderJobsRoute("/jobs", transport); + const { queryClient } = await renderJobsRoute("/jobs", transport); const user = userEvent.setup(); const historyCalls = () => transport @@ -995,7 +1087,7 @@ describe("Dashboard jobs route", () => { test("applies all run filter drafts as one global-history query", async () => { const transport = new JobsRouteTransport(); - const { queryClient } = renderJobsRoute("/jobs", transport); + const { queryClient } = await renderJobsRoute("/jobs", transport); const user = userEvent.setup(); const historyCalls = () => transport @@ -1048,7 +1140,7 @@ describe("Dashboard jobs route", () => { }); transport.runs = [newest, older]; transport.runPages = [[newest], [newest, older]]; - renderJobsRoute("/jobs", transport); + await renderJobsRoute("/jobs", transport); const user = userEvent.setup(); expect( @@ -1086,7 +1178,7 @@ describe("Dashboard jobs route", () => { }); transport.runs = [run]; transport.addRunDetail(run); - renderJobsRoute(`/jobs?runId=${runId}`, transport); + await renderJobsRoute(`/jobs?runId=${runId}`, transport); const user = userEvent.setup(); expect( @@ -1143,7 +1235,7 @@ describe("Dashboard jobs route", () => { transport.scheduleRuns = [run]; transport.addRunDetail(run); transport.addScheduleDetail(schedule); - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); await user.click( @@ -1163,7 +1255,7 @@ describe("Dashboard jobs route", () => { const schedule = scheduleSummary(); transport.schedules = [schedule]; transport.addScheduleDetail(schedule); - renderJobsRoute("/jobs", transport); + await renderJobsRoute("/jobs", transport); const user = userEvent.setup(); await user.click( @@ -1183,7 +1275,7 @@ describe("Dashboard jobs route", () => { const schedule = scheduleSummary(); transport.schedules = [schedule]; transport.addScheduleDetail(schedule); - const { queryClient } = renderJobsRoute( + const { queryClient } = await renderJobsRoute( `/jobs?scheduleId=${scheduleId}`, transport ); @@ -1227,7 +1319,7 @@ describe("Dashboard jobs route", () => { const schedule = scheduleSummary(); transport.schedules = [schedule]; transport.addScheduleDetail(schedule); - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); expect( @@ -1294,7 +1386,7 @@ describe("Dashboard jobs route", () => { const schedule = scheduleSummary(); transport.schedules = [schedule]; transport.addScheduleDetail(schedule); - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); expect( @@ -1324,7 +1416,7 @@ describe("Dashboard jobs route", () => { transport.schedules = [schedule]; transport.addScheduleDetail(schedule); transport.failNextCommittedScheduleRunResponses = 1; - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); expect( @@ -1374,7 +1466,7 @@ describe("Dashboard jobs route", () => { transport.schedules = [schedule]; transport.addScheduleDetail(schedule); transport.failNextMutationCounts.set("schedules.update", 1); - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); const failureMessage = "The request could not be completed. Try again."; @@ -1400,7 +1492,7 @@ describe("Dashboard jobs route", () => { transport.schedules = [schedule]; transport.addScheduleDetail(schedule); transport.failNextMutationCounts.set("schedules.run", 1); - renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); + await renderJobsRoute(`/jobs?scheduleId=${scheduleId}`, transport); const user = userEvent.setup(); const failureMessage = "The request could not be completed. Try again."; @@ -1425,7 +1517,7 @@ describe("Dashboard jobs route", () => { test("switches to the isolated OpenClaw cron source with one exact bounded query", async () => { const transport = new JobsRouteTransport(); - const { router } = renderJobsRoute("/jobs", transport); + const { router } = await renderJobsRoute("/jobs", transport); const user = userEvent.setup(); const source = await screen.findByRole("group", { name: "Job source" }); @@ -1472,7 +1564,7 @@ describe("Dashboard jobs route", () => { const transport = new JobsRouteTransport(); transport.authStatus = { state: "anonymous" }; const realtimeClient = new ControlledDashboardRealtimeClient(); - const { queryClient, router } = renderJobsRoute( + const { queryClient, router } = await renderJobsRoute( `/jobs?runId=${runId}`, transport, realtimeClient diff --git a/greenfield/src/browser/jobs/JobsRoute.tsx b/greenfield/src/browser/jobs/JobsRoute.tsx index 31de54102..41c225d05 100644 --- a/greenfield/src/browser/jobs/JobsRoute.tsx +++ b/greenfield/src/browser/jobs/JobsRoute.tsx @@ -2,6 +2,7 @@ import { useNavigate, useSearch } from "@tanstack/react-router"; import { useState } from "react"; import { OpenClawCronBrowser } from "../openClawCron/OpenClawCronBrowser.tsx"; +import { OverviewServiceActionsSection } from "../overview/OverviewServiceActionsSection.tsx"; import { Button } from "../ui/Button.tsx"; import { PageHeader } from "../ui/PageHeader.tsx"; import { parseJobsRouteSearch } from "./jobRouteSearch.ts"; @@ -18,6 +19,7 @@ function DashboardJobsContent() { return (
    + diff --git a/greenfield/src/browser/overview/OverviewRoute.test.tsx b/greenfield/src/browser/overview/OverviewRoute.test.tsx index 88b82b167..0131b218d 100644 --- a/greenfield/src/browser/overview/OverviewRoute.test.tsx +++ b/greenfield/src/browser/overview/OverviewRoute.test.tsx @@ -232,7 +232,9 @@ const jobRunPage = Object.freeze({ const serviceActionsStatus = Object.freeze({ actions: [ { availability: "unavailable", id: "openclaw-cleanup" }, + { availability: "unavailable", id: "openclaw-restart" }, { availability: "unavailable", id: "openclaw-update" }, + { availability: "unavailable", id: "system-cleanup" }, { availability: "unavailable", id: "system-restart" }, { availability: "unavailable", id: "system-update" }, ], diff --git a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx index b93409c2d..9ac089fbd 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsCard.tsx @@ -1,3 +1,4 @@ +import { Link } from "@tanstack/react-router"; import { Wrench } from "lucide-react"; import { useId, useState } from "react"; @@ -48,7 +49,17 @@ function RunObservation({ label, run }: RunObservationProps) { {formatDashboardDateTime(run.updatedAtMs)} -
    Run {run.id}
    +
    + Run{" "} + + {run.id} + +
    ); } @@ -131,6 +142,7 @@ export interface OverviewServiceActionsCardProps { readonly recoveryPending: (actionId: ServiceActionId) => boolean; readonly requestActionId: ServiceActionId | undefined; readonly requestBusy: boolean; + readonly showJobsLink?: boolean; } /** @@ -149,6 +161,7 @@ export function OverviewServiceActionsCard({ recoveryPending, requestActionId, requestBusy, + showJobsLink = true, }: OverviewServiceActionsCardProps) { const headingId = useId(); const [selectedActionId, setSelectedActionId] = useState(); @@ -172,15 +185,17 @@ export function OverviewServiceActionsCard({ Service actions - Queue four fixed, audited worker operations. Recent - multi-factor authentication is required; arbitrary commands - are not accepted. + Queue fixed, audited worker operations. Recent multi-factor + authentication is required; arbitrary commands are not + accepted.
    - - View Dashboard jobs - + {showJobsLink && ( + + View Dashboard jobs + + )} { ).toBeTruthy(); expect(harness.transport.queryCalls[0]?.input).toEqual({}); expect(screen.getByRole("heading", { name: "OpenClaw cleanup" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "OpenClaw restart" })).toBeTruthy(); expect(screen.getByRole("heading", { name: "OpenClaw update" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "System cleanup" })).toBeTruthy(); expect(screen.getByRole("heading", { name: "System restart" })).toBeTruthy(); expect(screen.getByRole("heading", { name: "System update" })).toBeTruthy(); - expect(screen.queryByText(/Gateway restart/iu)).toBeNull(); - expect(screen.queryByText(/system cleanup/iu)).toBeNull(); expect(screen.queryByText(/terminal|command to run/iu)).toBeNull(); expect( screen.getByRole("button", { name: "Queue OpenClaw update" }) @@ -343,6 +351,16 @@ describe("OverviewServiceActionsSection", () => { "href", "/jobs" ); + expect( + screen.getByRole("link", { + name: `Open Dashboard job ${runningRun.id}`, + }) + ).toHaveAttribute("href", `/jobs?runId=${runningRun.id}`); + expect( + screen.getByRole("link", { + name: `Open Dashboard job ${succeededRun.id}`, + }) + ).toHaveAttribute("href", `/jobs?runId=${succeededRun.id}`); }); test("clears an active action after a same-tab job-run event", async () => { @@ -416,10 +434,28 @@ describe("OverviewServiceActionsSection", () => { expect(screen.getByText(/System updates can take a long time/iu)).toBeTruthy(); await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(screen.getByRole("button", { name: "Queue system cleanup" })); + const cleanupDialog = screen.getByRole("dialog", { + name: "Queue a system cleanup?", + }); + expect(cleanupDialog).toHaveTextContent( + /unused Docker content older than seven days/iu + ); + expect(cleanupDialog).toHaveTextContent(/Docker volumes are never deleted/iu); + await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(screen.getByRole("button", { name: "Queue OpenClaw update" })); expect(screen.getByText(/OpenClaw updates can take time/iu)).toBeTruthy(); await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(screen.getByRole("button", { name: "Queue OpenClaw restart" })); + expect(screen.getByText(/interrupts active Gateway sessions/iu)).toBeTruthy(); + expect(screen.getByText(/durable result/iu)).toBeTruthy(); + expect( + screen.getByText(/does not confirm that the restart completed/iu) + ).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(screen.getByRole("button", { name: "Queue OpenClaw cleanup" })); expect( screen.getByText(/source-owned OpenClaw session and artifact maintenance/iu) diff --git a/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx index bb584b59f..a0c34a5cf 100644 --- a/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx +++ b/greenfield/src/browser/overview/OverviewServiceActionsSection.tsx @@ -152,8 +152,14 @@ function useServiceActionRequest() { }; } +export interface OverviewServiceActionsSectionProps { + readonly showJobsLink?: boolean; +} + /** @returns Fixed service-action status, requests, and partial-read handling. */ -export function OverviewServiceActionsSection() { +export function OverviewServiceActionsSection({ + showJobsLink = true, +}: OverviewServiceActionsSectionProps = {}) { useServiceActionsRealtimeInvalidation(); const client = useDashboardTrpcClient(); const query = useQuery(serviceActionsStatusQueryOptions(client)); @@ -201,6 +207,7 @@ export function OverviewServiceActionsSection() { recoveryPending={request.recoveryPending} requestActionId={request.variables} requestBusy={request.isPending} + showJobsLink={showJobsLink} /> ); diff --git a/greenfield/src/browser/overview/serviceActionsOperations.test.ts b/greenfield/src/browser/overview/serviceActionsOperations.test.ts index b0c7d2c38..71f35fa11 100644 --- a/greenfield/src/browser/overview/serviceActionsOperations.test.ts +++ b/greenfield/src/browser/overview/serviceActionsOperations.test.ts @@ -82,18 +82,28 @@ describe("service action browser operations", () => { queryClient.clear(); }); - test("builds only the four exact confirmation inputs", () => { + test("builds only the six exact confirmation inputs", () => { const idempotencyKey = "a".repeat(32); expect(serviceActionRequestInput("openclaw-cleanup", idempotencyKey)).toEqual({ actionId: "openclaw-cleanup", confirmation: "cleanup-openclaw", idempotencyKey, }); + expect(serviceActionRequestInput("openclaw-restart", idempotencyKey)).toEqual({ + actionId: "openclaw-restart", + confirmation: "restart-openclaw", + idempotencyKey, + }); expect(serviceActionRequestInput("openclaw-update", idempotencyKey)).toEqual({ actionId: "openclaw-update", confirmation: "update-openclaw", idempotencyKey, }); + expect(serviceActionRequestInput("system-cleanup", idempotencyKey)).toEqual({ + actionId: "system-cleanup", + confirmation: "cleanup-system", + idempotencyKey, + }); expect(serviceActionRequestInput("system-restart", idempotencyKey)).toEqual({ actionId: "system-restart", confirmation: "restart-system", @@ -116,9 +126,27 @@ describe("service action browser operations", () => { ) ).toEqual({ "openclaw-cleanup": "Retry OpenClaw cleanup request", + "openclaw-restart": "Retry OpenClaw restart request", "openclaw-update": "Retry OpenClaw update request", + "system-cleanup": "Retry system cleanup request", "system-restart": "Retry system restart request", "system-update": "Retry system update request", }); }); + + test("states the bounded system cleanup policy without volume deletion", () => { + const presentation = serviceActionPresentations["system-cleanup"]; + + expect(presentation.description).toContain("orphan packages and caches"); + expect(presentation.description).toContain("older than seven days"); + expect(presentation.warning).toContain("volumes are never deleted"); + }); + + test("states OpenClaw Gateway interruption and durable-result semantics", () => { + const presentation = serviceActionPresentations["openclaw-restart"]; + + expect(presentation.warning).toContain("interrupts active Gateway sessions"); + expect(presentation.warning).toContain("durable result"); + expect(presentation.warning).toContain("does not confirm"); + }); }); diff --git a/greenfield/src/browser/overview/serviceActionsOperations.ts b/greenfield/src/browser/overview/serviceActionsOperations.ts index e6cca7273..c1d44df41 100644 --- a/greenfield/src/browser/overview/serviceActionsOperations.ts +++ b/greenfield/src/browser/overview/serviceActionsOperations.ts @@ -34,6 +34,17 @@ export const serviceActionPresentations = Object.freeze({ warning: "This queues OpenClaw's own bounded session and artifact maintenance. Review Dashboard jobs for the durable result.", }, + "openclaw-restart": { + actionLabel: "OpenClaw restart", + buttonLabel: "Queue OpenClaw restart", + confirmationLabel: "Queue restart", + confirmationTitle: "Queue an OpenClaw restart?", + description: + "Restarts the OpenClaw Gateway through the existing fixed worker-owned lifecycle action.", + retryLabel: "Retry OpenClaw restart request", + warning: + "Restarting the OpenClaw Gateway interrupts active Gateway sessions. Review Dashboard jobs for the durable result; a queued request does not confirm that the restart completed.", + }, "openclaw-update": { actionLabel: "OpenClaw update", buttonLabel: "Queue OpenClaw update", @@ -45,6 +56,17 @@ export const serviceActionPresentations = Object.freeze({ warning: "OpenClaw updates can take time and may restart the Gateway. The Dashboard only confirms that the durable request was queued.", }, + "system-cleanup": { + actionLabel: "System cleanup", + buttonLabel: "Queue system cleanup", + confirmationLabel: "Queue cleanup", + confirmationTitle: "Queue a system cleanup?", + description: + "Cleans orphan packages and caches, bounds journal retention, and removes unused Docker content older than seven days without deleting volumes.", + retryLabel: "Retry system cleanup request", + warning: + "System cleanup removes only fixed categories: orphan packages and caches, bounded journal history, and unused Docker content older than seven days. Docker volumes are never deleted. Review Dashboard jobs for the durable result.", + }, "system-restart": { actionLabel: "System restart", buttonLabel: "Queue system restart", @@ -184,6 +206,13 @@ export function serviceActionRequestInput( idempotencyKey, }; } + case "openclaw-restart": { + return { + actionId, + confirmation: "restart-openclaw", + idempotencyKey, + }; + } case "openclaw-update": { return { actionId, @@ -191,6 +220,13 @@ export function serviceActionRequestInput( idempotencyKey, }; } + case "system-cleanup": { + return { + actionId, + confirmation: "cleanup-system", + idempotencyKey, + }; + } case "system-restart": { return { actionId, diff --git a/greenfield/src/contracts/serviceActions.test.ts b/greenfield/src/contracts/serviceActions.test.ts index c2f3f1f59..1dc1e4b15 100644 --- a/greenfield/src/contracts/serviceActions.test.ts +++ b/greenfield/src/contracts/serviceActions.test.ts @@ -128,7 +128,9 @@ describe("service action contracts", () => { test("accepts only exact fixed action confirmations and idempotency keys", () => { const valid = [ ["openclaw-cleanup", "cleanup-openclaw"], + ["openclaw-restart", "restart-openclaw"], ["openclaw-update", "update-openclaw"], + ["system-cleanup", "cleanup-system"], ["system-restart", "restart-system"], ["system-update", "update-system"], ] as const; @@ -144,6 +146,11 @@ describe("service action contracts", () => { } for (const input of [ + { + actionId: "openclaw-restart", + confirmation: "restart-openclaw-gateway", + idempotencyKey, + }, { actionId: "system-restart", confirmation: "update-system", @@ -151,7 +158,7 @@ describe("service action contracts", () => { }, { actionId: "system-cleanup", - confirmation: "cleanup-system", + confirmation: "cleanup-host", idempotencyKey, }, { @@ -183,7 +190,7 @@ describe("service action contracts", () => { test("requires the complete canonical fixed inventory and bounded run projections", () => { const actions = serviceActionIds.map((id, index) => ({ ...(index === 0 ? { activeRun: queuedRun("openclaw.sessions.cleanup") } : {}), - availability: index === 3 ? "unavailable" : "available", + availability: index === 5 ? "unavailable" : "available", id, })); expect( diff --git a/greenfield/src/contracts/serviceActions.ts b/greenfield/src/contracts/serviceActions.ts index 8aaefa557..e79b790a0 100644 --- a/greenfield/src/contracts/serviceActions.ts +++ b/greenfield/src/contracts/serviceActions.ts @@ -12,7 +12,9 @@ import type { ProcedureContract } from "./registry.ts"; /** Fixed privileged operations accepted by the purpose-built service-actions boundary. */ export const serviceActionIds = [ "openclaw-cleanup", + "openclaw-restart", "openclaw-update", + "system-cleanup", "system-restart", "system-update", ] as const; @@ -84,6 +86,14 @@ export const requestServiceActionInputSchema = v.variant("actionId", [ ), ...serviceActionRequestBase, }), + v.strictObject({ + actionId: v.literal("openclaw-restart"), + confirmation: v.literal( + "restart-openclaw", + "OpenClaw restart confirmation is invalid" + ), + ...serviceActionRequestBase, + }), v.strictObject({ actionId: v.literal("openclaw-update"), confirmation: v.literal( @@ -92,6 +102,14 @@ export const requestServiceActionInputSchema = v.variant("actionId", [ ), ...serviceActionRequestBase, }), + v.strictObject({ + actionId: v.literal("system-cleanup"), + confirmation: v.literal( + "cleanup-system", + "System cleanup confirmation is invalid" + ), + ...serviceActionRequestBase, + }), v.strictObject({ actionId: v.literal("system-restart"), confirmation: v.literal( diff --git a/greenfield/src/server/database/migrations/jobsSchema.test.ts b/greenfield/src/server/database/migrations/jobsSchema.test.ts index ad583c8e0..81604314d 100644 --- a/greenfield/src/server/database/migrations/jobsSchema.test.ts +++ b/greenfield/src/server/database/migrations/jobsSchema.test.ts @@ -45,6 +45,7 @@ interface QueuedRunFixture { idempotencyKey: string; requestedById: string; requestedByKind: "automation" | "system" | "user"; + requiredWorkerReleaseId: string | null; resourceKeysJson: string; retrySafe: number; scheduledForAt: number | null; @@ -170,6 +171,7 @@ function insertQueuedRun( cancellationPolicy: "cooperative", requestedById: "job-scheduler", requestedByKind: "system", + requiredWorkerReleaseId: null, resourceKeysJson: "[]", retrySafe: 1, scheduledForAt: null, @@ -184,12 +186,13 @@ function insertQueuedRun( action_key, attempt_limit, available_at, cancellation_policy, display_name, enqueue_sha256, id, idempotency_key, payload_json, priority, queued_at, requested_by_id, requested_by_kind, - resource_class, resource_keys_json, retry_safe, scheduled_for_at, + required_worker_release_id, resource_class, resource_keys_json, + retry_safe, scheduled_for_at, scheduled_job_id, scheduled_job_version, state, timeout_ms, trigger_type, updated_at ) VALUES ( 'system.worker-smoke', ?, 1000, ?, 'Worker smoke', ?, ?, ?, '{}', - 0, 1000, ?, ?, 'light', ?, ?, ?, ?, ?, 'queued', 10000, ?, 1000 + 0, 1000, ?, ?, ?, 'light', ?, ?, ?, ?, ?, 'queued', 10000, ?, 1000 )`, [ fixture.attemptLimit, @@ -199,6 +202,7 @@ function insertQueuedRun( fixture.idempotencyKey, fixture.requestedById, fixture.requestedByKind, + fixture.requiredWorkerReleaseId, fixture.resourceKeysJson, fixture.retrySafe, fixture.scheduledForAt, @@ -321,7 +325,8 @@ describe("jobs baseline schema", () => { SELECT name, strict, wr FROM pragma_table_list WHERE name IN ( - 'job_disable_intents', 'job_run_events', 'job_runs', + 'host_restart_claim_fence', 'job_disable_intents', + 'job_run_events', 'job_runs', 'job_worker_control', 'resource_leases', 'scheduled_jobs', 'worker_instances' ) @@ -330,6 +335,7 @@ describe("jobs baseline schema", () => { ) .all() ).toEqual([ + { name: "host_restart_claim_fence", strict: 1, wr: 1 }, { name: "job_disable_intents", strict: 1, wr: 1 }, { name: "job_run_events", strict: 1, wr: 1 }, { name: "job_runs", strict: 1, wr: 1 }, @@ -665,6 +671,13 @@ describe("jobs baseline schema", () => { resourceKeysJson: '["UPPER",7,"duplicate","duplicate"]', }) ).toThrow("job_runs resource keys must be canonical"); + expect(() => + insertQueuedRun(database, { + id: uuid(13), + idempotencyKey: idempotencyKey(13), + requiredWorkerReleaseId: "A".repeat(40), + }) + ).toThrow("job_runs_required_worker_release_id_check"); } finally { database.sqlite.close(true); } @@ -2261,6 +2274,18 @@ describe("jobs baseline schema", () => { ['{"policyId":"docker-managed"}'], "action_key=? AND payload_json=?" ); + expectUsesIndexWithoutTemporarySort( + database, + `SELECT id FROM job_runs + WHERE action_key = ? + AND action_key IN ('openclaw.sessions.cleanup', 'openclaw.gateway.restart', 'openclaw.installation.update', 'host.system.cleanup', 'host.system.restart', 'host.system.update') + AND payload_json = '{}' + AND state IN ('cancelled', 'failed', 'succeeded', 'timed-out') + ORDER BY queued_at DESC, id DESC LIMIT 1`, + "job_runs_service_action_terminal_idx", + ["openclaw.sessions.cleanup"], + "action_key=?" + ); expectUsesIndexWithoutTemporarySort( database, `SELECT id FROM job_runs diff --git a/greenfield/src/server/database/migrations/migrationGraph.test.ts b/greenfield/src/server/database/migrations/migrationGraph.test.ts index 64c59412d..1b090174f 100644 --- a/greenfield/src/server/database/migrations/migrationGraph.test.ts +++ b/greenfield/src/server/database/migrations/migrationGraph.test.ts @@ -44,6 +44,7 @@ const expectedTables: string[] = [ "chat_runs", "chat_runtime_snapshots", "chat_transcript_generations", + "host_restart_claim_fence", "incident_observations", "incidents", "job_disable_intents", @@ -131,6 +132,8 @@ describe("database migration graph", () => { "chat_transcript_generations_reject_identity_update", "chat_transcript_generations_reject_replace", "chat_transcript_generations_validate_monotonic_update", + "host_restart_claim_fence_reject_update", + "host_restart_claim_fence_validate_insert", "reports_validate_metadata_insert", "reports_validate_metadata_update", "incidents_validate_details_insert", @@ -220,6 +223,7 @@ describe("database migration graph", () => { ).toBe(1); for (const tableName of [ "cache_entries", + "host_restart_claim_fence", "job_disable_intents", "job_run_events", "job_runs", diff --git a/greenfield/src/server/database/schema/checks.ts b/greenfield/src/server/database/schema/checks.ts index 2f4ddad93..c991eb24f 100644 --- a/greenfield/src/server/database/schema/checks.ts +++ b/greenfield/src/server/database/schema/checks.ts @@ -85,3 +85,12 @@ export function timestampMillisecondsCheck(column: SQLWrapper) { export function uuidV7TextCheck(column: SQLWrapper) { return sql`length(${column}) = 36 AND ${nulFreeTextCheck(column)} AND length(replace(${column}, '-', '')) = 32 AND replace(${column}, '-', '') NOT GLOB '*[^0-9a-f]*' AND substr(${column}, 9, 1) = '-' AND substr(${column}, 14, 1) = '-' AND substr(${column}, 15, 1) = '7' AND substr(${column}, 19, 1) = '-' AND substr(${column}, 20, 1) GLOB '[89ab]' AND substr(${column}, 24, 1) = '-'`; } + +/** + * Builds a SQLite check matching a canonical lowercase UUID of any version. + * @param column SQLite text column to validate. + * @returns Drizzle SQL expression for the storage constraint. + */ +export function lowercaseUuidTextCheck(column: SQLWrapper) { + return sql`length(${column}) = 36 AND ${nulFreeTextCheck(column)} AND length(replace(${column}, '-', '')) = 32 AND replace(${column}, '-', '') NOT GLOB '*[^0-9a-f]*' AND substr(${column}, 9, 1) = '-' AND substr(${column}, 14, 1) = '-' AND substr(${column}, 19, 1) = '-' AND substr(${column}, 24, 1) = '-'`; +} diff --git a/greenfield/src/server/database/schema/drizzleSchema.ts b/greenfield/src/server/database/schema/drizzleSchema.ts index 3334d5552..7eb87c6fd 100644 --- a/greenfield/src/server/database/schema/drizzleSchema.ts +++ b/greenfield/src/server/database/schema/drizzleSchema.ts @@ -18,6 +18,7 @@ export { chatRuntimeSnapshots } from "./chatRuntimeSnapshots.ts"; export { chatTranscriptGenerations } from "./chatTranscriptGenerations.ts"; export { incidentObservations } from "./incidentObservations.ts"; export { incidents } from "./incidents.ts"; +export { hostRestartClaimFence } from "./hostRestartClaimFence.ts"; export { jobDisableIntents } from "./jobDisableIntents.ts"; export { jobRunEvents } from "./jobRunEvents.ts"; export { jobRuns } from "./jobRuns.ts"; diff --git a/greenfield/src/server/database/schema/hostRestartClaimFence.ts b/greenfield/src/server/database/schema/hostRestartClaimFence.ts new file mode 100644 index 000000000..4bb08125f --- /dev/null +++ b/greenfield/src/server/database/schema/hostRestartClaimFence.ts @@ -0,0 +1,49 @@ +import { sql } from "drizzle-orm"; +import { check, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; + +import { + lowercaseUuidTextCheck, + timestampMillisecondsCheck, + uuidV7TextCheck, +} from "./checks.ts"; +import { jobRuns } from "./jobRuns.ts"; +import { workerInstances } from "./workerInstances.ts"; + +/** Singleton cross-process admission fence armed only by a running host-restart claim. */ +export const hostRestartClaimFence = sqliteTable( + "host_restart_claim_fence", + { + armedAt: integer("armed_at", { mode: "timestamp_ms" }).notNull(), + bootIdentity: text("boot_identity").notNull(), + expiresAt: integer("expires_at", { mode: "timestamp_ms" }).notNull(), + id: integer("id").notNull().primaryKey(), + jobRunId: text("job_run_id") + .notNull() + .references(() => jobRuns.id, { + onDelete: "restrict", + onUpdate: "restrict", + }), + leaseToken: text("lease_token").notNull(), + workerInstanceId: text("worker_instance_id") + .notNull() + .references(() => workerInstances.id, { + onDelete: "restrict", + onUpdate: "restrict", + }), + }, + (table) => [ + check( + "host_restart_claim_fence_boot_identity_check", + lowercaseUuidTextCheck(table.bootIdentity) + ), + check("host_restart_claim_fence_id_check", sql`${table.id} = 1`), + check( + "host_restart_claim_fence_lease_token_check", + uuidV7TextCheck(table.leaseToken) + ), + check( + "host_restart_claim_fence_time_check", + sql`${timestampMillisecondsCheck(table.armedAt)} AND ${timestampMillisecondsCheck(table.expiresAt)} AND ${table.expiresAt} > ${table.armedAt}` + ), + ] +); diff --git a/greenfield/src/server/database/schema/jobRuns.ts b/greenfield/src/server/database/schema/jobRuns.ts index 70f5f8ff3..0685a8cb8 100644 --- a/greenfield/src/server/database/schema/jobRuns.ts +++ b/greenfield/src/server/database/schema/jobRuns.ts @@ -78,6 +78,7 @@ export const jobRuns = sqliteTable( requestedByKind: text("requested_by_kind", { enum: ["automation", "system", "user"], }).notNull(), + requiredWorkerReleaseId: text("required_worker_release_id"), resourceClass: text("resource_class", { enum: ["exclusive", "host-heavy", "interactive", "light", "network"], }).notNull(), @@ -152,6 +153,10 @@ export const jobRuns = sqliteTable( allowSystem: true, }) ), + check( + "job_runs_required_worker_release_id_check", + sql`${table.requiredWorkerReleaseId} IS NULL OR ${lowercaseHexTextCheck(table.requiredWorkerReleaseId, 40)}` + ), check( "job_runs_resource_class_check", sql`${table.resourceClass} IN ('exclusive', 'host-heavy', 'interactive', 'light', 'network')` @@ -219,6 +224,11 @@ export const jobRuns = sqliteTable( .where( sql`${table.actionKey} = ${sql.raw(`'${logMaintenanceJobActionKey}'`)} AND length(CAST(${table.payloadJson} AS BLOB)) <= ${sql.raw(String(logMaintenanceJobPayloadIndexMaximumBytes))} AND ${table.state} IN ('cancelled', 'failed', 'succeeded', 'timed-out')` ), + index("job_runs_service_action_terminal_idx") + .on(table.actionKey, desc(table.queuedAt), desc(table.id)) + .where( + sql`${table.actionKey} IN ('openclaw.sessions.cleanup', 'openclaw.gateway.restart', 'openclaw.installation.update', 'host.system.cleanup', 'host.system.restart', 'host.system.update') AND ${table.payloadJson} = '{}' AND ${table.state} IN ('cancelled', 'failed', 'succeeded', 'timed-out')` + ), index("job_runs_queued_id_idx").on(table.queuedAt, table.id), index("job_runs_schedule_queued_id_idx").on( table.scheduledJobId, diff --git a/greenfield/src/server/database/validation/hostRestartClaimFence.ts b/greenfield/src/server/database/validation/hostRestartClaimFence.ts new file mode 100644 index 000000000..b9acc1f29 --- /dev/null +++ b/greenfield/src/server/database/validation/hostRestartClaimFence.ts @@ -0,0 +1,46 @@ +import { createInsertSchema, createSelectSchema } from "drizzle-orm/valibot"; +import * as v from "valibot"; + +import { linuxBootIdentitySchema } from "../../../shared/linuxBootIdentity.ts"; +import { hostRestartClaimFence } from "../schema/hostRestartClaimFence.ts"; +import { nonnegativeDateSchema, uuidV7TextSchema } from "./scalars.ts"; + +const refinements = { + armedAt: nonnegativeDateSchema, + bootIdentity: () => linuxBootIdentitySchema, + expiresAt: nonnegativeDateSchema, + jobRunId: uuidV7TextSchema, + leaseToken: uuidV7TextSchema, + workerInstanceId: uuidV7TextSchema, +}; + +const generatedSelectSchema = createSelectSchema(hostRestartClaimFence, refinements); +const selectObjectSchema = v.strictObject(generatedSelectSchema.entries); +type HostRestartClaimFenceSelect = v.InferOutput; + +function selectedFenceIsConsistent(fence: HostRestartClaimFenceSelect): boolean { + return fence.id === 1 && fence.expiresAt.getTime() > fence.armedAt.getTime(); +} + +/** Validates one durable singleton restart fence read from SQLite. */ +export const hostRestartClaimFenceSelectSchema = v.pipe( + selectObjectSchema, + v.check(selectedFenceIsConsistent, "Stored host restart claim fence is inconsistent") +); + +const generatedInsertSchema = createInsertSchema(hostRestartClaimFence, refinements); +const insertObjectSchema = v.strictObject({ + ...generatedInsertSchema.entries, + id: v.literal(1), +}); +type HostRestartClaimFenceInsert = v.InferOutput; + +function insertedFenceIsConsistent(fence: HostRestartClaimFenceInsert): boolean { + return fence.id === 1 && fence.expiresAt.getTime() > fence.armedAt.getTime(); +} + +/** Validates one exact restart fence before atomic insertion. */ +export const hostRestartClaimFenceInsertSchema = v.pipe( + insertObjectSchema, + v.check(insertedFenceIsConsistent, "Stored host restart claim fence is inconsistent") +); diff --git a/greenfield/src/server/database/validation/jobRuns.ts b/greenfield/src/server/database/validation/jobRuns.ts index 14915bccf..9d18087ca 100644 --- a/greenfield/src/server/database/validation/jobRuns.ts +++ b/greenfield/src/server/database/validation/jobRuns.ts @@ -30,13 +30,21 @@ import { } from "../../../contracts/jobModel.ts"; import { utf8ByteLength } from "../../../shared/encoding.ts"; import { parseJsonText } from "../../../shared/json.ts"; -import { nonnegativeSafeIntegerSchema } from "../../../shared/validation.ts"; +import { + fullCommitShaAction, + nonnegativeSafeIntegerSchema, +} from "../../../shared/validation.ts"; import { jobRuns } from "../schema/jobRuns.ts"; import { jobActorIdentityIsValid } from "./jobActors.ts"; import { nonnegativeDateSchema, uuidV7TextSchema } from "./scalars.ts"; import { sha256TextSchema } from "./securityScalars.ts"; const actorKindSchema = v.picklist(["automation", "system", "user"]); +const requiredWorkerReleaseIdMessage = "Stored required worker release id is invalid"; +const requiredWorkerReleaseIdSchema = v.pipe( + v.string(requiredWorkerReleaseIdMessage), + fullCommitShaAction(requiredWorkerReleaseIdMessage) +); function jsonObjectTextSchema( maximumBytes: number, @@ -253,7 +261,7 @@ function jobRunIsConsistent(run: StoredJobRun): boolean { ); } -const jobRunRefinements = { +const jobRunSharedRefinements = { actionKey: () => jobActionKeySchema, attemptCount: () => jobAttemptCountSchema, attemptLimit: () => jobAttemptLimitSchema, @@ -296,7 +304,11 @@ const jobRunRefinements = { updatedAt: nonnegativeDateSchema, }; -const generatedJobRunSelectSchema = createSelectSchema(jobRuns, jobRunRefinements); +const generatedJobRunSelectSchema = createSelectSchema(jobRuns, { + ...jobRunSharedRefinements, + requiredWorkerReleaseId: (schema: v.StringSchema) => + v.pipe(schema, fullCommitShaAction(requiredWorkerReleaseIdMessage)), +}); const jobRunSelectObjectSchema = v.strictObject(generatedJobRunSelectSchema.entries); /** Validates one complete durable job-run row read from SQLite. */ @@ -306,7 +318,13 @@ export const jobRunSelectSchema = v.pipe( ); const generatedJobRunInsertSchema = v.omit( - createInsertSchema(jobRuns, jobRunRefinements), + createInsertSchema(jobRuns, { + ...jobRunSharedRefinements, + requiredWorkerReleaseId: v.optional( + v.nullable(requiredWorkerReleaseIdSchema), + null + ), + }), ["attemptCount", "eventBytes", "eventCount", "payloadEventCount", "stateVersion"] ); const jobRunInsertObjectSchema = v.strictObject(generatedJobRunInsertSchema.entries); diff --git a/greenfield/src/server/database/validation/rowSchemas.test.ts b/greenfield/src/server/database/validation/rowSchemas.test.ts index c4059b9ee..f07422d5c 100644 --- a/greenfield/src/server/database/validation/rowSchemas.test.ts +++ b/greenfield/src/server/database/validation/rowSchemas.test.ts @@ -117,6 +117,7 @@ const validJobRunRow = Object.freeze({ queuedAt: jobUpdatedAt, requestedById: jobUserId, requestedByKind: "user" as const, + requiredWorkerReleaseId: null, resourceClass: "light" as const, resourceKeysJson: '["database"]', resultJson: null, diff --git a/greenfield/src/server/domains/cache/repository.test.ts b/greenfield/src/server/domains/cache/repository.test.ts index b108c6cf1..aeb8532ec 100644 --- a/greenfield/src/server/domains/cache/repository.test.ts +++ b/greenfield/src/server/domains/cache/repository.test.ts @@ -130,6 +130,7 @@ async function runningClaim( }, }); const claim = await jobs.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(2000), leaseExpiresAt: new Date(20_000), leaseToken, @@ -308,6 +309,7 @@ describe("cache repository", () => { }); const nextLeaseToken = uuid(4); const next = await fixture.jobs.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(5000), leaseExpiresAt: new Date(30_000), leaseToken: nextLeaseToken, diff --git a/greenfield/src/server/domains/files/jobScheduler.test.ts b/greenfield/src/server/domains/files/jobScheduler.test.ts index 3e9ea92fe..7921e2959 100644 --- a/greenfield/src/server/domains/files/jobScheduler.test.ts +++ b/greenfield/src/server/domains/files/jobScheduler.test.ts @@ -50,6 +50,7 @@ function repositoryFixture() { eventBytes: 0, eventCount: 1, payloadEventCount: 0, + requiredWorkerReleaseId: input.run.requiredWorkerReleaseId ?? null, stateVersion: 1, }; return Promise.resolve({ kind: "inserted", run: stored }); diff --git a/greenfield/src/server/domains/jobs/actionExecutors.test.ts b/greenfield/src/server/domains/jobs/actionExecutors.test.ts index dc9d2b528..76d99da83 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.test.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.test.ts @@ -24,11 +24,14 @@ import { type JobCacheAttemptCommit, JobActionOutcomeUnknownError, JobActionRetryableError, + hostSystemRestartJobActionDefinition, jobActionDefinitions, } from "./actionRegistry.ts"; function executionContext(attempts: JobCacheAttemptCommit[]): JobActionExecutionContext { return { + armHostRestartClaimFence: () => Promise.resolve(), + clearHostRestartClaimFence: () => Promise.resolve(), commitCacheAttempt: (attempt) => { attempts.push(attempt); return Promise.resolve("committed"); @@ -79,6 +82,7 @@ describe("worker-only job executor registry", () => { }); expect(findAction("openclaw.sessions.cleanup")).toBeDefined(); expect(findAction("openclaw.installation.update")).toBeDefined(); + expect(findAction("host.system.cleanup")).toBeUndefined(); expect(findAction("host.system.restart")).toBeUndefined(); expect(findAction("host.system.update")).toBeUndefined(); expect(findAction("system.shell")).toBeUndefined(); @@ -106,10 +110,22 @@ describe("worker-only job executor registry", () => { test("persists only fixed host-operation settlement and rejects mismatched results", async () => { const calls: unknown[] = []; + const fenceEvents: string[] = []; + const restartContext: JobActionExecutionContext = { + ...executionContext([]), + armHostRestartClaimFence: () => { + fenceEvents.push("arm"); + return Promise.resolve(); + }, + clearHostRestartClaimFence: () => { + fenceEvents.push("clear"); + return Promise.resolve(); + }, + }; const hostOperations = { availableOperations: () => Promise.resolve([]), request( - operationId: "system-restart" | "system-update", + operationId: "system-cleanup" | "system-restart" | "system-update", signal?: AbortSignal ) { calls.push({ operationId, signal }); @@ -123,11 +139,20 @@ describe("worker-only job executor registry", () => { expect( await Effect.runPromise( createHostOperationJobExecutor(hostOperations, "system-restart")( - executionContext([]), + restartContext, {} ) ) ).toEqual({ completedAtMs: 5000, status: "accepted" }); + expect(fenceEvents).toEqual(["arm"]); + expect( + await Effect.runPromise( + createHostOperationJobExecutor(hostOperations, "system-cleanup")( + executionContext([]), + {} + ) + ) + ).toEqual({ completedAtMs: 5000, status: "completed" }); expect( await Effect.runPromise( createHostOperationJobExecutor(hostOperations, "system-update")( @@ -138,6 +163,7 @@ describe("worker-only job executor registry", () => { ).toEqual({ completedAtMs: 5000, status: "completed" }); expect(calls).toMatchObject([ { operationId: "system-restart", signal: expect.any(AbortSignal) }, + { operationId: "system-cleanup", signal: expect.any(AbortSignal) }, { operationId: "system-update", signal: expect.any(AbortSignal) }, ]); @@ -148,9 +174,46 @@ describe("worker-only job executor registry", () => { request: () => Promise.resolve({ status: "completed" }), }, "system-restart" - )(executionContext([]), {}) + )(restartContext, {}) ).catch((error: unknown) => error); expect(failure).toBeInstanceOf(Error); + expect(fenceEvents).toEqual(["arm", "arm"]); + + let restartDispatchAccepted = false; + const brokerFailure = await Effect.runPromise( + createHostOperationJobExecutor( + { + availableOperations: () => Promise.resolve([]), + request: () => { + restartDispatchAccepted = true; + return Promise.reject( + new Error("response lost after systemctl accepted dispatch") + ); + }, + }, + "system-restart" + )(restartContext, {}) + ).catch((error: unknown) => error); + expect(brokerFailure).toBeInstanceOf(Error); + expect((brokerFailure as Error).message).toBe("Fixed host operation failed"); + expect((brokerFailure as Error).message).not.toContain("systemctl"); + expect(restartDispatchAccepted).toBeTrue(); + expect(fenceEvents).toEqual(["arm", "arm", "arm"]); + expect(hostSystemRestartJobActionDefinition).toMatchObject({ + attemptLimit: 1, + retrySafe: false, + }); + + const cleanupFailure = await Effect.runPromise( + createHostOperationJobExecutor( + { + availableOperations: () => Promise.resolve([]), + request: () => Promise.resolve({ status: "accepted" }), + }, + "system-cleanup" + )(executionContext([]), {}) + ).catch((error: unknown) => error); + expect(cleanupFailure).toBeInstanceOf(Error); }); test("persists only aggregate OpenClaw cleanup and validated update summaries", async () => { diff --git a/greenfield/src/server/domains/jobs/actionExecutors.ts b/greenfield/src/server/domains/jobs/actionExecutors.ts index f7196c6e0..05ec0c3ee 100644 --- a/greenfield/src/server/domains/jobs/actionExecutors.ts +++ b/greenfield/src/server/domains/jobs/actionExecutors.ts @@ -7,6 +7,7 @@ import { type LogMaintenanceExecutionSummary, logMaintenancePolicyIdSchema, } from "../../../contracts/logs.ts"; +import type { HostOperationId } from "../../../shared/hostOperations.ts"; import type { JsonObject } from "../../../shared/json.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; import { @@ -23,6 +24,9 @@ import { type JobActionSuccessfulSettlementHandler, JobActionOutcomeUnknownError, JobActionRetryableError, + hostSystemCleanupJobActionDefinition, + hostSystemCleanupJobActionKey, + hostSystemCleanupJobResultSchema, hostSystemRestartJobActionDefinition, hostSystemRestartJobActionKey, hostSystemRestartJobResultSchema, @@ -47,14 +51,7 @@ import { workspaceFileWriteJobActionKey, } from "./actionRegistry.ts"; -/** Complete contract-ordered inventory of reviewed privileged host operations. */ -export const hostOperationIds = Object.freeze([ - "system-restart", - "system-update", -] as const); - -/** One exact reviewed privileged host operation. */ -export type HostOperationId = (typeof hostOperationIds)[number]; +export { hostOperationIds } from "../../../shared/hostOperations.ts"; /** Secret-free result returned by one future, separately privileged host adapter. */ export type FixedHostOperationResult = @@ -348,13 +345,25 @@ export function createHostOperationJobExecutor( catch: () => new Error("Fixed host operation failed"), try: async (signal) => { v.parse(emptyPayloadSchema, payload); - const result = await hostOperations.request(operationId, signal); if (operationId === "system-restart") { + await context.armHostRestartClaimFence(); + // The fixed broker cannot prove that an error happened before + // `systemctl start --no-block` accepted the reboot timer. Once + // armed, retain the fence for new-boot reconciliation or its + // bounded same-boot expiry on every ambiguous dispatch outcome. + const result = await hostOperations.request(operationId, signal); return v.parse(hostSystemRestartJobResultSchema, { completedAtMs: context.nowMs(), status: result.status, }); } + const result = await hostOperations.request(operationId, signal); + if (operationId === "system-cleanup") { + return v.parse(hostSystemCleanupJobResultSchema, { + completedAtMs: context.nowMs(), + status: result.status, + }); + } return v.parse(hostSystemUpdateJobResultSchema, { completedAtMs: context.nowMs(), status: result.status, @@ -522,6 +531,7 @@ export function createJobWorkerActionResolver( ...(dependencies.hostOperations === undefined ? [] : [ + hostSystemCleanupJobActionDefinition, hostSystemRestartJobActionDefinition, hostSystemUpdateJobActionDefinition, ]), @@ -583,6 +593,15 @@ export function createJobWorkerActionResolver( "openclaw-update" ) ), + ...gatedExecutor( + hostSystemCleanupJobActionKey, + dependencies.hostOperations === undefined + ? undefined + : createHostOperationJobExecutor( + dependencies.hostOperations, + "system-cleanup" + ) + ), ...gatedExecutor( hostSystemRestartJobActionKey, dependencies.hostOperations === undefined diff --git a/greenfield/src/server/domains/jobs/actionRegistry.test.ts b/greenfield/src/server/domains/jobs/actionRegistry.test.ts index 5dab2bbef..e877b9867 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.test.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.test.ts @@ -4,6 +4,7 @@ import * as v from "valibot"; import { findJobActionDefinition, + hostSystemCleanupJobActionDefinition, hostSystemRestartJobActionDefinition, hostSystemUpdateJobActionDefinition, isRegisteredJobSchedule, @@ -143,7 +144,7 @@ describe("durable job action registry", () => { cancellationPolicy: "never", manualExposure: "none", resourceClass: "exclusive", - resourceKeys: ["openclaw.gateway"], + resourceKeys: ["host.mutation", "openclaw.gateway"], retrySafe: false, }); expect(openClawGatewayRestartJobActionDefinition).not.toHaveProperty( @@ -151,9 +152,11 @@ describe("durable job action registry", () => { ); }); - test("publishes four fixed Service Actions with cross-domain exclusive locks", () => { + test("publishes six fixed Service Actions with cross-domain exclusive locks", () => { for (const definition of [ openClawSessionsCleanupJobActionDefinition, + openClawGatewayRestartJobActionDefinition, + hostSystemCleanupJobActionDefinition, hostSystemRestartJobActionDefinition, hostSystemUpdateJobActionDefinition, ]) { @@ -171,9 +174,17 @@ describe("durable job action registry", () => { "host.mutation", "openclaw.gateway", ]); + expect(openClawGatewayRestartJobActionDefinition.resourceKeys).toEqual([ + "host.mutation", + "openclaw.gateway", + ]); expect(hostSystemRestartJobActionDefinition.resourceKeys).toEqual([ "host.mutation", ]); + expect(hostSystemCleanupJobActionDefinition.resourceKeys).toEqual([ + "host.logs", + "host.mutation", + ]); expect(hostSystemUpdateJobActionDefinition.resourceKeys).toEqual([ "host.mutation", ]); @@ -186,6 +197,7 @@ describe("durable job action registry", () => { retrySafe: false, }); expect(hostSystemRestartJobActionDefinition.timeoutMs).toBe(60_000); + expect(hostSystemCleanupJobActionDefinition.timeoutMs).toBe(2_100_000); expect(hostSystemUpdateJobActionDefinition.timeoutMs).toBe(7_200_000); expect(openClawSessionsCleanupJobActionDefinition.timeoutMs).toBe(630_000); expect(openClawInstallationUpdateJobActionDefinition.timeoutMs).toBe(2_130_000); diff --git a/greenfield/src/server/domains/jobs/actionRegistry.ts b/greenfield/src/server/domains/jobs/actionRegistry.ts index 7c01c52ee..c6e97b5bd 100644 --- a/greenfield/src/server/domains/jobs/actionRegistry.ts +++ b/greenfield/src/server/domains/jobs/actionRegistry.ts @@ -48,6 +48,8 @@ export const openClawGatewayRestartJobActionKey = "openclaw.gateway.restart"; export const openClawSessionsCleanupJobActionKey = "openclaw.sessions.cleanup"; /** Fixed worker-only OpenClaw update identity selected by Service Actions. */ export const openClawInstallationUpdateJobActionKey = "openclaw.installation.update"; +/** Fixed root-brokered host cleanup identity selected by Service Actions. */ +export const hostSystemCleanupJobActionKey = "host.system.cleanup"; /** Fixed root-brokered host restart identity selected by Service Actions. */ export const hostSystemRestartJobActionKey = "host.system.restart"; /** Fixed root-brokered host update identity selected by Service Actions. */ @@ -65,6 +67,12 @@ export const hostSystemRestartJobResultSchema = v.strictObject({ status: v.literal("accepted", "Host restart result is invalid"), }); +/** Redacted terminal result for one fixed host cleanup unit. */ +export const hostSystemCleanupJobResultSchema = v.strictObject({ + completedAtMs: jobTimestampSchema, + status: v.literal("completed", "Host cleanup result is invalid"), +}); + /** Redacted terminal result for one fixed host update unit. */ export const hostSystemUpdateJobResultSchema = v.strictObject({ completedAtMs: jobTimestampSchema, @@ -168,6 +176,8 @@ const jobActionOutputMessageSchema = v.pipe( /** Safe execution context supplied by the worker without host or shell authority. */ export interface JobActionExecutionContext { + readonly armHostRestartClaimFence: () => Promise; + readonly clearHostRestartClaimFence: () => Promise; readonly commitCacheAttempt: ( attempt: JobCacheAttemptCommit ) => Promise; @@ -463,7 +473,7 @@ export const openClawGatewayRestartJobActionDefinition = manualExposure: "none", priority: 20, resourceClass: "exclusive", - resourceKeys: Object.freeze(["openclaw.gateway"]), + resourceKeys: Object.freeze(["host.mutation", "openclaw.gateway"]), retrySafe: false, timeoutMs: 60_000, }); @@ -507,6 +517,16 @@ export const openClawInstallationUpdateJobActionDefinition = serviceActionDefini timeoutMs: 35 * 60_000 + 30_000, }); +/** Non-retryable bounded host cleanup reserved for a separately privileged adapter. */ +export const hostSystemCleanupJobActionDefinition = serviceActionDefinition({ + actionKey: hostSystemCleanupJobActionKey, + description: + "Cleans orphan packages and caches, bounded journal history, and unused Docker content older than seven days without deleting volumes.", + displayName: "Clean up host system", + resourceKeys: Object.freeze(["host.logs", "host.mutation"]), + timeoutMs: 35 * 60_000, +}); + /** Accepted-only host restart request reserved for a separately privileged adapter. */ export const hostSystemRestartJobActionDefinition = serviceActionDefinition({ actionKey: hostSystemRestartJobActionKey, diff --git a/greenfield/src/server/domains/jobs/coordinator.test.ts b/greenfield/src/server/domains/jobs/coordinator.test.ts index 88b94f208..728f1a2bb 100644 --- a/greenfield/src/server/domains/jobs/coordinator.test.ts +++ b/greenfield/src/server/domains/jobs/coordinator.test.ts @@ -123,6 +123,7 @@ function claimedRun(workerId: string, actionKey = "system.worker-smoke"): JobRun queuedAt: at, requestedById: "system.scheduler", requestedByKind: "system", + requiredWorkerReleaseId: null, resourceClass: "light", resourceKeysJson: '["database"]', resultJson: null, @@ -286,6 +287,9 @@ function repositoryFixture(options: RepositoryFixtureOptions = {}) { const eventRun = claims.find((result) => result.kind === "claimed")?.run; let dueSchedules = [...(options.dueSchedules ?? [])]; const repository = { + armHostRestartClaimFence() { + return Promise.resolve({ kind: "lost-claim" as const }); + }, appendClaimEvent(input) { events.push(`append:${input.kind}`); const result = options.appendEvent?.(input) ?? { kind: "dropped" }; @@ -318,6 +322,9 @@ function repositoryFixture(options: RepositoryFixtureOptions = {}) { worker, }); }, + clearHostRestartClaimFence() { + return Promise.resolve({ kind: "changed" as const }); + }, async claimNextRun(input) { claimInputs.push(input); const result = claims.shift() ?? ({ kind: "empty" } as const); @@ -459,6 +466,7 @@ function coordinatorOptions( ); if (smokeDefinition === undefined) throw new Error("Missing smoke definition"); return { + bootIdentity: "00000000-0000-0000-0000-000000000001", databaseReleaseId: releaseId, actionDefinitions: [smokeDefinition], findAction: findJobWorkerAction, diff --git a/greenfield/src/server/domains/jobs/coordinator.ts b/greenfield/src/server/domains/jobs/coordinator.ts index ce76686da..568ffc1af 100644 --- a/greenfield/src/server/domains/jobs/coordinator.ts +++ b/greenfield/src/server/domains/jobs/coordinator.ts @@ -12,6 +12,10 @@ import { } from "../../../contracts/jobModel.ts"; import type { JsonObject } from "../../../shared/json.ts"; import { parseJsonText } from "../../../shared/json.ts"; +import { + type LinuxBootIdentity, + linuxBootIdentitySchema, +} from "../../../shared/linuxBootIdentity.ts"; import { serializeWorkerActionKeys } from "../../database/validation/workerInstances.ts"; import { sha256Hex } from "../../shared/crypto.ts"; import { @@ -59,8 +63,10 @@ export const jobExpiredClaimRecoveryLimit = 32; type JobWorkerRepository = Pick< JobRepository, + | "armHostRestartClaimFence" | "appendClaimEvent" | "beginWorkerDrain" + | "clearHostRestartClaimFence" | "claimNextRun" | "enqueueNextDueSchedule" | "expireDisableIntents" @@ -107,6 +113,7 @@ export interface JobWorkerCoordinatorTimings { export interface JobWorkerCoordinatorOptions { readonly actionDefinitions?: readonly JobExecutableActionDefinition[]; readonly databaseReleaseId: string; + readonly bootIdentity: LinuxBootIdentity; readonly commitCacheAttempt?: (input: { readonly at: Date; readonly attempt: number; @@ -546,6 +553,7 @@ async function waitForActiveExecution( } interface ExecuteClaimOptions { + readonly bootIdentity: LinuxBootIdentity; readonly commitCacheAttempt: JobWorkerCoordinatorOptions["commitCacheAttempt"]; readonly databaseReleaseId: string; readonly findAction: (actionKey: string) => JobActionRegistration | undefined; @@ -677,11 +685,49 @@ async function executeClaim(options: ExecuteClaimOptions): Promise { return result.kind; }; + let hostRestartClaimFence: + | Extract< + Awaited>, + { kind: "armed" } + >["fence"] + | undefined; + let parsedActionPayload: JsonObject | undefined; const action = Effect.suspend(() => { parsedActionPayload = v.parse(jobPayloadSchema, parseJsonText(run.payloadJson)); return registration.execute( Object.freeze({ + armHostRestartClaimFence: async () => { + if (hostRestartClaimFence !== undefined) { + throw new Error("Host restart claim fence is already armed"); + } + const arm = await options.repository.armHostRestartClaimFence({ + at: new Date(options.nowMs()), + bootIdentity: options.bootIdentity, + leaseToken, + runId: run.id, + workerId: options.workerInstanceId, + }); + if (arm.kind === "lost-claim") throw new JobClaimLostError(); + hostRestartClaimFence = arm.fence; + }, + clearHostRestartClaimFence: async () => { + const fence = hostRestartClaimFence; + if (fence === undefined) { + throw new Error("Host restart claim fence is not armed"); + } + const cleared = await options.repository.clearHostRestartClaimFence({ + armedAt: fence.armedAt, + at: new Date(options.nowMs()), + bootIdentity: fence.bootIdentity, + expiresAt: fence.expiresAt, + leaseToken, + runId: run.id, + workerId: options.workerInstanceId, + }); + if (cleared.kind === "changed") throw new JobClaimLostError(); + hostRestartClaimFence = undefined; + }, commitCacheAttempt: async (outcome: JobCacheAttemptCommit) => { if ( registration.manualExposure !== "cache-write" || @@ -779,6 +825,7 @@ async function executeClaim(options: ExecuteClaimOptions): Promise { export function createJobWorkerCoordinator( options: JobWorkerCoordinatorOptions ): JobWorkerCoordinator { + const bootIdentity = v.parse(linuxBootIdentitySchema, options.bootIdentity); const timings = resolveTimings(options.timings); const nowMs = options.nowMs ?? Date.now; const generateId = options.generateId ?? (() => Bun.randomUUIDv7()); @@ -998,6 +1045,7 @@ export function createJobWorkerCoordinator( const leaseToken = generateId(); const claim = await options.repository.claimNextRun({ at, + bootIdentity, ...(claimCursor === undefined ? {} : { cursor: claimCursor }), leaseExpiresAt: addMilliseconds(at, timings.claimLeaseMs), leaseToken, @@ -1057,6 +1105,7 @@ export function createJobWorkerCoordinator( return; } activeExecution = executeClaim({ + bootIdentity, commitCacheAttempt: options.commitCacheAttempt, databaseReleaseId: options.databaseReleaseId, findAction, diff --git a/greenfield/src/server/domains/jobs/logMaintenanceQueue.test.ts b/greenfield/src/server/domains/jobs/logMaintenanceQueue.test.ts index 6266cc2c6..2f2488704 100644 --- a/greenfield/src/server/domains/jobs/logMaintenanceQueue.test.ts +++ b/greenfield/src/server/domains/jobs/logMaintenanceQueue.test.ts @@ -33,6 +33,7 @@ function repositoryFixture() { eventBytes: 0, eventCount: 1, payloadEventCount: 0, + requiredWorkerReleaseId: input.run.requiredWorkerReleaseId ?? null, stateVersion: 1, }; runs.push(stored); diff --git a/greenfield/src/server/domains/jobs/records.ts b/greenfield/src/server/domains/jobs/records.ts index 62e58a8be..df3591430 100644 --- a/greenfield/src/server/domains/jobs/records.ts +++ b/greenfield/src/server/domains/jobs/records.ts @@ -21,6 +21,7 @@ import { scheduleSummarySchema, } from "../../../contracts/jobModel.ts"; import { parseJsonText } from "../../../shared/json.ts"; +import { hostRestartClaimFenceSelectSchema } from "../../database/validation/hostRestartClaimFence.ts"; import { jobDisableIntentSelectSchema } from "../../database/validation/jobDisableIntents.ts"; import { jobRunEventSelectSchema } from "../../database/validation/jobRunEvents.ts"; import { jobRunSelectSchema } from "../../database/validation/jobRuns.ts"; @@ -30,6 +31,9 @@ import { workerInstanceSelectSchema } from "../../database/validation/workerInst import { findJobActionDefinition, isRegisteredJobSchedule } from "./actionRegistry.ts"; export type JobDisableIntentRecord = v.InferOutput; +export type HostRestartClaimFenceRecord = v.InferOutput< + typeof hostRestartClaimFenceSelectSchema +>; export type JobRunEventRecord = v.InferOutput; export type JobRunRecord = v.InferOutput; export type JobWorkerControlRecord = v.InferOutput; diff --git a/greenfield/src/server/domains/jobs/repository.test.ts b/greenfield/src/server/domains/jobs/repository.test.ts index 2768546ba..0aa0285b6 100644 --- a/greenfield/src/server/domains/jobs/repository.test.ts +++ b/greenfield/src/server/domains/jobs/repository.test.ts @@ -14,6 +14,7 @@ import { openFreshMigratedDatabase } from "../../test/support/freshDatabase.ts"; import type { WorkerInstanceRecord } from "./records.ts"; import { createJobRepository, + hostRestartClaimFenceDurationMs, type JobMutationSideEffects, type JobRunEventInsert, type JobRunInsert, @@ -92,6 +93,7 @@ function queuedRun(index: number, overrides: Partial = {}): JobRun queuedAt, requestedById: userId, requestedByKind: "user", + requiredWorkerReleaseId: null, resourceClass: "light", resourceKeysJson: '["database"]', resultJson: null, @@ -1511,6 +1513,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(101_000), leaseExpiresAt: new Date(130_000), leaseToken, @@ -1621,6 +1624,7 @@ describe("durable jobs repository", () => { let claimSideEffectAt: Date | undefined; const firstClaim = await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: (claimed) => { claimSideEffectAt = claimed.updatedAt; return noSideEffects; @@ -1678,6 +1682,7 @@ describe("durable jobs repository", () => { renewedAt: new Date(25_000), }); const secondClaim = await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: new Date(3000), leaseExpiresAt: new Date(13_000), @@ -1767,6 +1772,7 @@ describe("durable jobs repository", () => { expect(settlementSideEffectAt).toEqual(new Date(25_000)); const nowUnblocked = await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: new Date(7000), leaseExpiresAt: new Date(17_000), @@ -1806,6 +1812,360 @@ describe("durable jobs repository", () => { } }); + test("claims release-fenced work only from the exact worker release", async () => { + const database = await openFreshMigratedDatabase(); + const repository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const requiredReleaseId = "b".repeat(40); + const fenced = queuedRun(133, { + requiredWorkerReleaseId: requiredReleaseId, + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + const general = queuedRun(134, { + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + + try { + for (const run of [fenced, general]) { + expect( + await repository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(run), + run, + }) + ).toMatchObject({ kind: "inserted" }); + } + await repository.registerWorker({ + ...noSideEffects, + worker: worker(workerOneId), + }); + await repository.registerWorker({ + ...noSideEffects, + worker: { ...worker(workerTwoId), releaseId: requiredReleaseId }, + }); + + const oldReleaseClaim = await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", + at: new Date(3000), + leaseExpiresAt: new Date(13_000), + leaseToken: uuid(135), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerOneId, + }); + expect(oldReleaseClaim).toMatchObject({ + kind: "claimed", + run: { id: general.id, requiredWorkerReleaseId: null }, + }); + expect( + await repository.settleClaim({ + at: new Date(4000), + leaseToken: uuid(135), + outcome: { kind: "succeeded", resultJson: '{"status":"ok"}' }, + runId: general.id, + sideEffectsForRun: () => noSideEffects, + workerId: workerOneId, + }) + ).toMatchObject({ kind: "settled" }); + expect( + await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", + at: new Date(5000), + leaseExpiresAt: new Date(15_000), + leaseToken: uuid(136), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerOneId, + }) + ).toEqual({ kind: "empty" }); + expect( + await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", + at: new Date(5000), + leaseExpiresAt: new Date(15_000), + leaseToken: uuid(137), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toMatchObject({ + kind: "claimed", + run: { id: fenced.id, requiredWorkerReleaseId: requiredReleaseId }, + }); + } finally { + database.sqlite.close(true); + } + }); + + test("blocks every repository claim behind one claim-owned restart fence", async () => { + const database = await openFreshMigratedDatabase(); + const restartRepository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const secondRepository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const bootIdentity = "00000000-0000-0000-0000-000000000001"; + const nextBootIdentity = "00000000-0000-0000-0000-000000000002"; + const restartLeaseToken = uuid(813); + const restartRun = queuedRun(813, { + actionKey: "host.system.restart", + attemptLimit: 1, + cancellationPolicy: "never", + displayName: "Restart host system", + priority: 20, + requiredWorkerReleaseId: "a".repeat(40), + resourceClass: "exclusive", + resourceKeysJson: '["host.mutation"]', + retrySafe: false, + scheduledJobId: null, + scheduledJobVersion: null, + }); + const ordinaryRun = queuedRun(814, { + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + const afterRebootRun = queuedRun(815, { + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + try { + await restartRepository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(restartRun), + run: restartRun, + }); + for (const workerId of [workerOneId, workerTwoId]) { + await restartRepository.registerWorker({ + ...noSideEffects, + worker: worker( + workerId, + 1, + '["host.system.restart","system.worker-smoke"]' + ), + }); + } + expect( + await restartRepository.claimNextRun({ + at: new Date(3000), + bootIdentity, + leaseExpiresAt: new Date(600_000), + leaseToken: restartLeaseToken, + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerOneId, + }) + ).toMatchObject({ kind: "claimed", run: { id: restartRun.id } }); + expect( + await restartRepository.armHostRestartClaimFence({ + at: new Date(3998), + bootIdentity, + leaseToken: uuid(899), + runId: restartRun.id, + workerId: workerOneId, + }) + ).toEqual({ kind: "lost-claim" }); + expect( + await restartRepository.armHostRestartClaimFence({ + at: new Date(3999), + bootIdentity, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerTwoId, + }) + ).toEqual({ kind: "lost-claim" }); + const armed = await restartRepository.armHostRestartClaimFence({ + at: new Date(4000), + bootIdentity, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerOneId, + }); + expect(armed).toMatchObject({ kind: "armed" }); + if (armed.kind !== "armed") throw new Error("Expected restart fence"); + + await secondRepository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(ordinaryRun), + run: ordinaryRun, + }); + expect( + await secondRepository.claimNextRun({ + at: new Date(5000), + bootIdentity, + leaseExpiresAt: new Date(30_000), + leaseToken: uuid(814), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toEqual({ kind: "restart-fenced" }); + expect( + await restartRepository.clearHostRestartClaimFence({ + armedAt: armed.fence.armedAt, + at: new Date(6000), + bootIdentity: armed.fence.bootIdentity, + expiresAt: armed.fence.expiresAt, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerOneId, + }) + ).toEqual({ kind: "cleared" }); + + const rearmed = await restartRepository.armHostRestartClaimFence({ + at: new Date(7000), + bootIdentity, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerOneId, + }); + expect(rearmed.kind).toBe("armed"); + expect( + await secondRepository.claimNextRun({ + at: new Date(8000), + bootIdentity: nextBootIdentity, + leaseExpiresAt: new Date(30_000), + leaseToken: uuid(815), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toMatchObject({ kind: "claimed", run: { id: ordinaryRun.id } }); + await secondRepository.settleClaim({ + at: new Date(9000), + leaseToken: uuid(815), + outcome: { kind: "succeeded", resultJson: "{}" }, + runId: ordinaryRun.id, + sideEffectsForRun: () => noSideEffects, + workerId: workerTwoId, + }); + + const expiryArmAt = new Date(10_000); + expect( + await restartRepository.armHostRestartClaimFence({ + at: expiryArmAt, + bootIdentity: nextBootIdentity, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerOneId, + }) + ).toMatchObject({ kind: "armed" }); + await secondRepository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(afterRebootRun), + run: afterRebootRun, + }); + expect( + await secondRepository.claimNextRun({ + at: new Date( + expiryArmAt.getTime() + hostRestartClaimFenceDurationMs + 1 + ), + bootIdentity: nextBootIdentity, + leaseExpiresAt: new Date( + expiryArmAt.getTime() + hostRestartClaimFenceDurationMs + 30_000 + ), + leaseToken: uuid(816), + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toMatchObject({ kind: "claimed", run: { id: afterRebootRun.id } }); + } finally { + database.sqlite.close(true); + } + }); + + test("refuses restart fence arming while any other run is globally running", async () => { + const database = await openFreshMigratedDatabase(); + const repository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + const bootIdentity = "00000000-0000-0000-0000-000000000001"; + const restartLeaseToken = uuid(817); + const ordinaryLeaseToken = uuid(818); + const restartRun = queuedRun(817, { + actionKey: "host.system.restart", + attemptLimit: 1, + cancellationPolicy: "never", + displayName: "Restart host system", + priority: 20, + requiredWorkerReleaseId: "a".repeat(40), + resourceClass: "exclusive", + resourceKeysJson: "[]", + retrySafe: false, + scheduledJobId: null, + scheduledJobVersion: null, + }); + const ordinaryRun = queuedRun(818, { + resourceKeysJson: "[]", + scheduledJobId: null, + scheduledJobVersion: null, + }); + try { + for (const run of [restartRun, ordinaryRun]) { + await repository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(run), + run, + }); + } + for (const workerId of [workerOneId, workerTwoId]) { + await repository.registerWorker({ + ...noSideEffects, + worker: worker( + workerId, + 1, + '["host.system.restart","system.worker-smoke"]' + ), + }); + } + expect( + await repository.claimNextRun({ + at: new Date(3000), + bootIdentity, + leaseExpiresAt: new Date(30_000), + leaseToken: restartLeaseToken, + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerOneId, + }) + ).toMatchObject({ kind: "claimed", run: { id: restartRun.id } }); + expect( + await repository.claimNextRun({ + at: new Date(3001), + bootIdentity, + leaseExpiresAt: new Date(30_000), + leaseToken: ordinaryLeaseToken, + minimumHeartbeatAt: new Date(1000), + sideEffectsForClaim: () => noSideEffects, + workerId: workerTwoId, + }) + ).toMatchObject({ kind: "claimed", run: { id: ordinaryRun.id } }); + expect( + await repository.armHostRestartClaimFence({ + at: new Date(4000), + bootIdentity, + leaseToken: restartLeaseToken, + runId: restartRun.id, + workerId: workerOneId, + }) + ).toEqual({ kind: "lost-claim" }); + } finally { + database.sqlite.close(true); + } + }); + test("settles a durable cancellation that races worker shutdown as cancelled", async () => { const database = await openFreshMigratedDatabase(); const repository = createJobRepository( @@ -1839,6 +2199,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(3000), leaseExpiresAt: new Date(30_000), leaseToken, @@ -2039,6 +2400,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(10_000), leaseExpiresAt: new Date(30_000), leaseToken: uuid(140), @@ -2072,6 +2434,7 @@ describe("durable jobs repository", () => { } const firstPage = await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(11_000), leaseExpiresAt: new Date(31_000), leaseToken: uuid(141), @@ -2103,6 +2466,7 @@ describe("durable jobs repository", () => { ).toMatchObject({ kind: "inserted" }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(12_000), cursor: firstPage.cursor, leaseExpiresAt: new Date(32_000), @@ -2117,6 +2481,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(12_000), cursor: firstPage.cursor, leaseExpiresAt: new Date(32_000), @@ -2212,6 +2577,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(3000), leaseExpiresAt: new Date(30_000), leaseToken: uuid(450), @@ -2226,6 +2592,7 @@ describe("durable jobs repository", () => { expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(6000), cursor, leaseExpiresAt: new Date(36_000), @@ -2572,6 +2939,7 @@ describe("durable jobs repository", () => { expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(5000), leaseExpiresAt: new Date(35_000), leaseToken: uuid(716), @@ -2585,6 +2953,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(5001), leaseExpiresAt: new Date(35_001), leaseToken: uuid(717), @@ -2638,6 +3007,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: new Date(3000), leaseExpiresAt: new Date(30_000), @@ -2725,6 +3095,7 @@ describe("durable jobs repository", () => { leaseToken = uuid(160 + attempt); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: retryAt, leaseExpiresAt: new Date(retryAt.getTime() + 30_000), @@ -3055,6 +3426,7 @@ describe("durable jobs repository", () => { ).toMatchObject({ control: { version: 2 }, kind: "updated" }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: new Date(3000), leaseExpiresAt: new Date(5000), @@ -3072,6 +3444,7 @@ describe("durable jobs repository", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", sideEffectsForClaim: () => noSideEffects, at: new Date(3000), leaseExpiresAt: new Date(5000), @@ -3144,6 +3517,7 @@ describe("durable jobs repository", () => { run: regressedRun, }); await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(8000), leaseExpiresAt: new Date(10_000), leaseToken: uuid(82), @@ -3310,6 +3684,7 @@ describe("durable jobs repository", () => { const activeReal = await enqueue(83, realPayload); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: new Date(4000), leaseExpiresAt: new Date(10_000), leaseToken: uuid(830), @@ -3409,4 +3784,71 @@ describe("durable jobs repository", () => { database.sqlite.close(true); } }); + + test("reads the latest terminal empty-payload Service Action run", async () => { + const database = await openFreshMigratedDatabase(); + const repository = createJobRepository( + database.orm, + testImmediateDatabaseWriteAdmission + ); + async function enqueueAndCancel(index: number, payloadJson: string) { + const run = queuedRun(index, { + actionKey: "host.system.update", + attemptLimit: 1, + cancellationPolicy: "cooperative", + displayName: "Update host system", + payloadJson, + resourceClass: "exclusive", + resourceKeysJson: '["host.mutation"]', + retrySafe: false, + scheduledJobId: null, + scheduledJobVersion: null, + }); + await repository.enqueueManualRun({ + ...noSideEffects, + queuedEvent: queuedEvent(run), + run, + }); + await repository.cancelRun({ + actor: { id: userId, kind: "user" }, + at: new Date(2000 + index), + id: run.id, + sideEffectsForRun: () => noSideEffects, + terminalCode: "job/cancel-requested", + terminalMessage: "Cancel the Service Action fixture.", + }); + return run; + } + try { + const older = await enqueueAndCancel(86, "{}"); + const latest = await enqueueAndCancel(87, "{}"); + const otherPayload = await enqueueAndCancel(88, '{"unexpected":true}'); + + expect( + repository.readActionPayloadRunSnapshots({ + actionKey: "host.system.update", + payloadJsons: ["{}"], + }) + ).toMatchObject([ + { + lastRun: { id: latest.id, state: "cancelled" }, + payloadJson: "{}", + }, + ]); + expect( + repository.readActionPayloadRunSnapshots({ + actionKey: "host.system.update", + payloadJsons: ['{"unexpected":true}'], + }) + ).toMatchObject([ + { + lastRun: { id: otherPayload.id, state: "cancelled" }, + payloadJson: '{"unexpected":true}', + }, + ]); + expect(older.id).not.toBe(latest.id); + } finally { + database.sqlite.close(true); + } + }); }); diff --git a/greenfield/src/server/domains/jobs/repository.ts b/greenfield/src/server/domains/jobs/repository.ts index 8a8d8c1ad..a25c955e9 100644 --- a/greenfield/src/server/domains/jobs/repository.ts +++ b/greenfield/src/server/domains/jobs/repository.ts @@ -49,6 +49,10 @@ import { } from "../../../contracts/schedules.ts"; import { utf8ByteLength } from "../../../shared/encoding.ts"; import { parseJsonText } from "../../../shared/json.ts"; +import { + linuxBootIdentitySchema, + type LinuxBootIdentity, +} from "../../../shared/linuxBootIdentity.ts"; import { logMaintenanceJobActionKey, logMaintenanceJobPayloadIndexMaximumBytes, @@ -56,6 +60,7 @@ import { import { fullCommitShaSchema } from "../../../shared/validation.ts"; import type { ImmediateDatabaseWriteAdmission } from "../../database/immediateWriteAdmission.ts"; import { auditEvents } from "../../database/schema/auditEvents.ts"; +import { hostRestartClaimFence } from "../../database/schema/hostRestartClaimFence.ts"; import { jobDisableIntents } from "../../database/schema/jobDisableIntents.ts"; import { jobRunEvents } from "../../database/schema/jobRunEvents.ts"; import { jobRuns } from "../../database/schema/jobRuns.ts"; @@ -65,6 +70,10 @@ import { resourceLeases } from "../../database/schema/resourceLeases.ts"; import { scheduledJobs } from "../../database/schema/scheduledJobs.ts"; import { workerInstances } from "../../database/schema/workerInstances.ts"; import { auditEventInsertSchema } from "../../database/validation/auditEvents.ts"; +import { + hostRestartClaimFenceInsertSchema, + hostRestartClaimFenceSelectSchema, +} from "../../database/validation/hostRestartClaimFence.ts"; import { jobDisableIntentCloseSchema, jobDisableIntentInsertSchema, @@ -100,6 +109,15 @@ import { } from "../../database/validation/workerInstances.ts"; import type { SecurityAuditEvent } from "../security/audit.ts"; import { + hostSystemCleanupJobActionKey, + hostSystemRestartJobActionKey, + hostSystemUpdateJobActionKey, + openClawGatewayRestartJobActionKey, + openClawInstallationUpdateJobActionKey, + openClawSessionsCleanupJobActionKey, +} from "./actionRegistry.ts"; +import { + type HostRestartClaimFenceRecord, type JobDisableIntentRecord, type JobRunEventRecord, type JobRunRecord, @@ -114,12 +132,15 @@ type JobPersistenceDatabase = JobTransaction | SQLiteBunDatabase; export type JobDisableIntentInsert = v.InferOutput; export type JobDisableIntentClose = v.InferOutput; -export type JobRunInsert = v.InferOutput; +export type JobRunInsert = v.InferInput; export type JobRunEventInsert = v.InferOutput; export type JobRealtimeEventInsert = v.InferOutput; export type ScheduledJobInsert = v.InferOutput; export type WorkerInstanceInsert = v.InferOutput; +/** Same-boot recovery ceiling when an accepted restart never reboots the host. */ +export const hostRestartClaimFenceDurationMs = 5 * 60_000; + export interface JobMutationSideEffects { readonly auditEvents: readonly SecurityAuditEvent[]; readonly realtimeEvents: readonly JobRealtimeEventInsert[]; @@ -418,6 +439,7 @@ export interface JobClaimCursor { export interface ClaimNextRunInput { readonly at: Date; + readonly bootIdentity: LinuxBootIdentity; readonly cursor?: JobClaimCursor; readonly leaseExpiresAt: Date; readonly leaseToken: string; @@ -434,6 +456,7 @@ export type JobClaimResult = readonly kind: "page-exhausted"; } | { readonly kind: "paused" } + | { readonly kind: "restart-fenced" } | { readonly kind: "worker-unavailable" }; export interface ClaimFenceInput { @@ -443,6 +466,24 @@ export interface ClaimFenceInput { readonly workerId: string; } +export interface ArmHostRestartClaimFenceInput extends ClaimFenceInput { + readonly bootIdentity: LinuxBootIdentity; +} + +export type ArmHostRestartClaimFenceResult = + | { readonly fence: HostRestartClaimFenceRecord; readonly kind: "armed" } + | { readonly kind: "lost-claim" }; + +export interface ClearHostRestartClaimFenceInput extends ClaimFenceInput { + readonly armedAt: Date; + readonly bootIdentity: LinuxBootIdentity; + readonly expiresAt: Date; +} + +export type ClearHostRestartClaimFenceResult = + | { readonly kind: "cleared" } + | { readonly kind: "changed" }; + export interface RenewClaimInput extends ClaimFenceInput { readonly leaseExpiresAt: Date; } @@ -534,9 +575,15 @@ export interface JobRunPageSnapshot { } export interface JobRepository extends JobRepositoryReader { + armHostRestartClaimFence( + input: ArmHostRestartClaimFenceInput + ): Promise; appendClaimEvent(input: AppendClaimEventInput): Promise; beginWorkerDrain(input: WorkerLifecycleMutationInput): Promise; cancelRun(input: CancelRunRepositoryInput): Promise; + clearHostRestartClaimFence( + input: ClearHostRestartClaimFenceInput + ): Promise; claimNextRun(input: ClaimNextRunInput): Promise; enqueueManualRun( input: EnqueueManualRunInput, @@ -581,6 +628,15 @@ const terminalRunStateList = [ "succeeded", "timed-out", ] as const satisfies readonly JobRunState[]; +const serviceActionJobActionKeyList = [ + openClawSessionsCleanupJobActionKey, + openClawGatewayRestartJobActionKey, + openClawInstallationUpdateJobActionKey, + hostSystemCleanupJobActionKey, + hostSystemRestartJobActionKey, + hostSystemUpdateJobActionKey, +] as const; +const serviceActionJobActionKeys = new Set(serviceActionJobActionKeyList); // SQLite partial-index matching requires the same literal state predicates as the DDL. function literalStateList(states: readonly JobRunState[]): SQL { return sql.raw(states.map((state) => `'${state}'`).join(", ")); @@ -588,6 +644,7 @@ function literalStateList(states: readonly JobRunState[]): SQL { const activeStateFilter = sql`${jobRuns.state} IN (${literalStateList(activeRunStateList)})`; const terminalStateFilter = sql`${jobRuns.state} IN (${literalStateList(terminalRunStateList)})`; const logMaintenanceSnapshotScopeFilter = sql`${jobRuns.actionKey} = ${sql.raw(`'${logMaintenanceJobActionKey}'`)} AND length(CAST(${jobRuns.payloadJson} AS BLOB)) <= ${sql.raw(String(logMaintenanceJobPayloadIndexMaximumBytes))}`; +const serviceActionSnapshotScopeFilter = sql`${jobRuns.actionKey} IN (${sql.raw(serviceActionJobActionKeyList.map((actionKey) => `'${actionKey}'`).join(", "))}) AND ${jobRuns.payloadJson} = '{}'`; const terminalRunStates = new Set(terminalRunStateList); function requiredRow(row: T | undefined, operation: string): T { @@ -869,6 +926,7 @@ class DrizzleJobReader implements JobRepositoryReader { "action payload run snapshot" ); const usesMaintenanceStatusIndex = actionKey === logMaintenanceJobActionKey; + const usesServiceActionStatusIndex = serviceActionJobActionKeys.has(actionKey); const payloadJsons = input.payloadJsons.map((payloadJson) => { if (utf8ByteLength(payloadJson) > jobPayloadMaximumBytes) { throw new TypeError( @@ -895,6 +953,12 @@ class DrizzleJobReader implements JobRepositoryReader { } return payloadJsons.map((payloadJson) => { const actionCondition = eq(jobRuns.actionKey, actionKey); + let terminalScopeCondition = actionCondition; + if (usesMaintenanceStatusIndex) { + terminalScopeCondition = logMaintenanceSnapshotScopeFilter; + } else if (usesServiceActionStatusIndex && payloadJson === "{}") { + terminalScopeCondition = sql`${actionCondition} AND ${serviceActionSnapshotScopeFilter}`; + } const activeRow = this.database .select() .from(jobRuns) @@ -912,9 +976,7 @@ class DrizzleJobReader implements JobRepositoryReader { .from(jobRuns) .where( and( - usesMaintenanceStatusIndex - ? logMaintenanceSnapshotScopeFilter - : actionCondition, + terminalScopeCondition, eq(jobRuns.payloadJson, payloadJson), terminalStateFilter ) @@ -1335,6 +1397,136 @@ class DrizzleJobWriter extends DrizzleJobReader { this.#transaction = transaction; } + #reconcileHostRestartClaimFence( + bootIdentity: LinuxBootIdentity, + at: Date + ): HostRestartClaimFenceRecord | undefined { + const row = this.#transaction + .select() + .from(hostRestartClaimFence) + .where(eq(hostRestartClaimFence.id, 1)) + .get(); + if (row === undefined) return; + const fence = v.parse(hostRestartClaimFenceSelectSchema, row); + if ( + fence.bootIdentity === bootIdentity && + getTime(fence.expiresAt) > getTime(at) + ) { + return fence; + } + const removed = this.#transaction + .delete(hostRestartClaimFence) + .where( + and( + eq(hostRestartClaimFence.id, fence.id), + eq(hostRestartClaimFence.armedAt, fence.armedAt), + eq(hostRestartClaimFence.bootIdentity, fence.bootIdentity), + eq(hostRestartClaimFence.expiresAt, fence.expiresAt), + eq(hostRestartClaimFence.jobRunId, fence.jobRunId), + eq(hostRestartClaimFence.leaseToken, fence.leaseToken), + eq(hostRestartClaimFence.workerInstanceId, fence.workerInstanceId) + ) + ) + .returning({ id: hostRestartClaimFence.id }) + .get(); + if (removed === undefined) { + throw new Error("Host restart claim fence reconciliation failed"); + } + return; + } + + public armHostRestartClaimFence( + input: ArmHostRestartClaimFenceInput + ): ArmHostRestartClaimFenceResult { + const bootIdentity = v.parse(linuxBootIdentitySchema, input.bootIdentity); + const atMs = v.parse(jobTimestampSchema, getTime(input.at)); + const at = new Date(atMs); + if (this.#reconcileHostRestartClaimFence(bootIdentity, at) !== undefined) { + return { kind: "lost-claim" }; + } + const run = this.#transaction + .select() + .from(jobRuns) + .where( + and( + eq(jobRuns.id, input.runId), + eq(jobRuns.actionKey, hostSystemRestartJobActionKey), + eq(jobRuns.payloadJson, "{}"), + eq(jobRuns.state, "running"), + eq(jobRuns.leaseOwnerId, input.workerId), + eq(jobRuns.leaseToken, input.leaseToken), + gt(jobRuns.leaseExpiresAt, at) + ) + ) + .get(); + if (run === undefined) return { kind: "lost-claim" }; + parseRun(run); + const runningCount = requiredRow( + this.#transaction + .select({ value: count() }) + .from(jobRuns) + .where(eq(jobRuns.state, "running")) + .get(), + "host restart global running count" + ).value; + if (runningCount !== 1) return { kind: "lost-claim" }; + const expiresAt = new Date( + v.parse(jobTimestampSchema, atMs + hostRestartClaimFenceDurationMs) + ); + const inserted = this.#transaction + .insert(hostRestartClaimFence) + .values( + v.parse(hostRestartClaimFenceInsertSchema, { + armedAt: at, + bootIdentity, + expiresAt, + id: 1, + jobRunId: input.runId, + leaseToken: input.leaseToken, + workerInstanceId: input.workerId, + }) + ) + .returning() + .get(); + return { + fence: v.parse( + hostRestartClaimFenceSelectSchema, + requiredRow(inserted, "host restart claim fence insert") + ), + kind: "armed", + }; + } + + public clearHostRestartClaimFence( + input: ClearHostRestartClaimFenceInput + ): ClearHostRestartClaimFenceResult { + const expected = v.parse(hostRestartClaimFenceInsertSchema, { + armedAt: input.armedAt, + bootIdentity: input.bootIdentity, + expiresAt: input.expiresAt, + id: 1, + jobRunId: input.runId, + leaseToken: input.leaseToken, + workerInstanceId: input.workerId, + }); + const removed = this.#transaction + .delete(hostRestartClaimFence) + .where( + and( + eq(hostRestartClaimFence.id, expected.id), + eq(hostRestartClaimFence.armedAt, expected.armedAt), + eq(hostRestartClaimFence.bootIdentity, expected.bootIdentity), + eq(hostRestartClaimFence.expiresAt, expected.expiresAt), + eq(hostRestartClaimFence.jobRunId, expected.jobRunId), + eq(hostRestartClaimFence.leaseToken, expected.leaseToken), + eq(hostRestartClaimFence.workerInstanceId, expected.workerInstanceId) + ) + ) + .returning({ id: hostRestartClaimFence.id }) + .get(); + return { kind: removed === undefined ? "changed" : "cleared" }; + } + public reconcileSchedules(input: ReconcileSchedulesInput): ScheduledJobRecord[] { const records: ScheduledJobRecord[] = []; const registeredScheduleIds = new Set( @@ -2136,6 +2328,11 @@ class DrizzleJobWriter extends DrizzleJobReader { } public claimNextRun(input: ClaimNextRunInput): JobClaimResult { + const bootIdentity = v.parse(linuxBootIdentitySchema, input.bootIdentity); + const at = new Date(v.parse(jobTimestampSchema, getTime(input.at))); + if (this.#reconcileHostRestartClaimFence(bootIdentity, at) !== undefined) { + return { kind: "restart-fenced" }; + } if (getTime(input.leaseExpiresAt) <= getTime(input.at)) { throw new RangeError("Claim lease expiry must be after claim time"); } @@ -2187,6 +2384,10 @@ class DrizzleJobWriter extends DrizzleJobReader { eq(jobRuns.state, "queued"), lte(jobRuns.availableAt, availableThrough), inArray(jobRuns.actionKey, workerActionKeys), + or( + isNull(jobRuns.requiredWorkerReleaseId), + eq(jobRuns.requiredWorkerReleaseId, worker.releaseId) + ), range ) ) @@ -2265,7 +2466,11 @@ class DrizzleJobWriter extends DrizzleJobReader { eq(jobRuns.id, candidate.id), eq(jobRuns.state, "queued"), eq(jobRuns.stateVersion, candidate.stateVersion), - lte(jobRuns.availableAt, availableThrough) + lte(jobRuns.availableAt, availableThrough), + or( + isNull(jobRuns.requiredWorkerReleaseId), + eq(jobRuns.requiredWorkerReleaseId, worker.releaseId) + ) ) ) .returning() @@ -2848,12 +3053,16 @@ export function createJobRepository( }); return Object.freeze({ + armHostRestartClaimFence: (input: ArmHostRestartClaimFenceInput) => + write((writer) => writer.armHostRestartClaimFence(input)), appendClaimEvent: (input: AppendClaimEventInput) => write((writer) => writer.appendClaimEvent(input)), beginWorkerDrain: (input: WorkerLifecycleMutationInput) => write((writer) => writer.beginWorkerDrain(input)), cancelRun: (input: CancelRunRepositoryInput) => write((writer) => writer.cancelRun(input)), + clearHostRestartClaimFence: (input: ClearHostRestartClaimFenceInput) => + write((writer) => writer.clearHostRestartClaimFence(input)), claimNextRun: (input: ClaimNextRunInput) => write((writer) => writer.claimNextRun(input)), enqueueManualRun: ( diff --git a/greenfield/src/server/domains/jobs/service.test.ts b/greenfield/src/server/domains/jobs/service.test.ts index 1d74699ee..7ef02bddb 100644 --- a/greenfield/src/server/domains/jobs/service.test.ts +++ b/greenfield/src/server/domains/jobs/service.test.ts @@ -291,6 +291,7 @@ describe("durable jobs service", () => { }); expect( await repository.claimNextRun({ + bootIdentity: "00000000-0000-0000-0000-000000000001", at: transitionAt, leaseExpiresAt: new Date(transitionAt.getTime() + 30_000), leaseToken, diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts index dd4c3501f..67078d6a8 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.test.ts @@ -34,6 +34,7 @@ const actor = Object.freeze({ kind: "user" as const, }); const idempotencyKey = "019fdf50-0000-4000-8000-000000000012"; +const requiredWorkerReleaseId = "b".repeat(40); function definition(actionId: ServiceActionId): JobUnscheduledActionDefinition { return Object.freeze({ @@ -74,6 +75,7 @@ function repositoryFixture() { eventBytes: 0, eventCount: 1, payloadEventCount: 0, + requiredWorkerReleaseId: input.run.requiredWorkerReleaseId ?? null, stateVersion: 1, }; return Promise.resolve({ kind: "inserted", run: stored }); @@ -129,6 +131,7 @@ function queueFixture( generateId: () => ids.shift()!, nowMs: () => 1000, repository: fixture.repository, + requiredWorkerReleaseId, ...overrides, }), }; @@ -172,6 +175,7 @@ describe("Service Action durable queue", () => { payloadJson: "{}", requestedById: actor.id, requestedByKind: "user", + requiredWorkerReleaseId, resourceClass: "exclusive", resourceKeysJson: JSON.stringify(definitions[actionId].resourceKeys), retrySafe: false, @@ -209,6 +213,28 @@ describe("Service Action durable queue", () => { expect(fixture.enqueues).toHaveLength(1); }); + test("fails closed without a verified release while preserving durable replays", async () => { + const fixture = repositoryFixture(); + const admitted = queueFixture(fixture); + const first = await admitted.queue.enqueue(request("system-update")); + const unverified = queueFixture(fixture, { + requiredWorkerReleaseId: undefined, + }); + + expect(await unverified.queue.enqueue(request("system-update"))).toEqual(first); + const failure = await unverified.queue + .enqueue( + request("system-update", { + idempotencyKey: "019fdf50-0000-4000-8000-000000000099", + }) + ) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(ServiceActionQueueError); + expect(failure).toMatchObject({ reason: "unavailable" }); + expect(fixture.enqueues).toHaveLength(1); + }); + test("binds one idempotency key to the exact action and authenticator session", async () => { const { fixture, queue } = queueFixture(); await queue.enqueue(request("system-update")); @@ -358,6 +384,7 @@ describe("Service Action durable queue", () => { generateId: () => runIds.shift()!, nowMs: () => 1000, repository: jobRepository, + requiredWorkerReleaseId, }); const authorizationFailure = new Error("authorization expired"); const authenticatedActor = Object.freeze({ @@ -420,6 +447,7 @@ describe("Service Action durable queue", () => { }, }, repository: repositoryFixture().repository, + requiredWorkerReleaseId, }) ).toThrow("Service Action definition is invalid"); @@ -438,6 +466,7 @@ describe("Service Action durable queue", () => { }, }, repository: repositoryFixture().repository, + requiredWorkerReleaseId, }) ).toThrow("Service Action definition is invalid"); } diff --git a/greenfield/src/server/domains/jobs/serviceActionQueue.ts b/greenfield/src/server/domains/jobs/serviceActionQueue.ts index cb7bcafc4..470d3be9a 100644 --- a/greenfield/src/server/domains/jobs/serviceActionQueue.ts +++ b/greenfield/src/server/domains/jobs/serviceActionQueue.ts @@ -6,10 +6,13 @@ import { serviceActionIds, } from "../../../contracts/serviceActions.ts"; import { parseJsonText } from "../../../shared/json.ts"; +import { fullCommitShaSchema } from "../../../shared/validation.ts"; import { sha256Hex } from "../../shared/crypto.ts"; import { + hostSystemCleanupJobActionKey, hostSystemRestartJobActionKey, hostSystemUpdateJobActionKey, + openClawGatewayRestartJobActionKey, openClawInstallationUpdateJobActionKey, openClawSessionsCleanupJobActionKey, type JobUnscheduledActionDefinition, @@ -27,7 +30,9 @@ const emptyPayloadSchema = v.strictObject({}); /** Exact worker action selected by each browser-visible Service Action. */ export const serviceActionJobActionKeys = Object.freeze({ "openclaw-cleanup": openClawSessionsCleanupJobActionKey, + "openclaw-restart": openClawGatewayRestartJobActionKey, "openclaw-update": openClawInstallationUpdateJobActionKey, + "system-cleanup": hostSystemCleanupJobActionKey, "system-restart": hostSystemRestartJobActionKey, "system-update": hostSystemUpdateJobActionKey, } as const satisfies Readonly>); @@ -76,6 +81,7 @@ export interface ServiceActionQueueDependencies { >; readonly generateId?: () => string; readonly nowMs?: () => number; + readonly requiredWorkerReleaseId?: string; readonly repository: Pick; readonly wakeEventPump?: () => Promise | void; } @@ -151,7 +157,7 @@ function result( } /** - * Creates the actor- and authenticator-bound durable queue for four exact Service Actions. + * Creates the actor- and authenticator-bound durable queue for six exact Service Actions. * The queue returns after durable admission and never waits for worker settlement. * @returns The purpose-built fixed-action enqueue boundary. */ @@ -161,6 +167,15 @@ export function createServiceActionQueue( const definitions = prepareDefinitions(dependencies.definitions); const generateId = dependencies.generateId ?? (() => Bun.randomUUIDv7()); const nowMs = dependencies.nowMs ?? Date.now; + const requiredWorkerReleaseId = + dependencies.requiredWorkerReleaseId === undefined + ? undefined + : v.parse( + fullCommitShaSchema( + "Required Service Action worker release is invalid" + ), + dependencies.requiredWorkerReleaseId + ); async function wakeQueuedRun(run: JobRunRecord): Promise { if (run.state !== "queued") return; @@ -203,6 +218,9 @@ export function createServiceActionQueue( if (run === undefined) throw new ServiceActionQueueError("conflict"); return result(request.actionId, run); } + if (requiredWorkerReleaseId === undefined) { + throw new ServiceActionQueueError("unavailable"); + } const atMs = nowMs(); if (!Number.isSafeInteger(atMs) || atMs < 0) { @@ -259,6 +277,7 @@ export function createServiceActionQueue( queuedAt: at, requestedById: request.actor.id, requestedByKind: request.actor.kind, + requiredWorkerReleaseId, resourceClass: definition.resourceClass, resourceKeysJson: JSON.stringify(definition.resourceKeys), resultJson: null, diff --git a/greenfield/src/server/domains/jobs/workerRuntime.test.ts b/greenfield/src/server/domains/jobs/workerRuntime.test.ts index 5a8de46a4..1e9df8d73 100644 --- a/greenfield/src/server/domains/jobs/workerRuntime.test.ts +++ b/greenfield/src/server/domains/jobs/workerRuntime.test.ts @@ -21,8 +21,10 @@ const noSideEffects = Object.freeze({ auditEvents: Object.freeze([]), realtimeEvents: Object.freeze([]), }); +const bootIdentity = "00000000-0000-0000-0000-000000000001"; const baseRuntimeOptions = { + bootIdentity, database: { migrationsDirectory: "/srv/mira-dashboard/releases/test/migrations", releaseId: "a".repeat(40), @@ -155,6 +157,7 @@ function runtimeFixture(initializationFailure?: Error) { }, createCoordinator(options) { events.push("coordinator-create"); + expect(options.bootIdentity).toBe(bootIdentity); expect(options.repository).toBe(repository); expect(options.commitCacheAttempt).toBeFunction(); expect(options.findAction?.("maintenance.rotate-logs")).toBeDefined(); @@ -362,10 +365,15 @@ describe("Dashboard worker runtime", () => { const options: DashboardWorkerRuntimeOptions = { ...fixture.options, hostOperations: { - availableOperations: () => Promise.resolve(["system-restart"]), - request: () => { + availableOperations: () => + Promise.resolve(["system-restart", "system-cleanup"]), + request: (operationId) => { requests += 1; - return Promise.resolve({ status: "accepted" }); + return Promise.resolve( + operationId === "system-restart" + ? ({ status: "accepted" } as const) + : ({ status: "completed" } as const) + ); }, }, }; @@ -375,6 +383,9 @@ describe("Dashboard worker runtime", () => { expect( coordinatorOptions.findAction?.("host.system.restart") ).toBeDefined(); + expect( + coordinatorOptions.findAction?.("host.system.cleanup") + ).toBeDefined(); expect( coordinatorOptions.findAction?.("host.system.update") ).toBeUndefined(); @@ -383,6 +394,11 @@ describe("Dashboard worker runtime", () => { ({ actionKey }) => actionKey ) ).toContain("host.system.restart"); + expect( + coordinatorOptions.actionDefinitions?.map( + ({ actionKey }) => actionKey + ) + ).toContain("host.system.cleanup"); return fixture.dependencies.createCoordinator(coordinatorOptions); }, }; diff --git a/greenfield/src/server/domains/jobs/workerRuntime.ts b/greenfield/src/server/domains/jobs/workerRuntime.ts index f87b23a1a..85d3b58fb 100644 --- a/greenfield/src/server/domains/jobs/workerRuntime.ts +++ b/greenfield/src/server/domains/jobs/workerRuntime.ts @@ -1,5 +1,6 @@ import { Cause, Effect, Exit, Fiber, ManagedRuntime } from "effect"; +import type { LinuxBootIdentity } from "../../../shared/linuxBootIdentity.ts"; import type { OpenClawGatewayLifecycleExecutionPort } from "../../../shared/openClawGatewayLifecycle.ts"; import type { OpenClawServiceActionsExecutionPort } from "../../../shared/openClawServiceActions.ts"; import type { @@ -27,6 +28,7 @@ import { } from "./actionExecutors.ts"; import { jobActionDefinitions, + hostSystemCleanupJobActionDefinition, hostSystemRestartJobActionDefinition, hostSystemUpdateJobActionDefinition, openClawGatewayRestartJobActionDefinition, @@ -48,6 +50,7 @@ import { } from "./sideEffects.ts"; export interface DashboardWorkerRuntimeOptions { + readonly bootIdentity: LinuxBootIdentity; readonly database: DatabaseRuntimeLayerOptions; readonly logMaintenance: LogMaintenanceExecutionPort; readonly hostOperations?: FixedHostOperationsExecutionPort; @@ -437,6 +440,9 @@ export function createDashboardWorkerRuntime( openClawSessionsCleanupJobActionDefinition, openClawInstallationUpdateJobActionDefinition, ]), + ...(availableHostOperationSet.has("system-cleanup") + ? [hostSystemCleanupJobActionDefinition] + : []), ...(availableHostOperationSet.has("system-restart") ? [hostSystemRestartJobActionDefinition] : []), @@ -470,6 +476,7 @@ export function createDashboardWorkerRuntime( }); coordinator = dependencies.createCoordinator({ actionDefinitions, + bootIdentity: options.bootIdentity, commitCacheAttempt: (input) => cacheRepository.commitAttempt(input), databaseReleaseId: options.releaseId, findAction, diff --git a/greenfield/src/server/domains/jobs/workerSystem.test.ts b/greenfield/src/server/domains/jobs/workerSystem.test.ts index b9b5af0ab..ca0d78d61 100644 --- a/greenfield/src/server/domains/jobs/workerSystem.test.ts +++ b/greenfield/src/server/domains/jobs/workerSystem.test.ts @@ -73,6 +73,7 @@ describe("durable job worker system", () => { const runId = Bun.randomUUIDv7(); const coordinator = createJobWorkerCoordinator({ actionDefinitions: jobActionDefinitions, + bootIdentity: "00000000-0000-0000-0000-000000000001", databaseReleaseId: "a".repeat(40), findAction: findJobWorkerAction, generateId: () => Bun.randomUUIDv7(), @@ -198,6 +199,7 @@ describe("durable job worker system", () => { const workerId = Bun.randomUUIDv7(); const coordinator = createJobWorkerCoordinator({ actionDefinitions: [definition], + bootIdentity: "00000000-0000-0000-0000-000000000001", commitCacheAttempt: (input) => cacheRepository.commitAttempt(input), databaseReleaseId: "a".repeat(40), findAction: (actionKey) => diff --git a/greenfield/src/server/domains/openClawSettings/restartQueue.test.ts b/greenfield/src/server/domains/openClawSettings/restartQueue.test.ts index 4af5bc95b..e93362697 100644 --- a/greenfield/src/server/domains/openClawSettings/restartQueue.test.ts +++ b/greenfield/src/server/domains/openClawSettings/restartQueue.test.ts @@ -31,6 +31,7 @@ function repositoryFixture() { eventBytes: 0, eventCount: 1, payloadEventCount: 0, + requiredWorkerReleaseId: input.run.requiredWorkerReleaseId ?? null, stateVersion: 1, }; return Promise.resolve({ kind: "inserted", run: stored }); @@ -137,6 +138,9 @@ describe("OpenClaw Gateway restart queue", () => { retrySafe: false, }); expect(fixture.enqueues[0]?.run.payloadJson).toBe("{}"); + expect(fixture.enqueues[0]?.run.resourceKeysJson).toBe( + '["host.mutation","openclaw.gateway"]' + ); expect(fixture.enqueues[0]?.auditEvents).toMatchObject([ { action: "openclaw.settings.restart.enqueue", diff --git a/greenfield/src/server/domains/serviceActions/procedures.test.ts b/greenfield/src/server/domains/serviceActions/procedures.test.ts index 0a8d354f8..5671e5721 100644 --- a/greenfield/src/server/domains/serviceActions/procedures.test.ts +++ b/greenfield/src/server/domains/serviceActions/procedures.test.ts @@ -30,7 +30,9 @@ const jobRunId = "018f6f50-6a9e-7b88-8000-000000000001"; const statusResult = Object.freeze({ actions: [ { availability: "available" as const, id: "openclaw-cleanup" as const }, + { availability: "available" as const, id: "openclaw-restart" as const }, { availability: "available" as const, id: "openclaw-update" as const }, + { availability: "available" as const, id: "system-cleanup" as const }, { availability: "available" as const, id: "system-restart" as const }, { availability: "unavailable" as const, id: "system-update" as const }, ], diff --git a/greenfield/src/server/domains/serviceActions/service.test.ts b/greenfield/src/server/domains/serviceActions/service.test.ts index 4c6716c2b..4ba07242f 100644 --- a/greenfield/src/server/domains/serviceActions/service.test.ts +++ b/greenfield/src/server/domains/serviceActions/service.test.ts @@ -90,7 +90,9 @@ function fixture( read: () => Promise.resolve([ { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-restart" }, { availability: "available", id: "openclaw-update" }, + { availability: "unavailable", id: "system-cleanup" }, { availability: "unavailable", id: "system-restart" }, { activeRun: queuedRun(jobRunId), @@ -182,7 +184,9 @@ describe("service actions service", () => { expect(result).toMatchObject({ actions: [ { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-restart" }, { availability: "available", id: "openclaw-update" }, + { availability: "unavailable", id: "system-cleanup" }, { availability: "unavailable", id: "system-restart" }, { activeRun: { id: jobRunId, state: "queued" }, @@ -219,7 +223,9 @@ describe("service actions service", () => { read: () => Promise.resolve([ { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-restart" }, { availability: "available", id: "openclaw-update" }, + { availability: "available", id: "system-cleanup" }, { availability: "available", id: "system-restart" }, { availability: "available", id: "system-update" }, ]), @@ -279,7 +285,9 @@ describe("service actions service", () => { read: () => Promise.resolve([ { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-restart" }, { availability: "available", id: "openclaw-update" }, + { availability: "available", id: "system-cleanup" }, { availability: "available", id: "system-restart" }, { availability: "unavailable", id: "system-update" }, ]), diff --git a/greenfield/src/server/domains/serviceActions/statusReader.test.ts b/greenfield/src/server/domains/serviceActions/statusReader.test.ts index cf485fb41..f8cd054ae 100644 --- a/greenfield/src/server/domains/serviceActions/statusReader.test.ts +++ b/greenfield/src/server/domains/serviceActions/statusReader.test.ts @@ -38,6 +38,7 @@ function run(id: string, actionKey: string, state: "failed" | "queued"): JobRunR queuedAt, requestedById: actorId, requestedByKind: "user", + requiredWorkerReleaseId: null, resourceClass: "exclusive", resourceKeysJson: '["host.mutation"]', resultJson: null, @@ -84,6 +85,7 @@ describe("Service Action status reader", () => { availabilityInputs.push(input); return Object.freeze([ serviceActionJobActionKeys["openclaw-cleanup"], + serviceActionJobActionKeys["openclaw-restart"], serviceActionJobActionKeys["system-update"], ]); }, @@ -92,7 +94,9 @@ describe("Service Action status reader", () => { expect(await reader.read()).toEqual([ { availability: "available", id: "openclaw-cleanup" }, + { availability: "available", id: "openclaw-restart" }, { availability: "unavailable", id: "openclaw-update" }, + { availability: "unavailable", id: "system-cleanup" }, { availability: "unavailable", id: "system-restart" }, { activeRun: expect.objectContaining({ id: active.id, state: "queued" }), @@ -105,7 +109,9 @@ describe("Service Action status reader", () => { { actionKeys: [ "openclaw.sessions.cleanup", + "openclaw.gateway.restart", "openclaw.installation.update", + "host.system.cleanup", "host.system.restart", "host.system.update", ], diff --git a/greenfield/src/shared/databaseMigrationManifest.ts b/greenfield/src/shared/databaseMigrationManifest.ts index dbf125cef..ee214044f 100644 --- a/greenfield/src/shared/databaseMigrationManifest.ts +++ b/greenfield/src/shared/databaseMigrationManifest.ts @@ -13,8 +13,8 @@ export const migrationManifest = Object.freeze> = + Object.freeze({ + "system-cleanup": "mira-dashboard-host-system-cleanup.service", + "system-restart": "mira-dashboard-host-system-restart.service", + "system-update": "mira-dashboard-host-system-update.service", + }); diff --git a/greenfield/src/shared/linuxBootIdentity.ts b/greenfield/src/shared/linuxBootIdentity.ts new file mode 100644 index 000000000..220f13d41 --- /dev/null +++ b/greenfield/src/shared/linuxBootIdentity.ts @@ -0,0 +1,14 @@ +import * as v from "valibot"; + +const linuxBootIdentityMessage = "Linux boot identity is invalid"; + +/** Canonical lowercase UUID exposed by Linux for one running kernel boot. */ +export const linuxBootIdentitySchema = v.pipe( + v.string(linuxBootIdentityMessage), + v.regex( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u, + linuxBootIdentityMessage + ) +); + +export type LinuxBootIdentity = v.InferOutput; diff --git a/greenfield/src/test/parity/fixtures/legacy-endpoints.json b/greenfield/src/test/parity/fixtures/legacy-endpoints.json index 84ece83ab..1629d6fd6 100644 --- a/greenfield/src/test/parity/fixtures/legacy-endpoints.json +++ b/greenfield/src/test/parity/fixtures/legacy-endpoints.json @@ -1593,7 +1593,7 @@ "id": "POST /api/exec/start", "method": "POST", "path": "/api/exec/start", - "purpose": "Partially replaced by the bounded PTY and fixed Service Actions. This row stays planned until legacy system_cleanup is decomposed without feature loss: Docker prune in Docker control, apt cleanup in host/package maintenance, and journald vacuum in log maintenance.", + "purpose": "Partially replaced by the bounded PTY and fixed Service Actions without a generic command API. The system-cleanup foundation preserves package autoremove/cache cleanup, journald rotation with 14-day and 1 GiB retention bounds, and Docker pruning only for unused content older than 168 hours; it never deletes volumes. This row remains planned until the distinct-worker production topology and separately approved root provisioning make that host action executable.", "section": "Exec And Terminal", "target": { "delivery": "planned", @@ -1845,7 +1845,7 @@ "target": { "delivery": "implemented", "kind": "procedure", - "names": ["openClawSettings.restartGateway"], + "names": ["openClawSettings.restartGateway", "serviceActions.request"], "phase": "phase-5" } }, diff --git a/greenfield/src/test/parity/parityInventory.test.ts b/greenfield/src/test/parity/parityInventory.test.ts index c0750e2b9..e82515cd0 100644 --- a/greenfield/src/test/parity/parityInventory.test.ts +++ b/greenfield/src/test/parity/parityInventory.test.ts @@ -316,11 +316,11 @@ describe("reviewed pre-cutover parity inventory", () => { kind: "reviewed-removal", reason: expect.stringContaining("synchronous generic command endpoint"), }); - expect(endpoints[3]?.purpose).toContain("Docker prune in Docker control"); - expect(endpoints[3]?.purpose).toContain( - "apt cleanup in host/package maintenance" - ); - expect(endpoints[3]?.purpose).toContain("journald vacuum in log maintenance"); + expect(endpoints[3]?.purpose).toContain("package autoremove/cache cleanup"); + expect(endpoints[3]?.purpose).toContain("14-day and 1 GiB retention"); + expect(endpoints[3]?.purpose).toContain("older than 168 hours"); + expect(endpoints[3]?.purpose).toContain("never deletes volumes"); + expect(endpoints[3]?.purpose).toContain("distinct-worker production topology"); }); test("records the bounded OpenClaw settings and operations slice", async () => { @@ -358,7 +358,11 @@ describe("reviewed pre-cutover parity inventory", () => { "implemented", ["openClawSettings.createConfigurationBackup"], ], - ["POST /api/restart", "implemented", ["openClawSettings.restartGateway"]], + [ + "POST /api/restart", + "implemented", + ["openClawSettings.restartGateway", "serviceActions.request"], + ], [ "POST /api/skills/:name", "implemented", diff --git a/greenfield/src/worker/system/fixedHostOperationsBroker.test.ts b/greenfield/src/worker/system/fixedHostOperationsBroker.test.ts new file mode 100644 index 000000000..435237019 --- /dev/null +++ b/greenfield/src/worker/system/fixedHostOperationsBroker.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, test } from "bun:test"; + +import { + fixedHostOperationUnits, + type HostOperationId, +} from "../../shared/hostOperations.ts"; +import type { FixedHostOperationProcess } from "./fixedHostOperationsBroker.ts"; +import { createFixedHostOperationsBroker } from "./fixedHostOperationsBroker.ts"; + +const encoder = new TextEncoder(); + +describe("fixed host operations broker", () => { + test("projects exact loaded units and dispatches fixed argv without a shell", async () => { + const calls: Array<{ + readonly arguments_: readonly string[]; + readonly executable: string; + }> = []; + const process: FixedHostOperationProcess = (executable, arguments_) => { + calls.push({ arguments_, executable }); + return Promise.resolve({ + exitCode: 0, + stderr: new Uint8Array(), + stdout: + arguments_[0] === "show" + ? encoder.encode("loaded\n") + : new Uint8Array(), + }); + }; + const broker = createFixedHostOperationsBroker({ process }); + + expect(await broker.availableOperations()).toEqual([ + "system-cleanup", + "system-restart", + "system-update", + ]); + expect(await broker.request("system-restart")).toEqual({ + status: "accepted", + }); + expect(await broker.request("system-update")).toEqual({ + status: "completed", + }); + expect(await broker.request("system-cleanup")).toEqual({ + status: "completed", + }); + expect(calls).toEqual([ + { + arguments_: [ + "show", + "--property=LoadState", + "--value", + fixedHostOperationUnits["system-cleanup"], + ], + executable: "/usr/bin/systemctl", + }, + { + arguments_: [ + "show", + "--property=LoadState", + "--value", + fixedHostOperationUnits["system-restart"], + ], + executable: "/usr/bin/systemctl", + }, + { + arguments_: [ + "show", + "--property=LoadState", + "--value", + fixedHostOperationUnits["system-update"], + ], + executable: "/usr/bin/systemctl", + }, + { + arguments_: [ + "start", + "--no-block", + "mira-dashboard-host-system-restart.service", + ], + executable: "/usr/bin/systemctl", + }, + { + arguments_: [ + "start", + "--wait", + "mira-dashboard-host-system-update.service", + ], + executable: "/usr/bin/systemctl", + }, + { + arguments_: [ + "start", + "--wait", + "mira-dashboard-host-system-cleanup.service", + ], + executable: "/usr/bin/systemctl", + }, + ]); + }); + + test("omits unavailable units and never exposes host diagnostics", async () => { + const broker = createFixedHostOperationsBroker({ + process: (_executable, arguments_) => + Promise.resolve({ + exitCode: + arguments_[0] === "start" || + arguments_.at(-1)?.includes("restart") + ? 1 + : 0, + stderr: encoder.encode("private /etc/apt failure"), + stdout: encoder.encode("loaded\n"), + }), + }); + + expect(await broker.availableOperations()).toEqual([ + "system-cleanup", + "system-update", + ]); + try { + await broker.request("system-update"); + throw new Error("Expected broker failure"); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe("Fixed host operations broker failed"); + expect(JSON.stringify(error)).not.toContain("/etc/apt"); + } + }); + + test("rejects unsafe operation IDs, executables, deadlines, and oversized output", () => { + let invocations = 0; + const process: FixedHostOperationProcess = () => { + invocations += 1; + return Promise.resolve({ + exitCode: 0, + stderr: new Uint8Array(), + stdout: new Uint8Array(64 * 1024 + 1), + }); + }; + const broker = createFixedHostOperationsBroker({ process }); + expect(broker.request("../../evil.service" as HostOperationId)).rejects.toThrow( + "Fixed host operations broker failed" + ); + expect(invocations).toBe(0); + expect(broker.request("system-update")).rejects.toThrow( + "Fixed host operations broker failed" + ); + expect(() => + createFixedHostOperationsBroker({ systemctlExecutable: "systemctl" }) + ).toThrow("Fixed host operations broker failed"); + expect(() => + createFixedHostOperationsBroker({ availabilityDeadlineMs: 0 }) + ).toThrow("Fixed host operations broker failed"); + expect(() => + createFixedHostOperationsBroker({ cleanupDeadlineMs: 35 * 60_000 + 1 }) + ).toThrow("Fixed host operations broker failed"); + }); + + test("bounds execution with a deadline and preserves caller abort", () => { + const observedSignals: AbortSignal[] = []; + const process: FixedHostOperationProcess = (_executable, _arguments, signal) => + new Promise((_resolve, reject) => { + observedSignals.push(signal); + signal.addEventListener("abort", () => reject(new Error("Aborted")), { + once: true, + }); + }); + const broker = createFixedHostOperationsBroker({ + process, + restartDeadlineMs: 1, + }); + expect(broker.request("system-restart")).rejects.toThrow( + "Fixed host operations broker failed" + ); + expect(observedSignals[0]?.aborted).toBe(true); + + const caller = new AbortController(); + const waiting = createFixedHostOperationsBroker({ process }).request( + "system-update", + caller.signal + ); + caller.abort(); + expect(waiting).rejects.toThrow("Fixed host operations broker failed"); + expect(observedSignals[1]?.aborted).toBe(true); + }); +}); diff --git a/greenfield/src/worker/system/fixedHostOperationsBroker.ts b/greenfield/src/worker/system/fixedHostOperationsBroker.ts new file mode 100644 index 000000000..665877834 --- /dev/null +++ b/greenfield/src/worker/system/fixedHostOperationsBroker.ts @@ -0,0 +1,209 @@ +import { + fixedHostOperationUnits, + type HostOperationId, +} from "../../shared/hostOperations.ts"; + +const systemctlDefault = "/usr/bin/systemctl"; +const availabilityDeadlineDefaultMs = 5000; +const restartDeadlineDefaultMs = 60_000; +const cleanupDeadlineDefaultMs = 35 * 60_000; +const updateDeadlineDefaultMs = 2 * 60 * 60_000; +const processOutputMaximumBytes = 64 * 1024; + +export interface FixedHostOperationProcessResult { + readonly exitCode: number; + readonly stderr: Uint8Array; + readonly stdout: Uint8Array; +} + +export type FixedHostOperationProcess = ( + executable: string, + arguments_: readonly string[], + signal: AbortSignal +) => Promise; + +export type FixedHostOperationResult = + | Readonly<{ status: "accepted" }> + | Readonly<{ status: "completed" }>; + +export interface FixedHostOperationsBroker { + readonly availableOperations: ( + signal?: AbortSignal + ) => Promise; + readonly request: ( + operationId: HostOperationId, + signal?: AbortSignal + ) => Promise; +} + +function brokerFailure(): Error { + return new Error("Fixed host operations broker failed"); +} + +async function readBounded(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + total += next.value.byteLength; + if (total > processOutputMaximumBytes) throw brokerFailure(); + chunks.push(next.value); + } + } finally { + reader.releaseLock(); + } + const output = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return output; +} + +const defaultProcess: FixedHostOperationProcess = async ( + executable, + arguments_, + signal +) => { + const child = Bun.spawn([executable, ...arguments_], { + env: { LANG: "C", LC_ALL: "C", PATH: "/usr/bin:/bin" }, + signal, + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBounded(child.stdout), + readBounded(child.stderr), + ]); + return { exitCode, stderr, stdout }; + } catch { + child.kill(); + await child.exited.catch(() => {}); + throw brokerFailure(); + } +}; + +function operationSignal( + signal: AbortSignal | undefined, + deadlineMs: number +): AbortSignal { + const deadline = AbortSignal.timeout(deadlineMs); + return signal === undefined ? deadline : AbortSignal.any([deadline, signal]); +} + +function requireBoundedSuccess(result: FixedHostOperationProcessResult): void { + if ( + result.exitCode !== 0 || + result.stdout.byteLength > processOutputMaximumBytes || + result.stderr.byteLength > processOutputMaximumBytes + ) { + throw brokerFailure(); + } +} + +function validAbsoluteExecutable(executable: string): boolean { + return ( + executable.startsWith("/") && + !executable.includes("\0") && + executable.length <= 4096 + ); +} + +/** + * Creates the worker-only client for three exact root-owned host operations. + * No path, command, systemd unit, environment value, or process output crosses this port. + * @param options Replaceable process boundary and bounded deadlines for tests/composition. + * @returns Frozen broker over the reviewed fixed operation inventory. + */ +export function createFixedHostOperationsBroker( + options: { + readonly availabilityDeadlineMs?: number; + readonly cleanupDeadlineMs?: number; + readonly process?: FixedHostOperationProcess; + readonly restartDeadlineMs?: number; + readonly systemctlExecutable?: string; + readonly updateDeadlineMs?: number; + } = {} +): FixedHostOperationsBroker { + const execute = options.process ?? defaultProcess; + const executable = options.systemctlExecutable ?? systemctlDefault; + const availabilityDeadlineMs = + options.availabilityDeadlineMs ?? availabilityDeadlineDefaultMs; + const cleanupDeadlineMs = options.cleanupDeadlineMs ?? cleanupDeadlineDefaultMs; + const restartDeadlineMs = options.restartDeadlineMs ?? restartDeadlineDefaultMs; + const updateDeadlineMs = options.updateDeadlineMs ?? updateDeadlineDefaultMs; + if ( + !validAbsoluteExecutable(executable) || + !Number.isSafeInteger(availabilityDeadlineMs) || + availabilityDeadlineMs < 1 || + availabilityDeadlineMs > availabilityDeadlineDefaultMs || + !Number.isSafeInteger(cleanupDeadlineMs) || + cleanupDeadlineMs < 1 || + cleanupDeadlineMs > cleanupDeadlineDefaultMs || + !Number.isSafeInteger(restartDeadlineMs) || + restartDeadlineMs < 1 || + restartDeadlineMs > restartDeadlineDefaultMs || + !Number.isSafeInteger(updateDeadlineMs) || + updateDeadlineMs < 1 || + updateDeadlineMs > updateDeadlineDefaultMs + ) { + throw brokerFailure(); + } + + const broker: FixedHostOperationsBroker = { + async availableOperations(signal?: AbortSignal) { + const available: HostOperationId[] = []; + for (const operationId of Object.keys( + fixedHostOperationUnits + ) as HostOperationId[]) { + const unit = fixedHostOperationUnits[operationId]; + try { + const result = await execute( + executable, + ["show", "--property=LoadState", "--value", unit], + operationSignal(signal, availabilityDeadlineMs) + ); + requireBoundedSuccess(result); + if (new TextDecoder().decode(result.stdout).trim() === "loaded") { + available.push(operationId); + } + } catch { + // Availability is a fixed projection; raw systemd diagnostics stay local. + } + } + return Object.freeze(available); + }, + async request(operationId: HostOperationId, signal?: AbortSignal) { + const unit = fixedHostOperationUnits[operationId]; + if (unit === undefined) throw brokerFailure(); + const isRestart = operationId === "system-restart"; + let deadlineMs = updateDeadlineMs; + if (operationId === "system-cleanup") { + deadlineMs = cleanupDeadlineMs; + } else if (isRestart) { + deadlineMs = restartDeadlineMs; + } + try { + const result = await execute( + executable, + ["start", isRestart ? "--no-block" : "--wait", unit], + operationSignal(signal, deadlineMs) + ); + requireBoundedSuccess(result); + return Object.freeze({ + status: isRestart ? "accepted" : "completed", + }); + } catch { + throw brokerFailure(); + } + }, + }; + return Object.freeze(broker); +} diff --git a/greenfield/src/worker/system/linuxBootIdentity.ts b/greenfield/src/worker/system/linuxBootIdentity.ts new file mode 100644 index 000000000..61e4b1094 --- /dev/null +++ b/greenfield/src/worker/system/linuxBootIdentity.ts @@ -0,0 +1,27 @@ +import { readFile } from "node:fs/promises"; + +import * as v from "valibot"; + +import { + type LinuxBootIdentity, + linuxBootIdentitySchema, +} from "../../shared/linuxBootIdentity.ts"; + +const linuxBootIdentityPath = "/proc/sys/kernel/random/boot_id"; + +/** + * Reads the kernel-owned identity for the current Linux boot. + * @returns One validated canonical boot UUID. + */ +export async function readLinuxBootIdentity(): Promise { + if (process.platform !== "linux") { + throw new Error("Linux boot identity is unavailable"); + } + try { + const contents = await readFile(linuxBootIdentityPath, "utf8"); + if (contents.length > 64) throw new Error("invalid boot identity"); + return v.parse(linuxBootIdentitySchema, contents.trim()); + } catch { + throw new Error("Linux boot identity is unavailable"); + } +} diff --git a/greenfield/src/worker/system/systemHostOperationsProvisioning.test.ts b/greenfield/src/worker/system/systemHostOperationsProvisioning.test.ts new file mode 100644 index 000000000..bac934798 --- /dev/null +++ b/greenfield/src/worker/system/systemHostOperationsProvisioning.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import { fixedHostOperationUnits } from "../../shared/hostOperations.ts"; + +const artifacts = path.resolve( + import.meta.dir, + "../../../scripts/delivery/provisioning/host-operations" +); + +describe("fixed host operations provisioning", () => { + test("keeps the helper to three exact operations and fixed commands", async () => { + const helper = await readFile( + path.join(artifacts, "mira-dashboard-host-operation"), + "utf8" + ); + expect(helper).toContain("system-restart)"); + expect(helper).toContain("system-update)"); + expect(helper).toContain("system-cleanup)"); + expect(helper).toContain( + "/usr/bin/systemctl start --no-block mira-dashboard-deferred-reboot.timer" + ); + expect(helper).toContain("/usr/bin/apt-get update"); + expect(helper).toContain("/usr/bin/apt-get full-upgrade -y"); + expect(helper).toContain("/usr/bin/dpkg --configure -a"); + expect(helper.indexOf("/usr/bin/dpkg --configure -a")).toBeGreaterThan( + helper.indexOf("/usr/bin/apt-get full-upgrade -y") + ); + expect(helper).toContain("/usr/bin/apt-get autoremove -y"); + expect(helper).toContain("/usr/bin/apt-get autoclean -y"); + expect(helper).toContain("/usr/bin/journalctl --rotate"); + expect(helper).toContain( + "/usr/bin/journalctl --vacuum-time=14d --vacuum-size=1G" + ); + expect(helper).toContain( + "/usr/bin/docker system prune --all --force --filter until=168h" + ); + expect(helper.match(/\|\| cleanup_status=1/gu)).toHaveLength(5); + expect(helper.indexOf('[ "$cleanup_status" -eq 0 ]')).toBeGreaterThan( + helper.indexOf("/usr/bin/docker system prune") + ); + expect(helper).not.toContain("eval"); + expect(helper).not.toContain("sh -c"); + expect(helper).not.toContain("sudo"); + expect(helper).not.toContain("--volumes"); + }); + + test("grants the worker group access to exactly the three public units", async () => { + const policy = await readFile( + path.join(artifacts, "60-mira-dashboard-host-operations.rules"), + "utf8" + ); + expect(policy.match(/mira-dashboard-host-system-/gu)).toHaveLength(3); + for (const unit of Object.values(fixedHostOperationUnits)) { + expect(policy).toContain(`"${unit}"`); + } + expect(policy).toContain('action.lookup("verb") !== "start"'); + expect(policy).toContain('!subject.isInGroup("mira-dashboard-host-operations")'); + expect(policy).not.toContain("deferred-reboot"); + expect(policy).not.toContain('action.lookup("verb") !== "stop"'); + expect(policy).not.toContain('action.lookup("verb") !== "restart"'); + }); + + test("defers reboot and preserves worker NoNewPrivileges", async () => { + const [restart, update, cleanup, timer, reboot, worker] = await Promise.all([ + readFile( + path.join(artifacts, "mira-dashboard-host-system-restart.service"), + "utf8" + ), + readFile( + path.join(artifacts, "mira-dashboard-host-system-update.service"), + "utf8" + ), + readFile( + path.join(artifacts, "mira-dashboard-host-system-cleanup.service"), + "utf8" + ), + readFile( + path.join(artifacts, "mira-dashboard-deferred-reboot.timer"), + "utf8" + ), + readFile( + path.join(artifacts, "mira-dashboard-deferred-reboot.service"), + "utf8" + ), + readFile( + path.resolve( + import.meta.dir, + "../../../systemd/mira-dashboard-worker.service" + ), + "utf8" + ), + ]); + expect(restart).toContain("NoNewPrivileges=true"); + expect(update).not.toContain("NoNewPrivileges="); + expect(cleanup).not.toContain("NoNewPrivileges="); + expect(restart).toContain("ProtectKernelModules=true"); + expect(update).not.toContain("ProtectKernelModules=true"); + expect(cleanup).not.toContain("ProtectKernelModules=true"); + expect(restart).toContain("ProtectSystem=strict"); + expect(update).not.toContain("ProtectSystem="); + expect(cleanup).not.toContain("ProtectSystem="); + expect(update).not.toContain("ReadWritePaths="); + expect(cleanup).not.toContain("ReadWritePaths="); + expect(restart).toContain("PrivateDevices=true"); + expect(update).not.toContain("PrivateDevices=true"); + expect(cleanup).not.toContain("PrivateDevices=true"); + for (const packageOperation of [update, cleanup]) { + expect(packageOperation).toContain("UMask=0022"); + expect(packageOperation).toContain("HOME=/root"); + expect(packageOperation).toContain("USER=root"); + expect(packageOperation).toContain("LOGNAME=root"); + expect(packageOperation).toContain("SHELL=/bin/sh"); + expect(packageOperation).not.toMatch( + /^(?:LockPersonality|MemoryDenyWriteExecute|Private|Protect|ReadWritePaths|RemoveIPC|Restrict|SystemCallArchitectures)=/mu + ); + } + expect(worker).toContain("NoNewPrivileges=true"); + expect(timer).toContain("OnActiveSec=10s"); + expect(timer).not.toContain("WantedBy="); + expect(reboot).toContain("ExecStart=/usr/bin/systemctl reboot"); + expect(update).toContain("StandardOutput=null"); + expect(update).toContain("StandardError=null"); + expect(update).toContain("ExecStopPost=/usr/bin/env -i"); + expect(update).toContain("/usr/bin/dpkg --configure -a"); + expect(update).toContain("TimeoutStartSec=115min"); + expect(restart).toContain("ExecStart=/usr/bin/env -i"); + expect(cleanup).toContain("ExecStart=/usr/bin/env -i"); + expect(cleanup).toContain("TimeoutStartSec=30min"); + expect(cleanup).toContain("StandardOutput=null"); + expect(cleanup).toContain("StandardError=null"); + }); +}); From 4373658c2db3c255e8d50d1bdd8f59ae069b7dc2 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 18:53:30 +0200 Subject: [PATCH 12/13] fix(greenfield): stabilize development reloads --- greenfield/bunfig.toml | 6 +- .../docs/development/local-development.md | 18 +- .../development/developmentFrontend.test.ts | 4 + .../developmentMigrationIdentity.test.ts | 206 ++++++++++++ .../developmentMigrationIdentity.ts | 306 ++++++++++++++++++ .../development/developmentRuntime.test.ts | 171 +++++++++- .../scripts/development/developmentRuntime.ts | 217 +++++++++++-- .../scripts/development/developmentState.ts | 79 +++-- .../tanStackRouterHmrWorkaroundPlugin.test.ts | 110 +++++++ .../tanStackRouterHmrWorkaroundPlugin.ts | 80 +++++ 10 files changed, 1142 insertions(+), 55 deletions(-) create mode 100644 greenfield/scripts/development/developmentMigrationIdentity.test.ts create mode 100644 greenfield/scripts/development/developmentMigrationIdentity.ts create mode 100644 greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts create mode 100644 greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.ts diff --git a/greenfield/bunfig.toml b/greenfield/bunfig.toml index 206486312..6520cff04 100644 --- a/greenfield/bunfig.toml +++ b/greenfield/bunfig.toml @@ -16,5 +16,9 @@ coveragePathIgnorePatterns = [ ] [serve.static] -plugins = ["./scripts/reactCompilerPlugin.ts", "bun-plugin-tailwind"] +plugins = [ + "./scripts/reactCompilerPlugin.ts", + "./scripts/development/tanStackRouterHmrWorkaroundPlugin.ts", + "bun-plugin-tailwind", +] environment = "PUBLIC_*" diff --git a/greenfield/docs/development/local-development.md b/greenfield/docs/development/local-development.md index 2afa21ca8..180600842 100644 --- a/greenfield/docs/development/local-development.md +++ b/greenfield/docs/development/local-development.md @@ -49,6 +49,12 @@ The browser development path follows Bun's native contracts: - production builds use the same compiler-first plugin order, while Bun removes the HMR-only data holder. +The development-only `tanStackRouterHmrWorkaroundPlugin` narrowly transforms Router Core's ESM +module around an upstream Bun evaluation cycle. It defers `replaceRouteChunk` access until the HMR +callback runs and full-reloads only TanStack lazy-route updates instead of refreshing corrupted +router state. Ordinary React and CSS Fast Refresh stay enabled, production builds do not load the +workaround, and an unrecognized upstream implementation fails closed during development bundling. + The implementation is anchored to Bun's official documentation: - [Hot reloading](https://bun.com/docs/bundler/hot-reloading) @@ -114,10 +120,14 @@ running. A crashed coordinator's exact stale lease is recovered only after its p longer active; direct runtime children are parent-death guarded so they cannot continue using SQLite after that recovery. -The database marker stores a deterministic fingerprint of the reviewed migration graph. When the -mutable pre-cutover baseline changes, the next start removes only the development SQLite database -and its sidecars, then initializes the current schema. The TOTP keyring, workspace, OpenClaw config, -and other state remain intact. +The database marker stores a deterministic fingerprint of the reviewed migration graph. The outer +coordinator polls that exact graph independently of Bun's watched children. When the mutable +pre-cutover baseline changes, it stops the frontend, web, and worker children, removes only the +development SQLite database and its sidecars, updates the marker, and restarts all three children +against the current schema. This also recovers automatically when a watched child observes a +partially written migration graph and exits before the manifest edit lands. Ordinary React, CSS, +and server-source edits remain on their existing Fast Refresh or Bun `--watch` paths. The TOTP +keyring, workspace, OpenClaw config, and other state remain intact. Manual resets are deliberately separate: diff --git a/greenfield/scripts/development/developmentFrontend.test.ts b/greenfield/scripts/development/developmentFrontend.test.ts index 74af4532c..b952da7ea 100644 --- a/greenfield/scripts/development/developmentFrontend.test.ts +++ b/greenfield/scripts/development/developmentFrontend.test.ts @@ -138,6 +138,10 @@ test("serves remote Bun HMR, React Fast Refresh, and React Compiler output toget expect(javascript).toContain("/_bun/hmr"); expect(javascript).toContain("react-refresh/runtime"); expect(javascript).toContain("dashboardBrowserRoot"); + expect(javascript).toMatch( + /\.prototype\._replaceRouteChunk = \(\.\.\.([A-Za-z_$][\w$]*)\) => [A-Za-z_$][\w$]*\.replaceRouteChunk\(\.\.\.\1\);/u + ); + expect(javascript).toContain("globalThis.location.reload()"); expect(javascript).toContain("useMemoCache"); } finally { await stopChild(child); diff --git a/greenfield/scripts/development/developmentMigrationIdentity.test.ts b/greenfield/scripts/development/developmentMigrationIdentity.test.ts new file mode 100644 index 000000000..dc716d5a4 --- /dev/null +++ b/greenfield/scripts/development/developmentMigrationIdentity.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + observeDevelopmentMigrationIdentity, + readDevelopmentMigrationIdentity, +} from "./developmentMigrationIdentity.ts"; + +const migrationId = "20260812000000_development-identity-test"; + +function sha256(contents: string): string { + return new Bun.CryptoHasher("sha256").update(contents).digest("hex"); +} + +function manifestSource( + migrationSql: string, + snapshot: string, + revisionComment = "" +): string { + return `/** Test migration manifest. */ +export interface MigrationManifestEntry { + readonly id: string; + readonly migrationSha256: string; + readonly snapshotSha256: string; +} + +${revisionComment} +export const migrationManifest = Object.freeze([ + Object.freeze({ + id: "${migrationId}", + migrationSha256: "${sha256(migrationSql)}", + snapshotSha256: "${sha256(snapshot)}", + }), +]); +`; +} + +async function writeMigrationFixture( + repositoryRoot: string, + migrationSql: string, + snapshot: string, + revisionComment = "" +): Promise { + const migrationRoot = path.join(repositoryRoot, "migrations", migrationId); + const manifestRoot = path.join(repositoryRoot, "src", "shared"); + await Promise.all([ + mkdir(migrationRoot, { recursive: true }), + mkdir(manifestRoot, { recursive: true }), + ]); + // Create these in reverse inventory order to prove directory enumeration order is irrelevant. + await writeFile(path.join(migrationRoot, "snapshot.json"), snapshot, "utf8"); + await writeFile(path.join(migrationRoot, "migration.sql"), migrationSql, "utf8"); + await writeFile( + path.join(manifestRoot, "databaseMigrationManifest.ts"), + manifestSource(migrationSql, snapshot, revisionComment), + "utf8" + ); +} + +async function withMigrationFixture( + task: (repositoryRoot: string) => Promise +): Promise { + const repositoryRoot = await mkdtemp( + path.join(tmpdir(), "mira-development-migration-identity-") + ); + try { + await writeMigrationFixture( + repositoryRoot, + "CREATE TABLE identity_one (id INTEGER PRIMARY KEY);\n", + '{"version":1}\n' + ); + await task(repositoryRoot); + } finally { + await rm(repositoryRoot, { force: true, recursive: true }); + } +} + +describe("development migration identity", () => { + test.each([ + { + label: "migration SQL", + migrationSql: "CREATE TABLE identity_two (id INTEGER PRIMARY KEY);\n", + revisionComment: "", + snapshot: '{"version":1}\n', + }, + { + label: "migration snapshot", + migrationSql: "CREATE TABLE identity_one (id INTEGER PRIMARY KEY);\n", + revisionComment: "", + snapshot: '{"version":2}\n', + }, + ])("changes when $label changes", async (update) => { + await withMigrationFixture(async (repositoryRoot) => { + const initial = await readDevelopmentMigrationIdentity(repositoryRoot); + await writeMigrationFixture( + repositoryRoot, + update.migrationSql, + update.snapshot, + update.revisionComment + ); + + const changed = await readDevelopmentMigrationIdentity(repositoryRoot); + + expect(changed).toMatch(/^[a-f\d]{64}$/u); + expect(changed).not.toBe(initial); + }); + }); + + test("ignores comments and formatting outside the semantic manifest body", async () => { + await withMigrationFixture(async (repositoryRoot) => { + const initial = await readDevelopmentMigrationIdentity(repositoryRoot); + await writeMigrationFixture( + repositoryRoot, + "CREATE TABLE identity_one (id INTEGER PRIMARY KEY);\n", + '{"version":1}\n', + "// semantic-neutral manifest formatting revision" + ); + + expect(await readDevelopmentMigrationIdentity(repositoryRoot)).toBe(initial); + }); + }); + + test("reports a change that lands between state preparation and observation", async () => { + await withMigrationFixture(async (repositoryRoot) => { + const initial = await readDevelopmentMigrationIdentity(repositoryRoot); + await writeMigrationFixture( + repositoryRoot, + "CREATE TABLE identity_changed_before_observation (id INTEGER PRIMARY KEY);\n", + '{"version":1}\n' + ); + const expected = await readDevelopmentMigrationIdentity(repositoryRoot); + + const observation = observeDevelopmentMigrationIdentity( + repositoryRoot, + initial + ); + try { + expect(await observation.ready).toBe(expected); + expect(await observation.changed).toBe(expected); + } finally { + observation.close(); + } + }); + }); + + test("ignores ordinary source edits so normal HMR remains child-owned", async () => { + await withMigrationFixture(async (repositoryRoot) => { + const initial = await readDevelopmentMigrationIdentity(repositoryRoot); + const observation = observeDevelopmentMigrationIdentity( + repositoryRoot, + initial + ); + try { + expect(await observation.ready).toBeUndefined(); + await writeFile( + path.join(repositoryRoot, "src", "unrelated.ts"), + "export const unrelated = true;\n", + "utf8" + ); + const changed = await Promise.race([ + observation.changed.then(() => true), + Bun.sleep(500).then(() => false), + ]); + expect(changed).toBeFalse(); + } finally { + observation.close(); + } + }); + }); + + test("rejects hash drift, extra artifacts, and symlinked artifacts", async () => { + await withMigrationFixture(async (repositoryRoot) => { + const migrationRoot = path.join(repositoryRoot, "migrations", migrationId); + const migrationPath = path.join(migrationRoot, "migration.sql"); + await writeFile(migrationPath, "unreviewed migration\n", "utf8"); + expect(readDevelopmentMigrationIdentity(repositoryRoot)).rejects.toThrow( + "Development migration identity is invalid" + ); + + await writeMigrationFixture( + repositoryRoot, + "CREATE TABLE identity_one (id INTEGER PRIMARY KEY);\n", + '{"version":1}\n' + ); + await writeFile( + path.join(migrationRoot, "unexpected.txt"), + "extra\n", + "utf8" + ); + expect(readDevelopmentMigrationIdentity(repositoryRoot)).rejects.toThrow( + "Development migration identity is invalid" + ); + await rm(path.join(migrationRoot, "unexpected.txt")); + + const outside = path.join(repositoryRoot, "outside.sql"); + await writeFile(outside, "outside\n", "utf8"); + await rm(migrationPath); + await symlink(outside, migrationPath); + expect(readDevelopmentMigrationIdentity(repositoryRoot)).rejects.toThrow( + "Development migration identity is invalid" + ); + }); + }); +}); diff --git a/greenfield/scripts/development/developmentMigrationIdentity.ts b/greenfield/scripts/development/developmentMigrationIdentity.ts new file mode 100644 index 000000000..c21d543ce --- /dev/null +++ b/greenfield/scripts/development/developmentMigrationIdentity.ts @@ -0,0 +1,306 @@ +import { createHash } from "node:crypto"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; + +import { readBoundedRegularFile } from "../files/boundedFile.ts"; + +const migrationIdentityPollIntervalMs = 250; +const migrationManifestMaximumBytes = 256 * 1024; +const migrationArtifactMaximumBytes = 4 * 1024 * 1024; +const migrationArtifactNames = Object.freeze(["migration.sql", "snapshot.json"]); +const migrationIdentityFailureMessage = "Development migration identity is invalid"; + +/** One source identity observation for the outer development coordinator. */ +export interface DevelopmentMigrationIdentityObservation { + readonly changed: Promise; + readonly ready: Promise; + close(): void; +} + +export type ObserveDevelopmentMigrationIdentity = ( + repositoryRoot: string, + initialFingerprint: string +) => DevelopmentMigrationIdentityObservation; + +function migrationIdentityFailure(): Error { + return new Error(migrationIdentityFailureMessage); +} + +/** + * Identifies the redacted, retryable failure emitted for an incomplete source graph. + * @param error Unknown failure raised while refreshing development state. + * @returns Whether the failure represents an incomplete migration identity. + */ +export function isDevelopmentMigrationIdentityFailure(error: unknown): boolean { + return error instanceof Error && error.message === migrationIdentityFailureMessage; +} + +function validFingerprint(fingerprint: string): boolean { + return /^[a-f\d]{64}$/u.test(fingerprint); +} + +interface ParsedMigrationIdentity { + readonly id: string; + readonly migrationSha256: string; + readonly snapshotSha256: string; +} + +const migrationIdPattern = /^\d{14}_[a-z\d][a-z\d_-]*$/u; +const migrationManifestPrefix = + "export const migrationManifest = Object.freeze(["; +const migrationManifestSuffix = "]);"; +const migrationManifestEntryPattern = + /Object\.freeze\(\{\s*id:\s*"([^"]+)",\s*migrationSha256:\s*"([a-f\d]{64})",\s*snapshotSha256:\s*"([a-f\d]{64})",?\s*\}\),?/gu; + +function parseMigrationManifestSource(bytes: Buffer): readonly ParsedMigrationIdentity[] { + let source: string; + try { + source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw migrationIdentityFailure(); + } + const prefixOffset = source.indexOf(migrationManifestPrefix); + const suffixOffset = source.indexOf( + migrationManifestSuffix, + prefixOffset + migrationManifestPrefix.length + ); + if ( + prefixOffset === -1 || + suffixOffset === -1 || + source.includes(migrationManifestPrefix, prefixOffset + 1) + ) { + throw migrationIdentityFailure(); + } + const body = source.slice( + prefixOffset + migrationManifestPrefix.length, + suffixOffset + ); + const entries: ParsedMigrationIdentity[] = []; + let covered = ""; + let previousEnd = 0; + for (const match of body.matchAll(migrationManifestEntryPattern)) { + const [matched, id, migrationSha256, snapshotSha256] = match; + const matchIndex = match.index; + if ( + matchIndex === undefined || + id === undefined || + migrationSha256 === undefined || + snapshotSha256 === undefined || + !migrationIdPattern.test(id) + ) { + throw migrationIdentityFailure(); + } + covered += body.slice(previousEnd, matchIndex); + previousEnd = matchIndex + matched.length; + entries.push(Object.freeze({ id, migrationSha256, snapshotSha256 })); + } + covered += body.slice(previousEnd); + const ids = entries.map(({ id }) => id); + const sortedIds = ids.toSorted(); + if ( + entries.length === 0 || + entries.length > 64 || + /[^\s,]/u.test(covered) || + new Set(ids).size !== ids.length || + ids.some((id, index) => id !== sortedIds[index]) + ) { + throw migrationIdentityFailure(); + } + return Object.freeze(entries); +} + +function sha256(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} + +function compareCanonicalText(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +async function stableFile( + repositoryRoot: string, + relativePath: string, + maximumBytes: number +): Promise { + return readBoundedRegularFile( + path.join(repositoryRoot, relativePath), + repositoryRoot, + maximumBytes, + migrationIdentityFailureMessage + ); +} + +/** + * Derives a stable content identity over parsed reviewed manifest entries and the + * exact migration artifact tree. This stays fresh even though the coordinator itself is + * not Bun-watched and therefore cannot rely on a re-evaluated module import. + * @param repositoryRoot Canonical development source root. + * @returns Lowercase SHA-256 over ordered relative paths and bytes. + */ +export async function readDevelopmentMigrationIdentity( + repositoryRoot: string +): Promise { + if ( + !path.isAbsolute(repositoryRoot) || + path.resolve(repositoryRoot) !== repositoryRoot || + repositoryRoot.includes("\0") + ) { + throw migrationIdentityFailure(); + } + try { + const migrationRoot = path.join(repositoryRoot, "migrations"); + const manifestRelativePath = "src/shared/databaseMigrationManifest.ts"; + const manifestBytes = await stableFile( + repositoryRoot, + manifestRelativePath, + migrationManifestMaximumBytes + ); + const manifest = parseMigrationManifestSource(manifestBytes); + const entries = await readdir(migrationRoot, { withFileTypes: true }); + const migrationIds = entries.map(({ name }) => name).toSorted(); + if ( + entries.some( + (entry) => + !entry.isDirectory() || + entry.isSymbolicLink() || + !migrationIdPattern.test(entry.name) + ) || + migrationIds.length !== manifest.length || + migrationIds.some((id, index) => id !== manifest[index]?.id) + ) { + throw migrationIdentityFailure(); + } + await Promise.all( + migrationIds.map(async (migrationId) => { + const artifactDirectory = path.join(migrationRoot, migrationId); + const artifactEntries = await readdir(artifactDirectory, { + withFileTypes: true, + }); + const artifacts = artifactEntries.toSorted((left, right) => + compareCanonicalText(left.name, right.name) + ); + if ( + artifacts.length !== migrationArtifactNames.length || + artifacts.some( + (artifact, index) => + !artifact.isFile() || + artifact.isSymbolicLink() || + artifact.name !== migrationArtifactNames[index] + ) + ) { + throw migrationIdentityFailure(); + } + }) + ); + const relativePaths = migrationIds.flatMap((migrationId) => + migrationArtifactNames.map( + (artifactName) => `migrations/${migrationId}/${artifactName}` + ) + ); + const files = await Promise.all( + relativePaths.map((relativePath) => + stableFile(repositoryRoot, relativePath, migrationArtifactMaximumBytes) + ) + ); + for (const [index, migration] of manifest.entries()) { + const migrationSql = files[index * 2]; + const snapshot = files[1 + index * 2]; + if ( + migrationSql === undefined || + snapshot === undefined || + sha256(migrationSql) !== migration.migrationSha256 || + sha256(snapshot) !== migration.snapshotSha256 + ) { + throw migrationIdentityFailure(); + } + } + const fingerprint = createHash("sha256"); + fingerprint.update("mira-dashboard-development-migration-graph:v2\0"); + for (const migration of manifest) { + fingerprint.update(migration.id); + fingerprint.update("\0"); + fingerprint.update(migration.migrationSha256); + fingerprint.update("\0"); + fingerprint.update(migration.snapshotSha256); + fingerprint.update("\0"); + } + for (const [index, relativePath] of relativePaths.entries()) { + const bytes = files[index]; + if (bytes === undefined) throw migrationIdentityFailure(); + fingerprint.update(relativePath); + fingerprint.update("\0"); + fingerprint.update(bytes); + fingerprint.update("\0"); + } + return fingerprint.digest("hex"); + } catch { + throw migrationIdentityFailure(); + } +} + +/** + * Watches migration identity outside Bun's watched children. A content change resolves + * once, allowing the coordinator to stop every child before reconciling SQLite. A bounded + * exact-graph poll avoids broad repository watchers and closes atomic-editor replacement gaps. + * @param repositoryRoot Canonical development source root. + * @param initialFingerprint Stable identity returned by state preparation. + * @returns Closeable one-shot observation resolving with the new stable identity. + */ +export function observeDevelopmentMigrationIdentity( + repositoryRoot: string, + initialFingerprint: string +): DevelopmentMigrationIdentityObservation { + if (!validFingerprint(initialFingerprint)) throw migrationIdentityFailure(); + let closed = false; + let reading = false; + let resolveChanged!: (fingerprint: string) => void; + let resolveReady!: (fingerprint: string | undefined) => void; + let readySettled = false; + const changed = new Promise((resolve) => { + resolveChanged = resolve; + }); + const ready = new Promise((resolve) => { + resolveReady = resolve; + }); + + const inspect = async () => { + if (closed || reading) return; + reading = true; + try { + const nextFingerprint = + await readDevelopmentMigrationIdentity(repositoryRoot); + if (nextFingerprint !== initialFingerprint) { + closed = true; + clearInterval(poll); + if (!readySettled) { + readySettled = true; + resolveReady(nextFingerprint); + } + resolveChanged(nextFingerprint); + } else if (!readySettled) { + readySettled = true; + resolveReady(undefined); + } + } catch { + // Editors may replace several graph files in steps; later polls revalidate them. + } finally { + reading = false; + } + }; + + const poll = setInterval(() => void inspect(), migrationIdentityPollIntervalMs); + poll.unref(); + void inspect(); + + return Object.freeze({ + changed, + ready, + close() { + if (closed) return; + closed = true; + clearInterval(poll); + }, + }); +} diff --git a/greenfield/scripts/development/developmentRuntime.test.ts b/greenfield/scripts/development/developmentRuntime.test.ts index 5a6671b84..6dff4c293 100644 --- a/greenfield/scripts/development/developmentRuntime.test.ts +++ b/greenfield/scripts/development/developmentRuntime.test.ts @@ -1,8 +1,9 @@ import { describe, expect, jest, test } from "bun:test"; -import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { readDevelopmentMigrationIdentity } from "./developmentMigrationIdentity.ts"; import { guardedDevelopmentChildCommand } from "./developmentProcessGuard.ts"; import { type DevelopmentChildProcess, @@ -13,6 +14,8 @@ import { prepareDevelopmentRuntimeState } from "./developmentState.ts"; const repositoryRoot = path.resolve(import.meta.dir, "../.."); const sourceCommit = "0".repeat(40); +const runtimeMigrationId = "20260812000001_development-runtime-test"; +const runtimeSnapshot = '{"version":1}\n'; const developmentTestEnvironment = Object.freeze({ MOLTBOOK_API_KEY: "moltbook-development-test-key", }); @@ -59,7 +62,49 @@ function fakeChild(options: { readonly ignoreSigterm?: boolean } = {}): FakeChil }; } -async function runtimeConfig(temporaryRoot: string) { +function sha256(contents: string): string { + return new Bun.CryptoHasher("sha256").update(contents).digest("hex"); +} + +function runtimeMigrationManifest(migrationSql: string): string { + return `export interface MigrationManifestEntry { + readonly id: string; + readonly migrationSha256: string; + readonly snapshotSha256: string; +} + +export const migrationManifest = Object.freeze([ + Object.freeze({ + id: "${runtimeMigrationId}", + migrationSha256: "${sha256(migrationSql)}", + snapshotSha256: "${sha256(runtimeSnapshot)}", + }), +]); +`; +} + +async function writeRuntimeMigrationFixture( + repositoryRootPath: string, + migrationSql: string +): Promise { + const migrationRoot = path.join(repositoryRootPath, "migrations", runtimeMigrationId); + const manifestRoot = path.join(repositoryRootPath, "src", "shared"); + await Promise.all([ + mkdir(migrationRoot, { recursive: true }), + mkdir(manifestRoot, { recursive: true }), + ]); + await Promise.all([ + writeFile(path.join(migrationRoot, "migration.sql"), migrationSql, "utf8"), + writeFile(path.join(migrationRoot, "snapshot.json"), runtimeSnapshot, "utf8"), + writeFile( + path.join(manifestRoot, "databaseMigrationManifest.ts"), + runtimeMigrationManifest(migrationSql), + "utf8" + ), + ]); +} + +async function runtimeConfig(temporaryRoot: string, repositoryRootPath = repositoryRoot) { const tokenPath = path.join(temporaryRoot, "gateway-token"); await writeFile(tokenPath, "development-test-token\n", { mode: 0o600 }); return resolveDevelopmentStackConfig( @@ -67,7 +112,7 @@ async function runtimeConfig(temporaryRoot: string) { MIRA_DASHBOARD_DEV_GATEWAY_TOKEN_FILE: tokenPath, MIRA_DASHBOARD_PROJECT_ROOT: path.join(temporaryRoot, "project"), }, - repositoryRoot + repositoryRootPath ); } @@ -204,6 +249,126 @@ describe("development runtime lifecycle", () => { } }); + test("repairs a watched-child migration failure by resetting SQLite and restarting every child", async () => { + const temporaryRoot = await mkdtemp( + path.join(tmpdir(), "mira-dashboard-development-runtime-migration-") + ); + const sourceRoot = path.join(temporaryRoot, "source"); + const initialSql = "CREATE TABLE runtime_one (id INTEGER PRIMARY KEY);\n"; + const changedSql = "CREATE TABLE runtime_two (id INTEGER PRIMARY KEY);\n"; + await writeRuntimeMigrationFixture(sourceRoot, initialSql); + const config = await runtimeConfig(temporaryRoot, sourceRoot); + const initialFingerprint = await readDevelopmentMigrationIdentity(sourceRoot); + const firstWeb = fakeChild(); + const firstWorker = fakeChild(); + const firstFrontend = fakeChild(); + const secondWeb = fakeChild(); + const secondWorker = fakeChild(); + const secondFrontend = fakeChild(); + const children = [ + firstWeb, + firstWorker, + firstFrontend, + secondWeb, + secondWorker, + secondFrontend, + ]; + const commands: Array = []; + let resolveFirstStarted!: () => void; + let resolveSecondStarted!: () => void; + const firstStarted = new Promise((resolve) => { + resolveFirstStarted = resolve; + }); + const secondStarted = new Promise((resolve) => { + resolveSecondStarted = resolve; + }); + let spawnCalls = 0; + let running: Promise | undefined; + + try { + running = runDevelopmentStack(config, { + environment: developmentTestEnvironment, + resolveSourceCommit: () => Promise.resolve(sourceCommit), + spawn(command) { + commands.push(command); + const next = children[spawnCalls]; + spawnCalls += 1; + if (spawnCalls === 3) resolveFirstStarted(); + if (spawnCalls === 6) resolveSecondStarted(); + if (next === undefined) throw new Error("Unexpected child spawn"); + return next.child; + }, + }); + await Promise.race([ + firstStarted, + Bun.sleep(5000).then(() => { + throw new Error("Initial development children did not start"); + }), + ]); + await Promise.all([ + writeFile(config.databasePath, "stale database", { mode: 0o600 }), + writeFile(`${config.databasePath}-wal`, "stale sidecar", { mode: 0o600 }), + ]); + + const migrationPath = path.join( + sourceRoot, + "migrations", + runtimeMigrationId, + "migration.sql" + ); + await writeFile(migrationPath, changedSql, "utf8"); + firstWeb.exit(1); + await Bun.sleep(150); + await writeFile( + path.join(sourceRoot, "src", "shared", "databaseMigrationManifest.ts"), + runtimeMigrationManifest(changedSql), + "utf8" + ); + + await Promise.race([ + secondStarted, + Bun.sleep(5000).then(() => { + throw new Error("Development children did not recover"); + }), + ]); + expect(firstWeb.signals).toEqual([]); + expect(firstWorker.signals).toEqual(["SIGTERM"]); + expect(firstFrontend.signals).toEqual(["SIGTERM"]); + expect(commands).toHaveLength(6); + expect(commands[3]).toEqual([ + process.execPath, + "--watch", + "src/app/developmentWeb.ts", + sourceCommit, + ]); + expect(await Bun.file(config.databasePath).exists()).toBeFalse(); + expect(await Bun.file(`${config.databasePath}-wal`).exists()).toBeFalse(); + const expectedFingerprint = + await readDevelopmentMigrationIdentity(sourceRoot); + const marker = JSON.parse( + await readFile( + path.join( + config.stateRoot, + ".mira-dashboard-development-database.json" + ), + "utf8" + ) + ) as { migrationFingerprint?: unknown }; + expect(marker.migrationFingerprint).toBe(expectedFingerprint); + expect(marker.migrationFingerprint).not.toBe(initialFingerprint); + + secondWeb.exit(7); + expect(await running).toBe(7); + expect(secondWorker.signals).toEqual(["SIGTERM"]); + expect(secondFrontend.signals).toEqual(["SIGTERM"]); + await expectLeaseReleased(config); + } finally { + for (const child of children) child.exit(0); + await running?.catch(() => {}); + await rm(temporaryRoot, { force: true, recursive: true }); + } + }); + test("escalates from SIGTERM to SIGKILL after the shutdown deadline", async () => { const temporaryRoot = await mkdtemp( path.join(tmpdir(), "mira-dashboard-development-runtime-force-") diff --git a/greenfield/scripts/development/developmentRuntime.ts b/greenfield/scripts/development/developmentRuntime.ts index d5eb30343..1f30e1357 100644 --- a/greenfield/scripts/development/developmentRuntime.ts +++ b/greenfield/scripts/development/developmentRuntime.ts @@ -4,6 +4,12 @@ import { developmentFrontendEnvironment, developmentProcessEnvironments, } from "./developmentEnvironment.ts"; +import { + isDevelopmentMigrationIdentityFailure, + observeDevelopmentMigrationIdentity, + readDevelopmentMigrationIdentity, + type ObserveDevelopmentMigrationIdentity, +} from "./developmentMigrationIdentity.ts"; import { guardedDevelopmentChildCommand } from "./developmentProcessGuard.ts"; import type { DevelopmentStackConfig } from "./developmentStackConfig.ts"; import { @@ -22,6 +28,8 @@ export interface DevelopmentChildProcess { export interface DevelopmentRuntimeDependencies { readonly environment?: Readonly>; + readonly observeMigrationIdentity?: ObserveDevelopmentMigrationIdentity; + readonly readMigrationIdentity?: typeof readDevelopmentMigrationIdentity; readonly resolveSourceCommit: (repositoryRoot: string) => Promise; readonly spawn: ( command: readonly string[], @@ -33,14 +41,17 @@ export interface DevelopmentRuntimeDependencies { } interface DevelopmentStopController { - readonly children: DevelopmentChildProcess[]; + children: DevelopmentChildProcess[]; forceRequested: boolean; readonly requestStop: () => void; settling?: Promise; stopRequested: boolean; + readonly stopped: Promise; } const defaultDependencies: DevelopmentRuntimeDependencies = Object.freeze({ + observeMigrationIdentity: observeDevelopmentMigrationIdentity, + readMigrationIdentity: readDevelopmentMigrationIdentity, resolveSourceCommit: readSourceCommit, spawn( command: readonly string[], @@ -120,8 +131,16 @@ async function settleChildren( function childExit( child: DevelopmentChildProcess, processName: DevelopmentProcessName -): Promise> { - return child.exited.then((code) => Object.freeze({ code, processName })); +): Promise< + Readonly<{ + code: number; + processName: DevelopmentProcessName; + status: "child-exited"; + }> +> { + return child.exited.then((code) => + Object.freeze({ code, processName, status: "child-exited" }) + ); } async function startDevelopmentChildren( @@ -161,6 +180,10 @@ async function startDevelopmentChildren( } function createStopController(): DevelopmentStopController { + let resolveStopped!: () => void; + const stopped = new Promise((resolve) => { + resolveStopped = resolve; + }); const controller: DevelopmentStopController = { children: [], forceRequested: false, @@ -171,9 +194,11 @@ function createStopController(): DevelopmentStopController { return; } controller.stopRequested = true; + resolveStopped(); controller.settling = settleChildren(controller.children, false); }, stopRequested: false, + stopped, }; return controller; } @@ -187,8 +212,16 @@ async function coordinateDevelopmentChildren( DevelopmentChildProcess, DevelopmentChildProcess, ], - stopController: DevelopmentStopController -): Promise { + stopController: DevelopmentStopController, + migrationIdentityChanged?: Promise +): Promise< + | Readonly<{ + fingerprint: string; + status: "migration-identity-changed"; + }> + | Awaited> + | Readonly<{ status: "stopped" }> +> { const [frontend, web, worker] = children; process.stdout.write( `${JSON.stringify({ @@ -203,20 +236,25 @@ async function coordinateDevelopmentChildren( })}\n` ); + const migrationChange = migrationIdentityChanged?.then((fingerprint) => + Object.freeze({ + fingerprint, + status: "migration-identity-changed" as const, + }) + ); const exited = await Promise.race([ childExit(frontend, "frontend"), childExit(web, "web"), childExit(worker, "worker"), + ...(migrationChange === undefined ? [] : [migrationChange]), ]); stopController.settling ??= settleChildren(children, false); await stopController.settling; if (stopController.forceRequested) await settleChildren(children, true); - if (stopController.stopRequested) return 0; - const reportedExitCode = exited.code || 1; - process.stderr.write( - `Development ${exited.processName} process exited with code ${reportedExitCode}\n` - ); - return reportedExitCode; + if (stopController.stopRequested) { + return Object.freeze({ status: "stopped" }); + } + return exited; } async function runPreparedDevelopmentStack( @@ -224,8 +262,9 @@ async function runPreparedDevelopmentStack( state: PreparedDevelopmentState, sourceCommit: string, dependencies: DevelopmentRuntimeDependencies, - stopController: DevelopmentStopController -): Promise { + stopController: DevelopmentStopController, + migrationIdentityChanged?: Promise +): Promise>> { const children = await startDevelopmentChildren( config, state, @@ -233,16 +272,156 @@ async function runPreparedDevelopmentStack( dependencies, stopController ); - if (children === undefined) return 0; + if (children === undefined) { + return Object.freeze({ status: "stopped" }); + } return coordinateDevelopmentChildren( config, state, sourceCommit, children, - stopController + stopController, + migrationIdentityChanged ); } +async function waitForReadableMigrationIdentity( + repositoryRoot: string, + dependencies: DevelopmentRuntimeDependencies, + stopController: DevelopmentStopController +): Promise { + const readIdentity = + dependencies.readMigrationIdentity ?? readDevelopmentMigrationIdentity; + while (!stopController.stopRequested) { + try { + return await readIdentity(repositoryRoot); + } catch { + const outcome = await Promise.race([ + Bun.sleep(100).then(() => "retry" as const), + stopController.stopped.then(() => "stopped" as const), + ]); + if (outcome === "stopped") return; + } + } + return; +} + +async function refreshDevelopmentState( + config: DevelopmentStackConfig, + stateSession: PreparedDevelopmentStateSession, + dependencies: DevelopmentRuntimeDependencies, + stopController: DevelopmentStopController +): Promise { + while (!stopController.stopRequested) { + try { + const previousFingerprint = stateSession.migrationFingerprint; + const state = await stateSession.refresh(); + if ( + state.database === "reused" && + stateSession.migrationFingerprint !== previousFingerprint + ) { + throw new Error( + "Development migration identity changed without safe SQLite state" + ); + } + const currentFingerprint = await waitForReadableMigrationIdentity( + config.repositoryRoot, + dependencies, + stopController + ); + if (currentFingerprint === undefined) return; + if (currentFingerprint !== stateSession.migrationFingerprint) continue; + return state; + } catch (error) { + if (!isDevelopmentMigrationIdentityFailure(error)) throw error; + const currentFingerprint = await waitForReadableMigrationIdentity( + config.repositoryRoot, + dependencies, + stopController + ); + if (currentFingerprint === undefined) return; + } + } + return; +} + +async function runPreparedDevelopmentLifecycle( + config: DevelopmentStackConfig, + stateSession: PreparedDevelopmentStateSession, + sourceCommit: string, + dependencies: DevelopmentRuntimeDependencies, + stopController: DevelopmentStopController +): Promise { + let state = stateSession.state; + while (!stopController.stopRequested) { + const observation = ( + dependencies.observeMigrationIdentity ?? observeDevelopmentMigrationIdentity + )(config.repositoryRoot, stateSession.migrationFingerprint); + const initialIdentityChange = await Promise.race([ + observation.ready, + stopController.stopped.then(() => "stopped" as const), + ]); + if (initialIdentityChange === "stopped") { + observation.close(); + return 0; + } + if (initialIdentityChange !== undefined) { + observation.close(); + const refreshed = await refreshDevelopmentState( + config, + stateSession, + dependencies, + stopController + ); + if (refreshed === undefined) return 0; + state = refreshed; + continue; + } + + let outcome: Awaited>; + try { + outcome = await runPreparedDevelopmentStack( + config, + state, + sourceCommit, + dependencies, + stopController, + observation.changed + ); + } finally { + observation.close(); + } + if (outcome.status === "stopped") return 0; + + stopController.children = []; + stopController.settling = undefined; + if (outcome.status === "child-exited") { + const currentFingerprint = await waitForReadableMigrationIdentity( + config.repositoryRoot, + dependencies, + stopController + ); + if (currentFingerprint === undefined) return 0; + if (currentFingerprint === stateSession.migrationFingerprint) { + const reportedExitCode = outcome.code || 1; + process.stderr.write( + `Development ${outcome.processName} process exited with code ${reportedExitCode}\n` + ); + return reportedExitCode; + } + } + const refreshed = await refreshDevelopmentState( + config, + stateSession, + dependencies, + stopController + ); + if (refreshed === undefined) return 0; + state = refreshed; + } + return 0; +} + async function settleRemainingChildren( stopController: DevelopmentStopController ): Promise { @@ -275,9 +454,9 @@ export async function runDevelopmentStackWithPreparedState( config.repositoryRoot ); if (stopController.stopRequested) return 0; - return await runPreparedDevelopmentStack( + return await runPreparedDevelopmentLifecycle( config, - stateSession.state, + stateSession, sourceCommit, dependencies, stopController @@ -311,9 +490,9 @@ export async function runDevelopmentStack( ); if (stopController.stopRequested) return 0; stateSession = await prepareDevelopmentRuntimeState(config); - return await runPreparedDevelopmentStack( + return await runPreparedDevelopmentLifecycle( config, - stateSession.state, + stateSession, sourceCommit, dependencies, stopController diff --git a/greenfield/scripts/development/developmentState.ts b/greenfield/scripts/development/developmentState.ts index 5ed22c653..6cb3f2a75 100644 --- a/greenfield/scripts/development/developmentState.ts +++ b/greenfield/scripts/development/developmentState.ts @@ -1,4 +1,4 @@ -import { createHash, randomBytes } from "node:crypto"; +import { randomBytes } from "node:crypto"; import { constants } from "node:fs"; import { type FileHandle, @@ -14,9 +14,9 @@ import { } from "node:fs/promises"; import path from "node:path"; -import { migrationManifest } from "../../src/shared/databaseMigrationManifest.ts"; import { prepareProtectedProductionStatePath } from "../delivery/productionStateFilesystem.ts"; import { prepareDevelopmentFileRoots } from "./developmentFileRoots.ts"; +import { readDevelopmentMigrationIdentity } from "./developmentMigrationIdentity.ts"; import { readDevelopmentPrivateFile } from "./developmentPrivateFile.ts"; import type { DevelopmentStackConfig } from "./developmentStackConfig.ts"; import { @@ -56,6 +56,8 @@ export interface PreparedDevelopmentState { } export interface PreparedDevelopmentStateSession { + readonly migrationFingerprint: string; + refresh(): Promise; readonly state: PreparedDevelopmentState; release(): Promise; } @@ -177,26 +179,13 @@ async function claimState(config: DevelopmentStackConfig): Promise { ); } -function currentMigrationFingerprint(): string { - const fingerprint = createHash("sha256"); - fingerprint.update("mira-dashboard-development-migrations:v1\0"); - for (const migration of migrationManifest) { - fingerprint.update(migration.id); - fingerprint.update("\0"); - fingerprint.update(migration.migrationSha256); - fingerprint.update("\0"); - fingerprint.update(migration.snapshotSha256); - fingerprint.update("\0"); - } - return fingerprint.digest("hex"); -} - function expectedDatabaseMarker( - config: DevelopmentStackConfig + config: DevelopmentStackConfig, + migrationFingerprint: string ): DevelopmentDatabaseMarker { return { formatVersion: 1, - migrationFingerprint: currentMigrationFingerprint(), + migrationFingerprint, owner: config.stateOwner, }; } @@ -311,28 +300,34 @@ async function removeDevelopmentDatabaseFiles( return removed; } -async function writeDatabaseMarker(config: DevelopmentStackConfig): Promise { +async function writeDatabaseMarker( + config: DevelopmentStackConfig, + migrationFingerprint: string +): Promise { const markerPath = path.join(config.stateRoot, databaseMarkerFileName); if (await pathExists(markerPath)) await readDatabaseMarker(config); await replacePrivateFile( markerPath, - `${JSON.stringify(expectedDatabaseMarker(config), undefined, 2)}\n` + `${JSON.stringify(expectedDatabaseMarker(config, migrationFingerprint), undefined, 2)}\n` ); } async function reconcileDevelopmentDatabase( - config: DevelopmentStackConfig + config: DevelopmentStackConfig, + migrationFingerprint: string ): Promise { const markerPath = path.join(config.stateRoot, databaseMarkerFileName); const marker = (await pathExists(markerPath)) ? await readDatabaseMarker(config) : undefined; - const expected = expectedDatabaseMarker(config); + const expected = expectedDatabaseMarker(config, migrationFingerprint); const needsReset = marker === undefined || marker.migrationFingerprint !== expected.migrationFingerprint; const removed = needsReset ? await removeDevelopmentDatabaseFiles(config) : false; - if (marker === undefined || needsReset) await writeDatabaseMarker(config); + if (marker === undefined || needsReset) { + await writeDatabaseMarker(config, migrationFingerprint); + } return removed; } @@ -396,13 +391,17 @@ async function developmentKeyring(config: DevelopmentStackConfig): Promise { const prepared = await prepareProtectedProductionStatePath(config.stateRoot); if (prepared.stateDirectory !== expectedDatabaseDirectory(config)) { throw new Error("Development database path is invalid"); } - const didResetDatabase = await reconcileDevelopmentDatabase(config); + const didResetDatabase = await reconcileDevelopmentDatabase( + config, + migrationFingerprint + ); let database: PreparedDevelopmentState["database"]; if (didResetDatabase) { database = "schema-reset"; @@ -447,17 +446,38 @@ export async function prepareDevelopmentRuntimeState( ): Promise { await claimState(config); const lease = await acquireDevelopmentStateLease(config); + let migrationFingerprint: string; let state: PreparedDevelopmentState; try { - state = await prepareClaimedDevelopmentState(config); + migrationFingerprint = await readDevelopmentMigrationIdentity( + config.repositoryRoot + ); + state = await prepareClaimedDevelopmentState(config, migrationFingerprint); } catch (error) { return releaseAfterFailure(lease, error); } return Object.freeze({ + get migrationFingerprint(): string { + return migrationFingerprint; + }, + async refresh(): Promise { + const nextMigrationFingerprint = await readDevelopmentMigrationIdentity( + config.repositoryRoot + ); + const nextState = await prepareClaimedDevelopmentState( + config, + nextMigrationFingerprint + ); + migrationFingerprint = nextMigrationFingerprint; + state = nextState; + return nextState; + }, async release(): Promise { await lease.release(); }, - state, + get state(): PreparedDevelopmentState { + return state; + }, }); } @@ -494,7 +514,10 @@ export async function resetDevelopmentDatabase( throw new Error("Development database path is invalid"); } const removed = await removeDevelopmentDatabaseFiles(config); - await writeDatabaseMarker(config); + await writeDatabaseMarker( + config, + await readDevelopmentMigrationIdentity(config.repositoryRoot) + ); return removed; } finally { await lease.release(); diff --git a/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts new file mode 100644 index 000000000..ea751202c --- /dev/null +++ b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; + +import { applyTanStackRouterHmrWorkaround } from "./tanStackRouterHmrWorkaroundPlugin.ts"; + +const upstreamPrototypeSetup = ` +let replaceRouteChunk; +let refreshClientRouteCalls = 0; +class RouterCore { + latestLocationUpdates = 0; + updateLatestLocation() { + this.latestLocationUpdates += 1; + } +} +if (process.env.NODE_ENV !== "production") { +\tRouterCore.prototype._replaceRouteChunk = replaceRouteChunk; +\tRouterCore.prototype._refreshRoute = async function() { +\t\tthis._serverResult = void 0; +\t\tthis.updateLatestLocation(); +\t\tawait refreshClientRoute(this); +\t}; +} +replaceRouteChunk = (...args) => replaceRouteChunkCalls.push(args); +async function refreshClientRoute() { + refreshClientRouteCalls += 1; +} +const replaceRouteChunkCalls = []; +export { RouterCore, refreshClientRouteCalls, replaceRouteChunkCalls }; +`; + +interface RouterFixtureModule { + readonly RouterCore: new () => { + readonly _refreshRoute: () => Promise; + readonly _replaceRouteChunk: (...arguments_: unknown[]) => void; + readonly latestLocationUpdates: number; + }; + readonly refreshClientRouteCalls: number; + readonly replaceRouteChunkCalls: readonly (readonly unknown[])[]; +} + +async function importFixture(source: string): Promise { + const encoded = Buffer.from(source).toString("base64"); + return (await import( + `data:text/javascript;base64,${encoded}` + )) as RouterFixtureModule; +} + +describe("TanStack Router Bun HMR workaround", () => { + test("defers route-chunk replacement and full-reloads route HMR", async () => { + const originalLocation = Object.getOwnPropertyDescriptor(globalThis, "location"); + let reloadCalls = 0; + Object.defineProperty(globalThis, "location", { + configurable: true, + value: { + reload: () => { + reloadCalls += 1; + }, + }, + }); + + try { + const transformed = applyTanStackRouterHmrWorkaround(upstreamPrototypeSetup); + const fixture = await importFixture(transformed); + const router = new fixture.RouterCore(); + + router._replaceRouteChunk("route", "lazy"); + await router._refreshRoute(); + + expect(fixture.replaceRouteChunkCalls).toEqual([["route", "lazy"]]); + expect(fixture.refreshClientRouteCalls).toBe(0); + expect(router.latestLocationUpdates).toBe(0); + expect(reloadCalls).toBe(1); + } finally { + if (originalLocation === undefined) { + Reflect.deleteProperty(globalThis, "location"); + } else { + Object.defineProperty(globalThis, "location", originalLocation); + } + } + }); + + test("is idempotent across incremental development rebuilds", () => { + const transformed = applyTanStackRouterHmrWorkaround(upstreamPrototypeSetup); + + expect(applyTanStackRouterHmrWorkaround(transformed)).toBe(transformed); + }); + + test.each([ + { + implementationName: "_replaceRouteChunk", + source: upstreamPrototypeSetup.replace( + "\tRouterCore.prototype._replaceRouteChunk = replaceRouteChunk;", + "\tRouterCore.prototype._replaceRouteChunk = replacement;" + ), + }, + { + implementationName: "_refreshRoute", + source: upstreamPrototypeSetup.replace( + "\t\tawait refreshClientRoute(this);", + "\t\tawait refreshRoutes(this);" + ), + }, + ])( + "fails closed when upstream $implementationName drifts", + ({ source, implementationName }) => { + expect(() => applyTanStackRouterHmrWorkaround(source)).toThrow( + `Unsupported @tanstack/router-core ${implementationName} implementation` + ); + } + ); +}); diff --git a/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.ts b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.ts new file mode 100644 index 000000000..efc4d9652 --- /dev/null +++ b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.ts @@ -0,0 +1,80 @@ +const routerCoreModuleFilter = + /[/\\]node_modules[/\\]@tanstack[/\\]router-core[/\\]dist[/\\]esm[/\\]router\.js$/u; + +const eagerReplaceRouteChunkAssignment = + "\tRouterCore.prototype._replaceRouteChunk = replaceRouteChunk;"; +const deferredReplaceRouteChunkAssignment = + "\tRouterCore.prototype._replaceRouteChunk = (...args) => replaceRouteChunk(...args);"; +const upstreamRefreshRouteImplementation = [ + "\tRouterCore.prototype._refreshRoute = async function() {", + "\t\tthis._serverResult = void 0;", + "\t\tthis.updateLatestLocation();", + "\t\tawait refreshClientRoute(this);", + "\t};", +].join("\n"); +const reloadRefreshRouteImplementation = [ + "\tRouterCore.prototype._refreshRoute = async function() {", + "\t\tglobalThis.location.reload();", + "\t};", +].join("\n"); + +function replaceKnownImplementation( + source: string, + upstream: string, + workaround: string, + implementationName: string +): string { + const upstreamIndex = source.indexOf(upstream); + const workaroundIndex = source.indexOf(workaround); + if (upstreamIndex === -1) { + if (workaroundIndex !== -1) return source; + throw new Error( + `Unsupported @tanstack/router-core ${implementationName} implementation` + ); + } + if ( + workaroundIndex !== -1 || + source.includes(upstream, upstreamIndex + upstream.length) + ) { + throw new Error( + `Ambiguous @tanstack/router-core ${implementationName} implementation` + ); + } + return source.replace(upstream, workaround); +} + +/** + * Applies the narrow development workaround for Bun's TanStack Router HMR cycle. + * @param source Installed ESM router-core module source. + * @returns Source with deferred lazy-route replacement and safe route refresh. + */ +export function applyTanStackRouterHmrWorkaround(source: string): string { + const deferredSource = replaceKnownImplementation( + source, + eagerReplaceRouteChunkAssignment, + deferredReplaceRouteChunkAssignment, + "_replaceRouteChunk" + ); + return replaceKnownImplementation( + deferredSource, + upstreamRefreshRouteImplementation, + reloadRefreshRouteImplementation, + "_refreshRoute" + ); +} + +const tanStackRouterHmrWorkaroundPlugin: Bun.BunPlugin = { + name: "tanstack-router-bun-hmr-workaround", + target: "browser", + setup(build) { + build.onLoad( + { filter: routerCoreModuleFilter, namespace: "file" }, + async ({ path }) => ({ + contents: applyTanStackRouterHmrWorkaround(await Bun.file(path).text()), + loader: "js", + }) + ); + }, +}; + +export default tanStackRouterHmrWorkaroundPlugin; From 67fe27c9c35f6405bc279e88004dab0a1f17805a Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Wed, 12 Aug 2026 19:02:00 +0200 Subject: [PATCH 13/13] test(greenfield): keep HMR fixture out of coverage --- .../tanStackRouterHmrWorkaroundPlugin.test.ts | 57 ++++--------------- 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts index ea751202c..3fcc31fe5 100644 --- a/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts +++ b/greenfield/scripts/development/tanStackRouterHmrWorkaroundPlugin.test.ts @@ -27,55 +27,18 @@ const replaceRouteChunkCalls = []; export { RouterCore, refreshClientRouteCalls, replaceRouteChunkCalls }; `; -interface RouterFixtureModule { - readonly RouterCore: new () => { - readonly _refreshRoute: () => Promise; - readonly _replaceRouteChunk: (...arguments_: unknown[]) => void; - readonly latestLocationUpdates: number; - }; - readonly refreshClientRouteCalls: number; - readonly replaceRouteChunkCalls: readonly (readonly unknown[])[]; -} - -async function importFixture(source: string): Promise { - const encoded = Buffer.from(source).toString("base64"); - return (await import( - `data:text/javascript;base64,${encoded}` - )) as RouterFixtureModule; -} - describe("TanStack Router Bun HMR workaround", () => { - test("defers route-chunk replacement and full-reloads route HMR", async () => { - const originalLocation = Object.getOwnPropertyDescriptor(globalThis, "location"); - let reloadCalls = 0; - Object.defineProperty(globalThis, "location", { - configurable: true, - value: { - reload: () => { - reloadCalls += 1; - }, - }, - }); - - try { - const transformed = applyTanStackRouterHmrWorkaround(upstreamPrototypeSetup); - const fixture = await importFixture(transformed); - const router = new fixture.RouterCore(); - - router._replaceRouteChunk("route", "lazy"); - await router._refreshRoute(); + test("defers route-chunk replacement and full-reloads route HMR", () => { + const transformed = applyTanStackRouterHmrWorkaround(upstreamPrototypeSetup); - expect(fixture.replaceRouteChunkCalls).toEqual([["route", "lazy"]]); - expect(fixture.refreshClientRouteCalls).toBe(0); - expect(router.latestLocationUpdates).toBe(0); - expect(reloadCalls).toBe(1); - } finally { - if (originalLocation === undefined) { - Reflect.deleteProperty(globalThis, "location"); - } else { - Object.defineProperty(globalThis, "location", originalLocation); - } - } + expect(transformed).toContain( + "RouterCore.prototype._replaceRouteChunk = (...args) => replaceRouteChunk(...args);" + ); + expect(transformed).toContain("globalThis.location.reload();"); + expect(transformed).not.toContain( + "RouterCore.prototype._replaceRouteChunk = replaceRouteChunk;" + ); + expect(transformed).not.toContain("await refreshClientRoute(this);"); }); test("is idempotent across incremental development rebuilds", () => {