From 2e61cffe2b4182dcda2ca40a2a24e9320e3da845 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Thu, 30 Jul 2026 02:17:03 +0200 Subject: [PATCH 1/3] feat: harden dashboard deployment and worker operations --- .../databaseMigrations/0008WorkerControl.ts | 16 + backend/src/databaseMigrations/index.ts | 2 + backend/src/databaseSchemaCompatibility.ts | 2 +- backend/src/managedBunRuntime.ts | 32 +- backend/src/managedDashboardSystemd.ts | 288 +++++++++++++++++ backend/src/managedDashboardUnitPolicy.ts | 24 ++ backend/src/observability.ts | 2 +- backend/src/releaseDeployment.ts | 24 +- backend/src/releaseLifecycle.ts | 5 + backend/src/releaseManager.ts | 139 +++++--- backend/src/releaseManifest.ts | 62 ++-- backend/src/routes/jobExecutionRoutes.ts | 38 ++- backend/src/services/cacheRefresh.ts | 60 ++-- backend/src/services/cacheRefreshMetrics.ts | 305 ++++++++++++++++++ backend/src/services/jobExecutionQueue.ts | 8 + backend/src/services/jobWorker.ts | 23 ++ backend/src/services/jobWorkerControl.ts | 57 ++++ backend/src/services/pullRequests.ts | 3 +- backend/src/services/scheduledJobs.ts | 4 + backend/test/cacheRefreshMetrics.test.ts | 176 ++++++++++ backend/test/databaseLifecycle.test.ts | 55 ++-- backend/test/httpApiBehavior.test.ts | 28 +- backend/test/jobExecutionQueue.test.ts | 32 ++ backend/test/managedBunRuntime.test.ts | 38 ++- backend/test/managedDashboardSystemd.test.ts | 255 +++++++++++++++ backend/test/multiFactorAuth.test.ts | 7 + backend/test/releaseManager.test.ts | 125 ++++--- backend/test/releaseManifest.test.ts | 40 ++- backend/test/routeAndServiceBehavior.test.ts | 40 +++ backend/test/serviceBehavior.test.ts | 12 +- contracts/jobs.ts | 31 ++ contracts/moltbook.ts | 15 +- docs/api/endpoints.md | 35 +- docs/architecture/database.md | 1 + docs/operations/scheduler-cache-backups.md | 21 +- docs/setup/new-vps.md | 6 +- docs/setup/production-deploy.md | 36 ++- .../features/jobs/JobExecutionQueueCard.tsx | 52 ++- frontend/src/hooks/index.ts | 1 + frontend/src/hooks/useJobExecutions.ts | 21 +- frontend/src/test/componentBehavior.test.tsx | 37 ++- frontend/src/test/contracts.test.ts | 57 ++++ frontend/src/test/frontendBehavior.test.tsx | 4 +- frontend/src/test/pageBehavior.test.tsx | 2 +- scripts/runManagedDashboardRelease.sh | 3 +- 45 files changed, 1926 insertions(+), 298 deletions(-) create mode 100644 backend/src/databaseMigrations/0008WorkerControl.ts create mode 100644 backend/src/managedDashboardSystemd.ts create mode 100644 backend/src/managedDashboardUnitPolicy.ts create mode 100644 backend/src/services/cacheRefreshMetrics.ts create mode 100644 backend/src/services/jobWorkerControl.ts create mode 100644 backend/test/cacheRefreshMetrics.test.ts create mode 100644 backend/test/managedDashboardSystemd.test.ts diff --git a/backend/src/databaseMigrations/0008WorkerControl.ts b/backend/src/databaseMigrations/0008WorkerControl.ts new file mode 100644 index 000000000..91f61a706 --- /dev/null +++ b/backend/src/databaseMigrations/0008WorkerControl.ts @@ -0,0 +1,16 @@ +import type { DatabaseMigration } from "./types.ts"; + +export const workerControlMigration: DatabaseMigration = { + version: 8, + name: "worker-control", + sql: ` +CREATE TABLE job_worker_control ( + id INTEGER PRIMARY KEY CHECK (id = 1), + claims_paused INTEGER NOT NULL DEFAULT 0 CHECK (claims_paused IN (0, 1)), + updated_at TEXT NOT NULL +); + +INSERT INTO job_worker_control (id, claims_paused, updated_at) +VALUES (1, 0, strftime('%Y-%m-%dT%H:%M:%fZ', 'now')); +`, +}; diff --git a/backend/src/databaseMigrations/index.ts b/backend/src/databaseMigrations/index.ts index 3162beaa5..4332c5a45 100644 --- a/backend/src/databaseMigrations/index.ts +++ b/backend/src/databaseMigrations/index.ts @@ -5,6 +5,7 @@ import { maintenanceCoverageMigration } from "./0004MaintenanceCoverage.ts"; import { auditEventsMigration } from "./0005AuditEvents.ts"; import { multiFactorAuthenticationMigration } from "./0006MultiFactorAuthentication.ts"; import { deploymentRetentionIndexMigration } from "./0007DeploymentRetentionIndex.ts"; +import { workerControlMigration } from "./0008WorkerControl.ts"; import type { DatabaseMigration } from "./types.ts"; export const databaseMigrations: readonly DatabaseMigration[] = [ @@ -15,6 +16,7 @@ export const databaseMigrations: readonly DatabaseMigration[] = [ auditEventsMigration, multiFactorAuthenticationMigration, deploymentRetentionIndexMigration, + workerControlMigration, ]; export interface DatabaseMigrationIdentity { diff --git a/backend/src/databaseSchemaCompatibility.ts b/backend/src/databaseSchemaCompatibility.ts index 02fe96374..86e37c484 100644 --- a/backend/src/databaseSchemaCompatibility.ts +++ b/backend/src/databaseSchemaCompatibility.ts @@ -9,7 +9,7 @@ const CURRENT_DATABASE_SCHEMA_VERSION = databaseMigrations.at(-1)?.version ?? 0; * remain bounded by the live schema. */ export const DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY = Object.freeze({ - maximum: 7, + maximum: 8, minimum: 6, target: CURRENT_DATABASE_SCHEMA_VERSION, }); diff --git a/backend/src/managedBunRuntime.ts b/backend/src/managedBunRuntime.ts index a2cd98569..a44b0d164 100644 --- a/backend/src/managedBunRuntime.ts +++ b/backend/src/managedBunRuntime.ts @@ -14,7 +14,6 @@ const RETIRED_RUNTIME_DIRECTORY_PATTERN = /^\.retired-[\da-f]{8}-[\da-f]{4}-7[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; const RUNTIME_CHECK_TIMEOUT_MS = 5000; let currentRuntimeIdentity: string | undefined; -let currentRuntimeVersion: string | undefined; export interface ManagedBunRuntimeInstallOptions { runtimeRoot?: string; @@ -116,20 +115,11 @@ function isSingleLinkRegularExecutable(filePath: string): boolean { } } -/** - * Reads a Bun executable's strict version without inheriting application secrets. - * @param executablePath Absolute executable path. - * @param argument Bun identity flag. - * @returns Reported Bun version, or undefined when verification fails. - */ -function bunExecutableReportedIdentity( - executablePath: string, - argument: "--revision" | "--version" -): string | undefined { +function readBunRevisionIdentity(executablePath: string): string | undefined { if (!isCanonicalRegularExecutable(executablePath)) { return undefined; } - const result = spawnSync(executablePath, [argument], { + const result = spawnSync(executablePath, ["--revision"], { encoding: "utf8", env: { LANG: "C", @@ -151,7 +141,7 @@ function bunExecutableReportedIdentity( * @returns Revision-qualified Bun identity, or undefined when verification fails. */ export function bunExecutableRuntimeIdentity(executablePath: string): string | undefined { - return bunExecutableReportedIdentity(executablePath, "--revision"); + return readBunRevisionIdentity(executablePath); } /** @@ -169,7 +159,7 @@ export function currentBunRuntimeIdentity(): string { } /** - * Checks an executable against either a revision-qualified or legacy version identity. + * Checks an executable against its revision-qualified release identity. * @param executablePath Absolute executable path. * @param identity Release-manifest Bun identity. * @returns Whether the executable exactly satisfies the release identity. @@ -181,27 +171,19 @@ export function bunExecutableMatchesRuntime( if (!isBunRuntimeVersion(identity)) { return false; } - return ( - bunExecutableRuntimeIdentity(executablePath) === identity || - bunExecutableReportedIdentity(executablePath, "--version") === identity - ); + return bunExecutableRuntimeIdentity(executablePath) === identity; } /** * Checks whether an identity matches the Bun process running Dashboard. * @param identity Release-manifest Bun identity. - * @returns Whether the current process satisfies the exact or legacy identity. + * @returns Whether the current process satisfies the exact identity. */ export function isCurrentBunRuntime(identity: string): boolean { if (!isBunRuntimeVersion(identity)) { return false; } - const revision = currentBunRuntimeIdentity(); - currentRuntimeVersion ??= bunExecutableReportedIdentity( - process.execPath, - "--version" - ); - return identity === revision || identity === currentRuntimeVersion; + return identity === currentBunRuntimeIdentity(); } /** diff --git a/backend/src/managedDashboardSystemd.ts b/backend/src/managedDashboardSystemd.ts new file mode 100644 index 000000000..cf3cf655c --- /dev/null +++ b/backend/src/managedDashboardSystemd.ts @@ -0,0 +1,288 @@ +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { guardedPath, writeTextNoFollowAnchoredGuarded } from "./lib/guardedOps.ts"; +import { runProcess } from "./lib/processes.ts"; +import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; +import { + MANAGED_DASHBOARD_UNIT_ARTIFACTS, + MANAGED_DASHBOARD_UNIT_NAMES, + type ManagedDashboardUnitName, +} from "./managedDashboardUnitPolicy.ts"; +import type { ManagedDashboardRelease } from "./releaseManager.ts"; + +const MAX_UNIT_FILE_BYTES = 256 * 1024; +const SYSTEMCTL_EXECUTABLE = "/usr/bin/systemctl"; +const SYSTEMCTL_TIMEOUT_MS = 30_000; + +interface ManagedDashboardSystemdCommandResult { + stderr: string; + stdout: string; +} + +interface ManagedDashboardUnitFile { + content: string; + mode: number; +} + +export type ManagedDashboardSystemdCommandRunner = ( + command: string, + arguments_: readonly string[] +) => Promise; + +export interface ManagedDashboardSystemdOptions { + commandRunner?: ManagedDashboardSystemdCommandRunner; + unitRoot?: string; +} + +export interface PreparedManagedDashboardUnits { + changed: ManagedDashboardUnitName[]; + rollback: () => Promise; +} + +function releaseHasManagedUnitBundle(release: ManagedDashboardRelease): boolean { + const artifacts = new Set( + release.manifest.artifacts.map((artifact) => artifact.path) + ); + return MANAGED_DASHBOARD_UNIT_ARTIFACTS.every((artifact) => artifacts.has(artifact)); +} + +async function ensureRealDirectory(directoryPath: string, mode: number): Promise { + await fsp.mkdir(directoryPath, { mode, recursive: true }); + const stat = await fsp.lstat(directoryPath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new TypeError( + `Managed Dashboard systemd path must be a real directory: ${directoryPath}` + ); + } + if ((await fsp.realpath(directoryPath)) !== path.resolve(directoryPath)) { + throw new TypeError( + `Managed Dashboard systemd path must not traverse symlinks: ${directoryPath}` + ); + } +} + +async function readBoundedUnitFile(filePath: string): Promise { + const file = await fsp.open( + filePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK + ); + try { + const stat = await file.stat(); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size === 0 || + stat.size > MAX_UNIT_FILE_BYTES + ) { + throw new TypeError( + "Managed Dashboard systemd units must be bounded single-link files" + ); + } + return { + content: await file.readFile("utf8"), + mode: stat.mode & 0o777, + }; + } finally { + await file.close(); + } +} + +async function readOptionalUnitFile( + filePath: string +): Promise { + try { + return await readBoundedUnitFile(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +async function defaultCommandRunner( + command: string, + arguments_: readonly string[] +): Promise { + const result = await runProcess(command, arguments_, { + maxBuffer: 1024 * 1024, + timeoutMs: SYSTEMCTL_TIMEOUT_MS, + }); + if (result.code !== 0) { + throw new Error( + `${command} ${arguments_.join(" ")} failed with exit code ${ + result.code + }: ${result.stderr.trim() || result.stdout.trim()}` + ); + } + return { stderr: result.stderr, stdout: result.stdout }; +} + +async function reloadAndVerifyUnits( + unitRoot: string, + commandRunner: ManagedDashboardSystemdCommandRunner +): Promise { + await commandRunner(SYSTEMCTL_EXECUTABLE, ["--user", "daemon-reload"]); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + const result = await commandRunner(SYSTEMCTL_EXECUTABLE, [ + "--user", + "show", + unit, + "--property=DropInPaths", + "--property=FragmentPath", + "--property=LoadState", + "--no-pager", + ]); + const properties = new Map( + result.stdout + .split("\n") + .filter(Boolean) + .map((line) => { + const separator = line.indexOf("="); + return separator === -1 + ? [line, ""] + : [line.slice(0, separator), line.slice(separator + 1)]; + }) + ); + if ( + (properties.get("DropInPaths") ?? "") !== "" || + properties.get("LoadState") !== "loaded" || + properties.get("FragmentPath") !== path.join(unitRoot, unit) + ) { + throw new Error( + `${unit} did not load exclusively from its managed unit path` + ); + } + } +} + +async function removeInstalledUnit(filePath: string): Promise { + try { + const stat = await fsp.lstat(filePath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1) { + throw new TypeError( + "Managed Dashboard systemd rollback target changed identity" + ); + } + await fsp.unlink(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } +} + +async function restoreInstalledUnits( + changed: readonly ManagedDashboardUnitName[], + previous: ReadonlyMap, + unitRoot: string, + commandRunner: ManagedDashboardSystemdCommandRunner +): Promise { + for (const unit of changed) { + const installed = previous.get(unit); + await (installed === undefined + ? removeInstalledUnit(path.join(unitRoot, unit)) + : writeTextNoFollowAnchoredGuarded( + guardedPath(unitRoot), + unit, + installed.content, + { mode: installed.mode } + )); + } + await commandRunner(SYSTEMCTL_EXECUTABLE, ["--user", "daemon-reload"]); +} + +/** + * Installs and verifies the target release's managed units, returning a + * compensating operation for a release transition that does not commit. + * @param release Verified target release. + * @param options Unit root and command dependency overrides. + * @returns Prepared unit state and its compensating rollback. + */ +export async function prepareManagedDashboardUnits( + release: ManagedDashboardRelease, + options: ManagedDashboardSystemdOptions = {} +): Promise { + if (!releaseHasManagedUnitBundle(release)) { + throw new Error( + `Release ${release.commitSha} does not contain managed systemd units` + ); + } + const unitRoot = resolveAbsoluteNonRootPath( + options.unitRoot ?? path.join(os.homedir(), ".config", "systemd", "user"), + "Managed Dashboard user unit root" + ); + await ensureRealDirectory(unitRoot, 0o755); + const desired = new Map( + await Promise.all( + MANAGED_DASHBOARD_UNIT_NAMES.map(async (unit) => { + const releaseUnit = await readBoundedUnitFile( + path.join(release.path, "systemd", unit) + ); + return [unit, releaseUnit.content] as const; + }) + ) + ); + const previous = new Map< + ManagedDashboardUnitName, + ManagedDashboardUnitFile | undefined + >(); + const changed: ManagedDashboardUnitName[] = []; + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + const installed = await readOptionalUnitFile(path.join(unitRoot, unit)); + previous.set(unit, installed); + if ( + !installed || + installed.content !== desired.get(unit) || + installed.mode !== 0o644 + ) { + changed.push(unit); + } + } + if (changed.length === 0) { + await reloadAndVerifyUnits( + unitRoot, + options.commandRunner ?? defaultCommandRunner + ); + return { + changed, + rollback: () => Promise.resolve(), + }; + } + + const commandRunner = options.commandRunner ?? defaultCommandRunner; + try { + for (const unit of changed) { + await writeTextNoFollowAnchoredGuarded( + guardedPath(unitRoot), + unit, + desired.get(unit) as string, + { mode: 0o644 } + ); + } + await reloadAndVerifyUnits(unitRoot, commandRunner); + } catch (reconcileError) { + let rollbackError: unknown; + try { + await restoreInstalledUnits(changed, previous, unitRoot, commandRunner); + } catch (error) { + rollbackError = error; + } + if (rollbackError !== undefined) { + const reconcileFailure = new AggregateError( + [reconcileError, rollbackError], + "Managed Dashboard systemd reconciliation and rollback failed", + { cause: reconcileError } + ); + throw reconcileFailure; + } + throw reconcileError; + } + return { + changed, + rollback: () => restoreInstalledUnits(changed, previous, unitRoot, commandRunner), + }; +} diff --git a/backend/src/managedDashboardUnitPolicy.ts b/backend/src/managedDashboardUnitPolicy.ts new file mode 100644 index 000000000..d1ee32702 --- /dev/null +++ b/backend/src/managedDashboardUnitPolicy.ts @@ -0,0 +1,24 @@ +export const MANAGED_DASHBOARD_UNITS = { + "mira-dashboard-worker.service": "dist/workerStart.js", + "mira-dashboard.service": "dist/serverStart.js", +} as const; + +export type ManagedDashboardUnitName = keyof typeof MANAGED_DASHBOARD_UNITS; + +export const MANAGED_DASHBOARD_UNIT_NAMES = Object.keys( + MANAGED_DASHBOARD_UNITS +) as ManagedDashboardUnitName[]; + +export const MANAGED_DASHBOARD_UNIT_ARTIFACTS = MANAGED_DASHBOARD_UNIT_NAMES.map( + (unit) => `systemd/${unit}` as const +); + +export const MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT = [ + "NODE_ENV", + "MIRA_DASHBOARD_PROJECT_ROOT", +] as const; + +export const MANAGED_DASHBOARD_UNIT_POLICY_ENVIRONMENT = { + "mira-dashboard-worker.service": ["NODE_ENV=production"], + "mira-dashboard.service": ["NODE_ENV=production"], +} as const satisfies Record; diff --git a/backend/src/observability.ts b/backend/src/observability.ts index 95fe1d214..9f5746caa 100644 --- a/backend/src/observability.ts +++ b/backend/src/observability.ts @@ -9,7 +9,7 @@ import gateway from "./gateway.ts"; import { getDatabaseOperationMetrics } from "./lib/databaseMetrics.ts"; import { getChildProcessMetrics } from "./lib/processes.ts"; import { getRuntimeMetrics } from "./lib/runtimeMetrics.ts"; -import { getCacheRefreshMetrics } from "./services/cacheRefresh.ts"; +import { getCacheRefreshMetrics } from "./services/cacheRefreshMetrics.ts"; import { getScheduledJobSchedulerMetrics } from "./services/scheduledJobs.ts"; function fileBytes(path: string): number { diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index de70c4012..8ab11fc53 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -13,6 +13,12 @@ import { resolveDashboardReleaseBuildBunExecutable, resolveManagedBunRuntimeRoot, } from "./managedBunRuntime.ts"; +import { + MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT, + MANAGED_DASHBOARD_UNIT_POLICY_ENVIRONMENT, + MANAGED_DASHBOARD_UNITS, + type ManagedDashboardUnitName, +} from "./managedDashboardUnitPolicy.ts"; import { type DashboardReleaseRetentionResult, loadManagedRelease, @@ -24,18 +30,10 @@ import { const RELEASE_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; const MAX_PROCESS_OUTPUT_BYTES = 20 * 1024 * 1024; -export const MANAGED_DASHBOARD_UNITS = { - "mira-dashboard-worker.service": "dist/workerStart.js", - "mira-dashboard.service": "dist/serverStart.js", -} as const; -export const MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT = [ - "NODE_ENV", - "MIRA_DASHBOARD_PROJECT_ROOT", -] as const; -const MANAGED_DASHBOARD_UNIT_POLICY_ENVIRONMENT = { - "mira-dashboard-worker.service": ["NODE_ENV=production"], - "mira-dashboard.service": ["NODE_ENV=production"], -} as const satisfies Record; +export { + MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT, + MANAGED_DASHBOARD_UNITS, +} from "./managedDashboardUnitPolicy.ts"; export interface DashboardReleaseCommandResult { stderr: string; @@ -80,8 +78,6 @@ export interface ManagedDashboardUnitContract { worktreeRoot: string; } -type ManagedDashboardUnitName = keyof typeof MANAGED_DASHBOARD_UNITS; - const MANAGED_RELEASE_BUILD_ENVIRONMENT = [ "HOME", "HTTPS_PROXY", diff --git a/backend/src/releaseLifecycle.ts b/backend/src/releaseLifecycle.ts index cf669c71d..98e3231c9 100644 --- a/backend/src/releaseLifecycle.ts +++ b/backend/src/releaseLifecycle.ts @@ -9,6 +9,7 @@ import { validateDatabaseMigrationHistory } from "./databaseMigrationRunner.ts"; import { writeCliError, writeCliOutput } from "./lib/cliOutput.ts"; import { resolveDashboardProjectPaths } from "./lib/dashboardPaths.ts"; import { resolveManagedBunRuntimeRoot } from "./managedBunRuntime.ts"; +import { prepareManagedDashboardUnits } from "./managedDashboardSystemd.ts"; import type { DashboardReleaseManagerOptions, DashboardReleaseState, @@ -153,6 +154,10 @@ export async function runReleaseLifecycleCommand( let state: DashboardReleaseState; const transitionOptions: DashboardReleaseManagerOptions = { ...options, + ...(path.resolve(releasesRoot) === + path.resolve(resolveDashboardProjectPaths().productionReleasesRoot) && { + prepareReleaseTransition: prepareManagedDashboardUnits, + }), transitionLockWaitMs: options.transitionLockWaitMs ?? RELEASE_TRANSITION_LOCK_WAIT_MS, }; diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index 7d59bbeca..706c25e7e 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -82,7 +82,14 @@ export interface DashboardReleaseRuntimeAvailabilityOptions { hasRuntime?: (version: string) => boolean; } +export interface DashboardReleaseTransitionPreparation { + rollback: () => Promise; +} + export interface DashboardReleaseManagerOptions extends DashboardReleaseRuntimeAvailabilityOptions { + prepareReleaseTransition?: ( + target: ManagedDashboardRelease + ) => Promise; readLiveSchemaState?: ( maximumCompatibleVersion: number ) => DashboardLiveSchemaState | Promise; @@ -1130,6 +1137,32 @@ async function withReleaseTransitionLock( return result as T; } +async function withPreparedReleaseTransition( + target: ManagedDashboardRelease, + options: DashboardReleaseManagerOptions, + transition: () => Promise +): Promise { + const preparation = await options.prepareReleaseTransition?.(target); + try { + return await transition(); + } catch (transitionError) { + if (!preparation) { + throw transitionError; + } + try { + await preparation.rollback(); + } catch (rollbackError) { + const transitionFailure = new AggregateError( + [transitionError, rollbackError], + "Managed release transition and preparation rollback failed", + { cause: transitionError } + ); + throw transitionFailure; + } + throw transitionError; + } +} + /** * Copies a verified build into the immutable release store while excluding * activation, rollback, pruning, and another publisher from its staging path. @@ -1342,28 +1375,30 @@ export async function activateDashboardRelease( liveSchemaState, "Activation" ); - if (state.current?.commitSha === candidate.commitSha) { - return state; - } - - const before = releaseLinkStateFromDashboardState(state); - const journal: ReleaseTransitionJournal = { - after: { - current: candidate.commitSha, - previous: before.current, - }, - before, - formatVersion: RELEASE_TRANSITION_FORMAT_VERSION, - operation: "activate", - }; - return await executeReleaseTransition(layout, journal, async () => { - const expectedReleases = new Map([ - [candidate.commitSha, candidate], - ]); - if (state.current) { - expectedReleases.set(state.current.commitSha, state.current); + return withPreparedReleaseTransition(candidate, options, async () => { + if (state.current?.commitSha === candidate.commitSha) { + return state; } - await applyReleaseLinkState(layout, journal.after, expectedReleases); + + const before = releaseLinkStateFromDashboardState(state); + const journal: ReleaseTransitionJournal = { + after: { + current: candidate.commitSha, + previous: before.current, + }, + before, + formatVersion: RELEASE_TRANSITION_FORMAT_VERSION, + operation: "activate", + }; + return await executeReleaseTransition(layout, journal, async () => { + const expectedReleases = new Map([ + [candidate.commitSha, candidate], + ]); + if (state.current) { + expectedReleases.set(state.current.commitSha, state.current); + } + await applyReleaseLinkState(layout, journal.after, expectedReleases); + }); }); }, options.transitionLockWaitMs @@ -1428,20 +1463,22 @@ export async function rollbackDashboardRelease( formatVersion: RELEASE_TRANSITION_FORMAT_VERSION, operation: "rollback", }; - return await executeReleaseTransition(layout, journal, async () => { - await replaceReleaseLink( - layout, - "current", - rollbackRelease.commitSha, - rollbackRelease - ); - await replaceReleaseLink( - layout, - "previous", - activeRelease.commitSha, - activeRelease - ); - }); + return withPreparedReleaseTransition(rollbackRelease, options, async () => + executeReleaseTransition(layout, journal, async () => { + await replaceReleaseLink( + layout, + "current", + rollbackRelease.commitSha, + rollbackRelease + ); + await replaceReleaseLink( + layout, + "previous", + activeRelease.commitSha, + activeRelease + ); + }) + ); }, options.transitionLockWaitMs ); @@ -1489,7 +1526,9 @@ export async function restoreDashboardReleaseAfterFailedActivation( state.current, options ); - return state; + return withPreparedReleaseTransition(state.current, options, () => + Promise.resolve(state) + ); } if ( state.current?.commitSha !== candidateCommitSha || @@ -1522,19 +1561,21 @@ export async function restoreDashboardReleaseAfterFailedActivation( formatVersion: RELEASE_TRANSITION_FORMAT_VERSION, operation: "restore", }; - return executeReleaseTransition(layout, journal, async () => { - const expectedReleases = new Map([ - [candidateCommitSha, candidateRelease], - [rollbackCommitSha, rollbackRelease], - ]); - if (restoredPreviousRelease) { - expectedReleases.set( - restoredPreviousRelease.commitSha, - restoredPreviousRelease - ); - } - await applyReleaseLinkState(layout, journal.after, expectedReleases); - }); + return withPreparedReleaseTransition(rollbackRelease, options, () => + executeReleaseTransition(layout, journal, async () => { + const expectedReleases = new Map([ + [candidateCommitSha, candidateRelease], + [rollbackCommitSha, rollbackRelease], + ]); + if (restoredPreviousRelease) { + expectedReleases.set( + restoredPreviousRelease.commitSha, + restoredPreviousRelease + ); + } + await applyReleaseLinkState(layout, journal.after, expectedReleases); + }) + ); }, options.transitionLockWaitMs ); diff --git a/backend/src/releaseManifest.ts b/backend/src/releaseManifest.ts index 878f393ad..93f82ddc0 100644 --- a/backend/src/releaseManifest.ts +++ b/backend/src/releaseManifest.ts @@ -19,6 +19,7 @@ import { isBunRuntimeVersion, isCurrentBunRuntime, } from "./managedBunRuntime.ts"; +import { MANAGED_DASHBOARD_UNIT_ARTIFACTS } from "./managedDashboardUnitPolicy.ts"; export { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY } from "./databaseSchemaCompatibility.ts"; @@ -33,16 +34,10 @@ const RELEASE_STATIC_ARTIFACTS = [ "bun.lock", "package.json", ] as const; -// Keep the immediately previous pre-root-workspace release verifiable for -// rollback. Remove this allowlist after both managed slots were built from the -// consolidated root package. -const PRE_ROOT_WORKSPACE_RELEASE_ARTIFACTS = [ - "backend/bun.lock", - "backend/package.json", -] as const; +const OPTIONAL_RELEASE_STATIC_ARTIFACTS = [...MANAGED_DASHBOARD_UNIT_ARTIFACTS] as const; const SAFE_RELEASE_STATIC_ARTIFACTS = [ ...RELEASE_STATIC_ARTIFACTS, - ...PRE_ROOT_WORKSPACE_RELEASE_ARTIFACTS, + ...OPTIONAL_RELEASE_STATIC_ARTIFACTS, ] as const; const REQUIRED_RELEASE_ARTIFACTS = [ ...RELEASE_STATIC_ARTIFACTS, @@ -288,25 +283,33 @@ export async function listReleaseArtifactPaths(releaseRoot: string): Promise { + try { + const stat = await fsp.lstat(artifactPath(realReleaseRoot, relativePath)); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new TypeError( + `Release artifact must be a regular file: ${relativePath}` + ); + } + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; } - throw error; + }) + ); + if ( + optionalArtifactPresence.some(Boolean) && + !optionalArtifactPresence.every(Boolean) + ) { + throw new TypeError("Managed systemd release artifacts must be complete"); + } + for (const [index, isPresent] of optionalArtifactPresence.entries()) { + if (isPresent) { + paths.push(OPTIONAL_RELEASE_STATIC_ARTIFACTS[index] as string); } } for (const directory of RELEASE_ARTIFACT_DIRECTORIES) { @@ -628,6 +631,9 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { const artifacts = value.artifacts.map((artifact) => parseArtifact(artifact)); const artifactPaths = artifacts.map((artifact) => artifact.path); const sortedArtifactPaths = artifactPaths.toSorted(compareStrings); + const managedSystemdArtifactCount = MANAGED_DASHBOARD_UNIT_ARTIFACTS.filter( + (artifactPath_) => artifactPaths.includes(artifactPath_) + ).length; if ( new Set(artifactPaths).size !== artifactPaths.length || artifactPaths.some( @@ -635,7 +641,9 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { ) || REQUIRED_RELEASE_ARTIFACTS.some( (requiredPath) => !artifactPaths.includes(requiredPath) - ) + ) || + (managedSystemdArtifactCount !== 0 && + managedSystemdArtifactCount !== MANAGED_DASHBOARD_UNIT_ARTIFACTS.length) ) { throw new TypeError("Release manifest artifact inventory is invalid"); } diff --git a/backend/src/routes/jobExecutionRoutes.ts b/backend/src/routes/jobExecutionRoutes.ts index 768e14866..5638a0b76 100644 --- a/backend/src/routes/jobExecutionRoutes.ts +++ b/backend/src/routes/jobExecutionRoutes.ts @@ -3,12 +3,15 @@ import type { JobExecutionCancelResponse, JobExecutionResponse, JobExecutionsResponse, + JobWorkerClaimsMutationResponse, } from "../../../contracts/jobs.ts"; +import { parseJobWorkerClaimsPatch } from "../../../contracts/jobs.ts"; import { json } from "../http.ts"; import { httpStatusCode } from "../lib/errors.ts"; import { createStructuredLogger } from "../lib/structuredLogger.ts"; import { type ParametersRequest, + readApiJson, routeErrorResponse, routeFailureResponse, } from "../routeSupport.ts"; @@ -19,6 +22,7 @@ import { type JobExecutionRecord, listJobExecutions, } from "../services/jobExecutionQueue.ts"; +import { setJobWorkerClaimsPaused } from "../services/jobWorkerControl.ts"; const logger = createStructuredLogger("job-execution-route"); @@ -54,6 +58,15 @@ function executionLimit(request: Request): number { return Number(value); } +function includeClaimsState(request: Request): boolean { + return ( + new URL(request.url).searchParams + .get("include") + ?.split(",") + .includes("claims") === true + ); +} + function isValidExecutionId(id: string): boolean { return /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test( id @@ -64,11 +77,17 @@ export const jobExecutionRoutes = { "/api/job-executions": { GET: (request: Request) => { try { + const summary = getJobExecutionSummary(); + const backwardCompatibleSummary = { ...summary }; + delete backwardCompatibleSummary.claimsPaused; + delete backwardCompatibleSummary.claimsPausedAt; return json({ executions: listJobExecutions(executionLimit(request)).map( (execution) => publicExecution(execution) ), - summary: getJobExecutionSummary(), + summary: includeClaimsState(request) + ? summary + : backwardCompatibleSummary, } satisfies JobExecutionsResponse); } catch (error) { logger.error("job_execution.queue_lookup_failed", { error }); @@ -80,6 +99,23 @@ export const jobExecutionRoutes = { } }, }, + "/api/job-executions/claims": { + PATCH: async (request: Request) => { + try { + const patch = await readApiJson(request, parseJobWorkerClaimsPatch); + return json({ + isOk: true, + state: setJobWorkerClaimsPaused(patch.paused), + } satisfies JobWorkerClaimsMutationResponse); + } catch (error) { + return routeErrorResponse(request, error, { + code: "job_worker_claims_update_failed", + context: "job-execution.claims", + message: "Job worker claim state update failed", + }); + } + }, + }, "/api/job-executions/:id": { GET: (request: ParametersRequest<"id">) => { const id = String(request.params.id); diff --git a/backend/src/services/cacheRefresh.ts b/backend/src/services/cacheRefresh.ts index 4d7f0deb9..37418b627 100644 --- a/backend/src/services/cacheRefresh.ts +++ b/backend/src/services/cacheRefresh.ts @@ -11,7 +11,6 @@ import { import os from "node:os"; import type { JobResourceClass, ScheduledJob } from "../../../contracts/jobs.ts"; -import type { CacheRefreshMetrics } from "../../../contracts/metrics.ts"; import { database } from "../database.ts"; import { getCacheEntry, @@ -36,6 +35,12 @@ import { type CacheTtlUnit, writeCacheSuccess, } from "./cacheEntryWriter.ts"; +import { + recordCacheRefreshCoalesced, + recordCacheRefreshFinished, + recordCacheRefreshRequest, + recordCacheRefreshStarted, +} from "./cacheRefreshMetrics.ts"; import { getDatabaseOverview, getIsolatedDatabaseOverview } from "./databaseOverview.ts"; import { evaluateOpenClawNotifications } from "./openclawNotifications.ts"; import { evaluateQuotaNotifications } from "./quotaNotifications.ts"; @@ -1898,33 +1903,13 @@ function redactOpenAiQuotaAccount(openai: Awaited>(); -const cacheRefreshMetricsState: Omit = { - active: 0, - coalesced: 0, - failures: 0, - lastDurationMs: 0, - maxDurationMs: 0, - refreshes: 0, - requests: 0, - totalDurationMs: 0, -}; -/** - * Returns aggregate producer timing without cache keys or cached payloads. - * @returns aggregate producer timing without cache keys or cached payloads. - */ -export function getCacheRefreshMetrics(): CacheRefreshMetrics { - return { - ...cacheRefreshMetricsState, - averageDurationMs: - cacheRefreshMetricsState.refreshes === 0 - ? 0 - : Math.round( - (cacheRefreshMetricsState.totalDurationMs / - cacheRefreshMetricsState.refreshes) * - 100 - ) / 100, - }; +function observeCacheRefreshMetric(event: string, operation: () => void): void { + try { + operation(); + } catch (error) { + logger.warn("cache_refresh.metrics_write_failed", { error, metricEvent: event }); + } } class SerialOperationQueue { @@ -2164,7 +2149,7 @@ export async function refreshCacheProducer( signal?: AbortSignal, options: { force?: boolean } = {} ) { - cacheRefreshMetricsState.requests += 1; + observeCacheRefreshMetric("request", recordCacheRefreshRequest); if (signal?.aborted) { throw abortError(); } @@ -2179,7 +2164,7 @@ export async function refreshCacheProducer( ) .toSorted(([left], [right]) => left.length - right.length)[0]?.[1]; if (existing !== undefined && !options.force) { - cacheRefreshMetricsState.coalesced += 1; + observeCacheRefreshMetric("coalesced", recordCacheRefreshCoalesced); return await waitForExistingRefresh(key, scopeKey, existing, signal); } const childRefreshes = inFlightEntries @@ -2196,27 +2181,20 @@ export async function refreshCacheProducer( ? refreshAfterChildRefreshes(childRefreshes, key, signal) : runBoundedCacheRefresh(() => refreshCacheProducerUnlocked(key), signal); const startedAt = performance.now(); - cacheRefreshMetricsState.active += 1; - cacheRefreshMetricsState.refreshes += 1; + observeCacheRefreshMetric("started", recordCacheRefreshStarted); inFlightCacheRefreshes.set(scopeKey, refresh); void (async () => { + let failed = false; try { await refresh; } catch { - cacheRefreshMetricsState.failures += 1; + failed = true; // The caller observes refresh failures. } finally { const durationMs = Math.round(Math.max(0, performance.now() - startedAt) * 100) / 100; - cacheRefreshMetricsState.lastDurationMs = durationMs; - cacheRefreshMetricsState.maxDurationMs = Math.max( - cacheRefreshMetricsState.maxDurationMs, - durationMs - ); - cacheRefreshMetricsState.totalDurationMs += durationMs; - cacheRefreshMetricsState.active = Math.max( - 0, - cacheRefreshMetricsState.active - 1 + observeCacheRefreshMetric("finished", () => + recordCacheRefreshFinished(durationMs, failed) ); if (inFlightCacheRefreshes.get(scopeKey) === refresh) { inFlightCacheRefreshes.delete(scopeKey); diff --git a/backend/src/services/cacheRefreshMetrics.ts b/backend/src/services/cacheRefreshMetrics.ts new file mode 100644 index 000000000..f30a2c5dd --- /dev/null +++ b/backend/src/services/cacheRefreshMetrics.ts @@ -0,0 +1,305 @@ +import fs from "node:fs"; +import path from "node:path"; + +import * as v from "valibot"; + +import type { CacheRefreshMetrics } from "../../../contracts/metrics.ts"; +import { cacheRefreshMetricsSchema } from "../../../contracts/metrics.ts"; + +const CACHE_REFRESH_METRICS_SNAPSHOT_VERSION = 1; +const MAX_CACHE_REFRESH_METRICS_SNAPSHOT_BYTES = 16 * 1024; +const RUNTIME_DIRECTORY_NAME = "mira-dashboard"; +const SNAPSHOT_FILE_NAME = "cache-refresh-metrics.json"; + +interface CacheRefreshMetricsSnapshot { + instanceId: string; + metrics: CacheRefreshMetrics; + pid: number; + startedAt: string; + version: typeof CACHE_REFRESH_METRICS_SNAPSHOT_VERSION; +} + +interface CacheRefreshMetricsSessionOptions { + environment?: Record; + snapshotPath?: string; +} + +const emptyCacheRefreshMetrics = (): CacheRefreshMetrics => ({ + active: 0, + averageDurationMs: 0, + coalesced: 0, + failures: 0, + lastDurationMs: 0, + maxDurationMs: 0, + refreshes: 0, + requests: 0, + totalDurationMs: 0, +}); + +let cacheRefreshMetricsState = emptyCacheRefreshMetrics(); +let activeSession: + | { + instanceId: string; + snapshotPath: string | undefined; + startedAt: string; + } + | undefined; + +function runtimeDirectory( + environment: Record +): string | undefined { + const configured = environment.XDG_RUNTIME_DIR?.trim(); + if (configured) { + return path.isAbsolute(configured) ? path.resolve(configured) : undefined; + } + if (environment.NODE_ENV === "production" && typeof process.getuid === "function") { + return `/run/user/${process.getuid()}`; + } + return undefined; +} + +/** + * Resolves the private, reboot-volatile IPC snapshot used by the production + * web process to sample metrics owned by the separate worker process. + * @param environment Runtime environment used to resolve the volatile root. + * @returns Snapshot path in production when a safe runtime root is available. + */ +export function resolveCacheRefreshMetricsSnapshotPath( + environment: Record = process.env +): string | undefined { + if (environment.NODE_ENV !== "production") return undefined; + const root = runtimeDirectory(environment); + if (!root || !path.isAbsolute(root) || path.parse(root).root === root) { + return undefined; + } + return path.join(root, RUNTIME_DIRECTORY_NAME, SNAPSHOT_FILE_NAME); +} + +function metricsSnapshot(): CacheRefreshMetrics { + return { + ...cacheRefreshMetricsState, + averageDurationMs: + cacheRefreshMetricsState.refreshes === 0 + ? 0 + : Math.round( + (cacheRefreshMetricsState.totalDurationMs / + cacheRefreshMetricsState.refreshes) * + 100 + ) / 100, + }; +} + +function ensurePrivateRuntimeDirectory(directoryPath: string): void { + fs.mkdirSync(directoryPath, { mode: 0o700, recursive: true }); + const stat = fs.lstatSync(directoryPath); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + fs.realpathSync(directoryPath) !== path.resolve(directoryPath) + ) { + throw new TypeError( + `Cache refresh metrics runtime path must be a real directory: ${directoryPath}` + ); + } + fs.chmodSync(directoryPath, 0o700); +} + +function writeSnapshot( + snapshotPath: string, + snapshot: CacheRefreshMetricsSnapshot +): void { + const directoryPath = path.dirname(snapshotPath); + ensurePrivateRuntimeDirectory(directoryPath); + const temporaryPath = path.join( + directoryPath, + `.${SNAPSHOT_FILE_NAME}.${process.pid}.${Bun.randomUUIDv7()}.tmp` + ); + let temporaryCreated = false; + try { + fs.writeFileSync(temporaryPath, `${JSON.stringify(snapshot)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + temporaryCreated = true; + fs.renameSync(temporaryPath, snapshotPath); + temporaryCreated = false; + } finally { + if (temporaryCreated) { + fs.rmSync(temporaryPath, { force: true }); + } + } +} + +function publishSnapshot(): void { + if (!activeSession?.snapshotPath) return; + writeSnapshot(activeSession.snapshotPath, { + instanceId: activeSession.instanceId, + metrics: metricsSnapshot(), + pid: process.pid, + startedAt: activeSession.startedAt, + version: CACHE_REFRESH_METRICS_SNAPSHOT_VERSION, + }); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function readSnapshot(snapshotPath: string): CacheRefreshMetricsSnapshot | undefined { + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + snapshotPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK + ); + const stat = fs.fstatSync(descriptor); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size <= 0 || + stat.size > MAX_CACHE_REFRESH_METRICS_SNAPSHOT_BYTES + ) { + return undefined; + } + const value: unknown = JSON.parse(fs.readFileSync(descriptor, "utf8")); + if ( + !isRecord(value) || + value.version !== CACHE_REFRESH_METRICS_SNAPSHOT_VERSION || + typeof value.instanceId !== "string" || + typeof value.startedAt !== "string" || + typeof value.pid !== "number" || + !Number.isSafeInteger(value.pid) || + value.pid <= 0 || + !isProcessAlive(value.pid) + ) { + return undefined; + } + const parsedMetrics = v.safeParse(cacheRefreshMetricsSchema, value.metrics); + if (!parsedMetrics.success) return undefined; + return { + instanceId: value.instanceId, + metrics: parsedMetrics.output, + pid: value.pid, + startedAt: value.startedAt, + version: CACHE_REFRESH_METRICS_SNAPSHOT_VERSION, + }; + } catch { + return undefined; + } finally { + if (descriptor !== undefined) { + fs.closeSync(descriptor); + } + } +} + +/** + * Starts a fresh in-memory metrics session and publishes its zero snapshot for + * production IPC. Repeated registration inside the same worker is idempotent. + */ +export function startCacheRefreshMetricsSession( + options: CacheRefreshMetricsSessionOptions = {} +): void { + if (activeSession) return; + cacheRefreshMetricsState = emptyCacheRefreshMetrics(); + activeSession = { + instanceId: Bun.randomUUIDv7(), + snapshotPath: + options.snapshotPath ?? + resolveCacheRefreshMetricsSnapshotPath(options.environment ?? process.env), + startedAt: new Date().toISOString(), + }; + publishSnapshot(); +} + +/** + * Removes only this worker instance's volatile snapshot. A replacement worker + * that has already published a newer instance is left intact. + */ +export function stopCacheRefreshMetricsSession(): void { + const session = activeSession; + activeSession = undefined; + if (!session?.snapshotPath) return; + const current = readSnapshot(session.snapshotPath); + if (current?.instanceId !== session.instanceId) return; + try { + fs.unlinkSync(session.snapshotPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } +} + +/** + * Returns runtime-only cache producer metrics for the active worker session. + * @param options Environment and snapshot overrides. + * @returns Current runtime metrics or an empty snapshot when unavailable. + */ +export function getCacheRefreshMetrics( + options: CacheRefreshMetricsSessionOptions = {} +): CacheRefreshMetrics { + if (activeSession || (options.environment ?? process.env).NODE_ENV !== "production") { + return metricsSnapshot(); + } + const snapshotPath = + options.snapshotPath ?? + resolveCacheRefreshMetricsSnapshotPath(options.environment ?? process.env); + return ( + (snapshotPath && readSnapshot(snapshotPath)?.metrics) || + emptyCacheRefreshMetrics() + ); +} + +/** Records one producer request before abort or coalescing decisions. */ +export function recordCacheRefreshRequest(): void { + cacheRefreshMetricsState.requests += 1; + publishSnapshot(); +} + +/** Records a request that shares an already-running producer. */ +export function recordCacheRefreshCoalesced(): void { + cacheRefreshMetricsState.coalesced += 1; + publishSnapshot(); +} + +/** Records the start of one real cache producer invocation. */ +export function recordCacheRefreshStarted(): void { + cacheRefreshMetricsState.active += 1; + cacheRefreshMetricsState.refreshes += 1; + publishSnapshot(); +} + +/** + * Records producer settlement and bounded duration aggregates. + * @param durationMs Producer duration. + * @param failed Whether the producer rejected. + */ +export function recordCacheRefreshFinished(durationMs: number, failed: boolean): void { + const boundedDuration = + Math.round(Math.max(0, Number.isFinite(durationMs) ? durationMs : 0) * 100) / 100; + cacheRefreshMetricsState.active = Math.max(0, cacheRefreshMetricsState.active - 1); + cacheRefreshMetricsState.failures += failed ? 1 : 0; + cacheRefreshMetricsState.lastDurationMs = boundedDuration; + cacheRefreshMetricsState.maxDurationMs = Math.max( + cacheRefreshMetricsState.maxDurationMs, + boundedDuration + ); + cacheRefreshMetricsState.totalDurationMs += boundedDuration; + publishSnapshot(); +} + +/** Resets module state between isolated tests. */ +export function resetCacheRefreshMetricsForTests(): void { + stopCacheRefreshMetricsSession(); + cacheRefreshMetricsState = emptyCacheRefreshMetrics(); +} diff --git a/backend/src/services/jobExecutionQueue.ts b/backend/src/services/jobExecutionQueue.ts index 73c3ad297..d26fc6b37 100644 --- a/backend/src/services/jobExecutionQueue.ts +++ b/backend/src/services/jobExecutionQueue.ts @@ -13,6 +13,7 @@ import { auditProvenanceForTarget, writeAuditEvent, } from "./auditEvents.ts"; +import { getJobWorkerClaimsState } from "./jobWorkerControl.ts"; const DEFAULT_LEASE_MS = 2 * 60 * 1000; const MAX_EXECUTION_LIST_LIMIT = 200; @@ -414,6 +415,7 @@ export function listJobExecutions(limit = 50): JobExecutionRecord[] { } export function getJobExecutionSummary(timestamp = Date.now()): JobExecutionSummary { + const claims = getJobWorkerClaimsState(); const counts = database .prepare( `SELECT @@ -457,6 +459,8 @@ export function getJobExecutionSummary(timestamp = Date.now()): JobExecutionSumm activeResourceClasses: activeRows .map((row) => row.resource_class) .filter((resourceClass) => isJobResourceClass(resourceClass)), + claimsPaused: claims.paused, + claimsPausedAt: claims.paused ? claims.updatedAt : undefined, oldestQueuedAgeMs: Number.isFinite(parsedOldestQueuedAt) ? Math.max(0, timestamp - parsedOldestQueuedAt) : undefined, @@ -645,6 +649,10 @@ export function claimNextJobExecution( database.run("BEGIN IMMEDIATE"); try { recoverExpiredJobExecutionsInTransaction(timestamp); + if (getJobWorkerClaimsState().paused) { + database.run("COMMIT"); + return undefined; + } const active = database .prepare( "SELECT COUNT(*) AS count FROM job_executions WHERE status = 'running'" diff --git a/backend/src/services/jobWorker.ts b/backend/src/services/jobWorker.ts index 32b62e6b2..72355cbe9 100644 --- a/backend/src/services/jobWorker.ts +++ b/backend/src/services/jobWorker.ts @@ -5,6 +5,10 @@ import { enqueueDatabaseSummaryRefresh, registerCacheRefreshScheduledJobs, } from "./cacheRefresh.ts"; +import { + startCacheRefreshMetricsSession, + stopCacheRefreshMetricsSession, +} from "./cacheRefreshMetrics.ts"; import { registerDockerExecutionActions } from "./dockerActions.ts"; import { registerDockerUpdaterScheduledJobs } from "./dockerUpdater.ts"; import { registerExecExecutionActions } from "./execJobs.ts"; @@ -87,6 +91,22 @@ function registerScheduledActions(profile = dashboardJobProfile()): void { registerPullRequestPreviewExecutionActions(); } +function startCacheRefreshMetrics(): void { + try { + startCacheRefreshMetricsSession(); + } catch (error) { + logger.warn("job_worker.cache_refresh_metrics_start_failed", { error }); + } +} + +function stopCacheRefreshMetrics(): void { + try { + stopCacheRefreshMetricsSession(); + } catch (error) { + logger.warn("job_worker.cache_refresh_metrics_stop_failed", { error }); + } +} + /** * Starts the persistent queue scheduler and its single-concurrency executor. * @param releaseCommit Release commit value. @@ -96,6 +116,7 @@ export function startDashboardJobWorker(releaseCommit = "development"): void { workerState.isStarted = true; const profile = dashboardJobProfile(); try { + startCacheRefreshMetrics(); registerScheduledActions(profile); startScheduledJobExecutor(releaseCommit); startScheduledJobScheduler(); @@ -106,6 +127,7 @@ export function startDashboardJobWorker(releaseCommit = "development"): void { try { await stopScheduledJobExecutor(); workerState.isStarted = false; + stopCacheRefreshMetrics(); } catch (cleanupError) { logger.error("job_worker.executor_startup_rollback_failed", { error: cleanupError, @@ -128,6 +150,7 @@ export async function stopDashboardJobWorker(): Promise { await trackWorkerStop(async () => { stopScheduledJobScheduler(); await stopScheduledJobExecutor(); + stopCacheRefreshMetrics(); // Release the startup guard only after executor cleanup succeeds. workerState.isStarted = false; logger.info("job_worker.stopped"); diff --git a/backend/src/services/jobWorkerControl.ts b/backend/src/services/jobWorkerControl.ts new file mode 100644 index 000000000..7d1aae8fa --- /dev/null +++ b/backend/src/services/jobWorkerControl.ts @@ -0,0 +1,57 @@ +import { database } from "../database.ts"; + +export interface JobWorkerClaimsState { + paused: boolean; + updatedAt: string; +} + +interface JobWorkerControlRow { + claims_paused: number; + updated_at: string; +} + +/** + * Reads the shared operator claim state used by both the web and worker + * processes. + * @returns Current persistent claim state. + */ +export function getJobWorkerClaimsState(): JobWorkerClaimsState { + const row = database + .prepare( + `SELECT claims_paused, updated_at + FROM job_worker_control + WHERE id = 1` + ) + .get() as JobWorkerControlRow | undefined; + if (!row) { + throw new Error("Job worker control state is unavailable"); + } + return { + paused: row.claims_paused === 1, + updatedAt: row.updated_at, + }; +} + +/** + * Persists whether the worker may claim another queued execution. Active work + * remains cooperative and is not cancelled. + * @param paused Whether new claims must remain paused. + * @param updatedAt State transition timestamp. + * @returns Updated persistent claim state. + */ +export function setJobWorkerClaimsPaused( + paused: boolean, + updatedAt = new Date().toISOString() +): JobWorkerClaimsState { + const result = database + .prepare( + `UPDATE job_worker_control + SET claims_paused = ?, updated_at = ? + WHERE id = 1` + ) + .run(paused ? 1 : 0, updatedAt); + if (result.changes !== 1) { + throw new Error("Job worker control state update failed"); + } + return { paused, updatedAt }; +} diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 8a4b56d20..dbb3108d2 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -2124,8 +2124,7 @@ function releaseCutoverShellFunctions(): string[] { ' [ "$(/usr/bin/realpath --canonicalize-existing "$runtime_path")" = "$runtime_path" ] || return 1', ' [ "$(/usr/bin/stat --format=\'%h\' -- "$runtime_path")" = 1 ] || return 1', ' runtime_revision="$("$runtime_path" --revision)" || return 1', - ' runtime_version="$("$runtime_path" --version)" || return 1', - ' [ "$runtime_revision" = "$bun_version" ] || [ "$runtime_version" = "$bun_version" ] || return 1', + ' [ "$runtime_revision" = "$bun_version" ] || return 1', ' printf "%s" "$runtime_path"', "}", "worker_identity() {", diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index bd041fdaf..b5eb86d9e 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -29,6 +29,7 @@ import { unregisterJobWorker, updateJobExecutionOutput, } from "./jobExecutionQueue.ts"; +import { getJobWorkerClaimsState } from "./jobWorkerControl.ts"; import { waitForJobExecution } from "./queuedJobExecution.ts"; const logger = createStructuredLogger("scheduled-jobs"); @@ -1511,6 +1512,9 @@ function executorTick(): void { } scheduledJobRuntimeState.isExecutorTickRunning = true; try { + if (getJobWorkerClaimsState().paused) { + return; + } // The in-memory pause is lost when the deployment restarts this worker. // Keep replacement workers idle until the detached guardian records a // terminal deployment status in the shared database. diff --git a/backend/test/cacheRefreshMetrics.test.ts b/backend/test/cacheRefreshMetrics.test.ts new file mode 100644 index 000000000..2bf2158e0 --- /dev/null +++ b/backend/test/cacheRefreshMetrics.test.ts @@ -0,0 +1,176 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { + getCacheRefreshMetrics, + recordCacheRefreshCoalesced, + recordCacheRefreshFinished, + recordCacheRefreshRequest, + recordCacheRefreshStarted, + resetCacheRefreshMetricsForTests, + resolveCacheRefreshMetricsSnapshotPath, + startCacheRefreshMetricsSession, + stopCacheRefreshMetricsSession, +} from "../src/services/cacheRefreshMetrics.ts"; + +const temporaryRoots: string[] = []; + +function temporaryRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), "mira-cache-refresh-metrics-")); + temporaryRoots.push(root); + return root; +} + +async function readText(stream: ReadableStream): Promise { + return new Response(stream).text(); +} + +afterEach(() => { + resetCacheRefreshMetricsForTests(); + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("cache refresh runtime metrics", () => { + it("publishes worker-owned counters for the separate production web process", async () => { + const runtimeRoot = temporaryRoot(); + const environment = { + NODE_ENV: "production", + XDG_RUNTIME_DIR: runtimeRoot, + }; + const snapshotPath = resolveCacheRefreshMetricsSnapshotPath(environment); + expect(snapshotPath).toBe( + path.join(runtimeRoot, "mira-dashboard", "cache-refresh-metrics.json") + ); + + startCacheRefreshMetricsSession({ environment }); + recordCacheRefreshRequest(); + recordCacheRefreshStarted(); + recordCacheRefreshCoalesced(); + recordCacheRefreshFinished(12.345, true); + + expect(getCacheRefreshMetrics()).toEqual({ + active: 0, + averageDurationMs: 12.35, + coalesced: 1, + failures: 1, + lastDurationMs: 12.35, + maxDurationMs: 12.35, + refreshes: 1, + requests: 1, + totalDurationMs: 12.35, + }); + expect(statSync(path.dirname(snapshotPath!)).mode & 0o777).toBe(0o700); + expect(statSync(snapshotPath!).mode & 0o777).toBe(0o600); + + const moduleUrl = pathToFileURL( + path.resolve(import.meta.dirname, "../src/services/cacheRefreshMetrics.ts") + ).href; + const child = Bun.spawn({ + cmd: [ + process.execPath, + "--eval", + `const metrics = await import(${JSON.stringify(moduleUrl)}); + console.log(JSON.stringify(metrics.getCacheRefreshMetrics()));`, + ], + env: { + ...process.env, + NODE_ENV: "production", + XDG_RUNTIME_DIR: runtimeRoot, + }, + stderr: "pipe", + stdout: "pipe", + }); + const [exitCode, stderr, stdout] = await Promise.all([ + child.exited, + readText(child.stderr), + readText(child.stdout), + ]); + expect({ exitCode, stderr }).toEqual({ exitCode: 0, stderr: "" }); + expect(JSON.parse(stdout)).toEqual(getCacheRefreshMetrics()); + + stopCacheRefreshMetricsSession(); + expect(existsSync(snapshotPath!)).toBe(false); + }); + + it("fails soft on missing or malformed runtime snapshots", () => { + const runtimeRoot = temporaryRoot(); + const environment = { + NODE_ENV: "production", + XDG_RUNTIME_DIR: runtimeRoot, + }; + expect( + resolveCacheRefreshMetricsSnapshotPath({ + NODE_ENV: "production", + XDG_RUNTIME_DIR: "relative-runtime", + }) + ).toBeUndefined(); + const snapshotPath = resolveCacheRefreshMetricsSnapshotPath(environment)!; + expect(getCacheRefreshMetrics({ environment })).toEqual({ + active: 0, + averageDurationMs: 0, + coalesced: 0, + failures: 0, + lastDurationMs: 0, + maxDurationMs: 0, + refreshes: 0, + requests: 0, + totalDurationMs: 0, + }); + + mkdirSync(path.dirname(snapshotPath), { mode: 0o700, recursive: true }); + writeFileSync(snapshotPath, '{"version":1,"metrics":"invalid"}\n', { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + expect(readFileSync(snapshotPath, "utf8")).toContain('"invalid"'); + expect(getCacheRefreshMetrics({ environment })).toEqual({ + active: 0, + averageDurationMs: 0, + coalesced: 0, + failures: 0, + lastDurationMs: 0, + maxDurationMs: 0, + refreshes: 0, + requests: 0, + totalDurationMs: 0, + }); + + rmSync(snapshotPath); + writeFileSync( + snapshotPath, + `${JSON.stringify({ + instanceId: Bun.randomUUIDv7(), + metrics: { + active: 0, + averageDurationMs: 4, + coalesced: 0, + failures: 0, + lastDurationMs: 4, + maxDurationMs: 4, + refreshes: 1, + requests: 1, + totalDurationMs: 4, + }, + pid: 999_999_999, + startedAt: "2026-07-30T08:00:00.000Z", + version: 1, + })}\n`, + { encoding: "utf8", flag: "wx", mode: 0o600 } + ); + expect(getCacheRefreshMetrics({ environment }).requests).toBe(0); + }); +}); diff --git a/backend/test/databaseLifecycle.test.ts b/backend/test/databaseLifecycle.test.ts index a121d46f9..f371975aa 100644 --- a/backend/test/databaseLifecycle.test.ts +++ b/backend/test/databaseLifecycle.test.ts @@ -114,10 +114,19 @@ describe("Dashboard SQLite lifecycle", () => { const first = applyDatabaseMigrations(database, databasePath); const second = applyDatabaseMigrations(database, databasePath); - expect(first.applied).toEqual([1, 2, 3, 4, 5, 6, 7]); + expect(first.applied).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); expect(first.backup).toBeUndefined(); expect(second).toEqual({ applied: [] }); - expect(validateDatabaseMigrationHistory(database)).toBe(7); + expect(validateDatabaseMigrationHistory(database)).toBe(8); + expect( + database + .query( + `SELECT claims_paused, updated_at + FROM job_worker_control + WHERE id = 1` + ) + .get() + ).toMatchObject({ claims_paused: 0 }); expect( database .query("SELECT name FROM pragma_table_info('auth_sessions')") @@ -250,7 +259,7 @@ describe("Dashboard SQLite lifecycle", () => { } }); - it("upgrades an existing version 3 database with migrations 4 through 7", () => { + it("upgrades an existing version 3 database with migrations 4 through 8", () => { const root = temporaryRoot("mira-db-migrations-v3-"); const databasePath = path.join(root, "dashboard.db"); const database = openWalDatabase(databasePath); @@ -259,9 +268,9 @@ describe("Dashboard SQLite lifecycle", () => { expect(validateDatabaseMigrationHistory(database)).toBe(3); expect(migrateDisposableDatabaseCopy(database)).toEqual({ - applied: [4, 5, 6, 7], + applied: [4, 5, 6, 7, 8], }); - expect(validateDatabaseMigrationHistory(database)).toBe(7); + expect(validateDatabaseMigrationHistory(database)).toBe(8); expect( database .query( @@ -293,7 +302,7 @@ describe("Dashboard SQLite lifecycle", () => { } }); - it("upgrades an existing version 4 database with migrations 5 through 7", () => { + it("upgrades an existing version 4 database with migrations 5 through 8", () => { const root = temporaryRoot("mira-db-migrations-v4-"); const databasePath = path.join(root, "dashboard.db"); const database = openWalDatabase(databasePath); @@ -302,9 +311,9 @@ describe("Dashboard SQLite lifecycle", () => { expect(validateDatabaseMigrationHistory(database)).toBe(4); expect(migrateDisposableDatabaseCopy(database)).toEqual({ - applied: [5, 6, 7], + applied: [5, 6, 7, 8], }); - expect(validateDatabaseMigrationHistory(database)).toBe(7); + expect(validateDatabaseMigrationHistory(database)).toBe(8); expect( database .query( @@ -374,9 +383,9 @@ describe("Dashboard SQLite lifecycle", () => { database.query("SELECT COUNT(*) AS count FROM auth_sessions").get() ).toEqual({ count: 2 }); expect(migrateDisposableDatabaseCopy(database)).toEqual({ - applied: [6, 7], + applied: [6, 7, 8], }); - expect(validateDatabaseMigrationHistory(database)).toBe(7); + expect(validateDatabaseMigrationHistory(database)).toBe(8); expect( database.query("SELECT COUNT(*) AS count FROM auth_sessions").get() ).toEqual({ count: 0 }); @@ -402,9 +411,9 @@ describe("Dashboard SQLite lifecycle", () => { expect(legacyIndex.sql).toContain("'restart-scheduled'"); expect(migrateDisposableDatabaseCopy(database)).toEqual({ - applied: [7], + applied: [7, 8], }); - expect(validateDatabaseMigrationHistory(database)).toBe(7); + expect(validateDatabaseMigrationHistory(database)).toBe(8); const currentIndex = database .query( @@ -507,7 +516,7 @@ describe("Dashboard SQLite lifecycle", () => { try { expect( database.query("SELECT COUNT(*) AS count FROM schema_migrations").get() - ).toEqual({ count: 7 }); + ).toEqual({ count: 8 }); } finally { database.close(); } @@ -533,7 +542,7 @@ describe("Dashboard SQLite lifecycle", () => { ); const result = applyDatabaseMigrations(database, databasePath); - expect(result.applied).toEqual([1, 2, 3, 4, 5, 6, 7]); + expect(result.applied).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); expect(result.backup).toMatchObject({ kind: "pre-migration", restoreVerified: true, @@ -587,23 +596,23 @@ describe("Dashboard SQLite lifecycle", () => { .prepare( `INSERT INTO schema_migrations ( version, name, checksum, applied_at - ) VALUES (8, 'unknown', 'unknown', ?)` + ) VALUES (9, 'unknown', 'unknown', ?)` ) .run("2026-07-23T00:00:00.000Z"); expect(() => validateDatabaseMigrationHistory(database)).toThrow( - "incompatible SQLite migration version 8" + "incompatible SQLite migration version 9" ); database .prepare( - "UPDATE schema_migrations SET name = ?, checksum = ? WHERE version = 8" + "UPDATE schema_migrations SET name = ?, checksum = ? WHERE version = 9" ) .run("future-additive", "a".repeat(64)); - expect(validateDatabaseMigrationHistory(database, 8)).toBe(8); - expect(() => validateDatabaseMigrationHistory(database, 7)).toThrow( - "incompatible SQLite migration version 8" + expect(validateDatabaseMigrationHistory(database, 9)).toBe(9); + expect(() => validateDatabaseMigrationHistory(database, 8)).toThrow( + "incompatible SQLite migration version 9" ); - database.prepare("DELETE FROM schema_migrations WHERE version = 8").run(); + database.prepare("DELETE FROM schema_migrations WHERE version = 9").run(); database.prepare("DELETE FROM schema_migrations WHERE version = 2").run(); expect(() => validateDatabaseMigrationHistory(database)).toThrow( "not contiguous" @@ -810,8 +819,8 @@ describe("Dashboard SQLite lifecycle", () => { expect(preflightResult).toMatchObject({ backup: { kind: "pre-deploy", restoreVerified: true }, migrationTest: { - applied: [1, 2, 3, 4, 5, 6, 7], - currentVersion: 7, + applied: [1, 2, 3, 4, 5, 6, 7, 8], + currentVersion: 8, }, }); expect(getSqliteBackupInventory(databasePath).count).toBe(1); diff --git a/backend/test/httpApiBehavior.test.ts b/backend/test/httpApiBehavior.test.ts index 95d796c66..8bc846f84 100644 --- a/backend/test/httpApiBehavior.test.ts +++ b/backend/test/httpApiBehavior.test.ts @@ -1801,9 +1801,10 @@ describe("Mira Dashboard backend integration", () => { const queue = await api<{ executions: Array<{ id: string; scheduledJobId?: string; status: string }>; - summary: { queued: number; running: number }; + summary: { claimsPaused?: boolean; queued: number; running: number }; }>("/api/job-executions"); expect(queue.status).toBe(200); + expect(queue.body.summary).not.toHaveProperty("claimsPaused"); expect(queue.body.summary).toMatchObject({ queued: 1, running: 0 }); expect(queue.body.executions).toContainEqual( expect.objectContaining({ @@ -1813,6 +1814,31 @@ describe("Mira Dashboard backend integration", () => { }) ); + const pausedClaims = await api<{ + isOk: boolean; + state: { paused: boolean; updatedAt: string }; + }>("/api/job-executions/claims", json("PATCH", { paused: true })); + expect(pausedClaims.status).toBe(200); + expect(pausedClaims.body).toMatchObject({ + isOk: true, + state: { paused: true }, + }); + const pausedQueue = await api<{ + summary: { claimsPaused?: boolean; queued: number }; + }>("/api/job-executions?include=claims"); + expect(pausedQueue.body.summary).toMatchObject({ + claimsPaused: true, + queued: 1, + }); + const resumedClaims = await api<{ + isOk: boolean; + state: { paused: boolean }; + }>("/api/job-executions/claims", json("PATCH", { paused: false })); + expect(resumedClaims.body).toMatchObject({ + isOk: true, + state: { paused: false }, + }); + const detail = await api<{ execution: { id: string; output: Record; status: string }; }>(`/api/job-executions/${run.body.run.executionId}`); diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 2d883565a..da72b32ef 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -24,6 +24,7 @@ import { registerJobWorker, unregisterJobWorker, } from "../src/services/jobExecutionQueue.ts"; +import { setJobWorkerClaimsPaused } from "../src/services/jobWorkerControl.ts"; import { waitForJobExecution } from "../src/services/queuedJobExecution.ts"; import { enqueueScheduledJob, @@ -45,6 +46,7 @@ const testDeploymentIds = new Set(); afterEach(async () => { await stopScheduledJobExecutor(); + setJobWorkerClaimsPaused(false); for (const executionId of testExecutionIds) { database.prepare("DELETE FROM job_executions WHERE id = ?").run(executionId); } @@ -602,6 +604,36 @@ printf 'LoadState=loaded\nActiveState=active\n' }); }); + it("persists an operator pause and atomically prevents new claims", () => { + const queued = enqueueJobExecution({ + actionKey: `test.paused-${Bun.randomUUIDv7()}`, + displayName: "Paused mutation", + resourceClass: "exclusive", + timeoutMs: 60_000, + }); + testExecutionIds.add(queued.id); + const pausedAt = "2026-07-30T08:00:00.000Z"; + + expect(setJobWorkerClaimsPaused(true, pausedAt)).toEqual({ + paused: true, + updatedAt: pausedAt, + }); + expect(claimNextJobExecution(`test-worker-${Bun.randomUUIDv7()}`, 1)).toBe( + undefined + ); + expect(getJobExecution(queued.id)).toMatchObject({ status: "queued" }); + expect(getJobExecutionSummary()).toMatchObject({ + claimsPaused: true, + claimsPausedAt: pausedAt, + queued: 1, + }); + + setJobWorkerClaimsPaused(false); + const workerId = `test-worker-${Bun.randomUUIDv7()}`; + expect(claimNextJobExecution(workerId, 1)?.id).toBe(queued.id); + finishJobExecution(queued.id, workerId, "success", undefined, {}); + }); + it("prioritizes interactive work and enforces global capacity", () => { const heavyJobId = createScheduledTestJob("host-heavy", "Heavy test job"); const interactiveJobId = createScheduledTestJob( diff --git a/backend/test/managedBunRuntime.test.ts b/backend/test/managedBunRuntime.test.ts index d88b626a8..efe75db8f 100644 --- a/backend/test/managedBunRuntime.test.ts +++ b/backend/test/managedBunRuntime.test.ts @@ -42,10 +42,6 @@ function writeFakeBun( writeFileSync( filePath, `#!/bin/sh -if [ "\${1:-}" = "--version" ]; then - printf '%s\\n' '${version}' - exit 0 -fi if [ "\${1:-}" = "--revision" ]; then printf '%s\\n' '${revision}' exit 0 @@ -91,13 +87,16 @@ describe("managed Bun runtimes", () => { expect(installed).toBe(managedBunRuntimeExecutablePath(identity, runtimeRoot)); expect(bunExecutableRuntimeIdentity(installed)).toBe(identity); expect(bunExecutableMatchesRuntime(installed, identity)).toBe(true); + expect(bunExecutableMatchesRuntime(installed, "1.3.14")).toBe(false); expect(hasManagedBunRuntime(identity, runtimeRoot)).toBe(true); expect(requireManagedBunRuntime(identity, runtimeRoot)).toBe(installed); expect(await installManagedBunRuntime(source, identity, { runtimeRoot })).toBe( installed ); expect( - readFileSync(installed, "utf8").includes(String.raw`printf '%s\n' '1.3.14'`) + readFileSync(installed, "utf8").includes( + String.raw`printf '%s\n' '1.3.14+0d9b296af'` + ) ).toBe(true); expect(readdirSync(path.join(runtimeRoot, identity))).toEqual(["bun"]); expect(statSync(installed).nlink).toBe(1); @@ -255,6 +254,35 @@ describe("managed Bun runtimes", () => { "dist/workerStart.js" ); + const versionOnlyRuntime = path.join( + projectRoot, + "production", + "runtimes", + "bun", + "2.0.0", + "bun" + ); + mkdirSync(path.dirname(versionOnlyRuntime), { recursive: true }); + writeFakeBun(versionOnlyRuntime, "2.0.0", "2.0.0+feedface"); + writeFileSync( + path.join(releaseRoot, "release-manifest.json"), + `${JSON.stringify({ bunVersion: "2.0.0" })}\n` + ); + const versionOnly = Bun.spawnSync({ + cmd: [launcher, "dist/workerStart.js"], + cwd: releaseBackend, + env: { + MIRA_DASHBOARD_PROJECT_ROOT: projectRoot, + PATH: "/usr/bin:/bin", + }, + stderr: "pipe", + stdout: "pipe", + }); + expect(versionOnly.exitCode).toBe(78); + expect(new TextDecoder().decode(versionOnly.stderr)).toContain( + "runtime version does not match" + ); + const rejected = Bun.spawnSync({ cmd: [launcher, "arbitrary.js"], env: { diff --git a/backend/test/managedDashboardSystemd.test.ts b/backend/test/managedDashboardSystemd.test.ts new file mode 100644 index 000000000..e81be77a1 --- /dev/null +++ b/backend/test/managedDashboardSystemd.test.ts @@ -0,0 +1,255 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + type ManagedDashboardSystemdCommandRunner, + prepareManagedDashboardUnits, +} from "../src/managedDashboardSystemd.ts"; +import { MANAGED_DASHBOARD_UNIT_NAMES } from "../src/managedDashboardUnitPolicy.ts"; +import { loadManagedRelease, managedReleasePath } from "../src/releaseManager.ts"; +import { createReleaseFixture } from "./support/releaseFixture.ts"; + +const COMMIT_SHA = "a".repeat(40); +const temporaryRoots: string[] = []; +const rejectUnexpectedSystemctl: ManagedDashboardSystemdCommandRunner = () => { + throw new Error("systemctl must not run for an incomplete bundle"); +}; + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(path.join(tmpdir(), prefix)); + temporaryRoots.push(root); + return root; +} + +async function managedReleaseWithUnits(releasesRoot: string) { + const releasePath = managedReleasePath(releasesRoot, COMMIT_SHA); + mkdirSync(path.join(releasePath, "systemd"), { recursive: true }); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + writeFileSync( + path.join(releasePath, "systemd", unit), + `[Unit]\nDescription=${unit}\n[Service]\nExecStart=/${unit}\n` + ); + } + await createReleaseFixture(releasePath, COMMIT_SHA); + return loadManagedRelease(releasesRoot, COMMIT_SHA); +} + +async function captureRejection(operation: () => Promise): Promise { + try { + await operation(); + } catch (error) { + return error; + } + throw new Error("Expected operation to reject"); +} + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("managed Dashboard systemd reconciliation", () => { + it("atomically installs and verifies the target release unit bundle", async () => { + const releasesRoot = temporaryRoot("mira-systemd-release-"); + const unitRoot = path.join(temporaryRoot("mira-systemd-user-"), "units"); + const release = await managedReleaseWithUnits(releasesRoot); + const calls: Array<[string, readonly string[]]> = []; + const commandRunner: ManagedDashboardSystemdCommandRunner = ( + command, + arguments_ + ) => { + calls.push([command, arguments_]); + const unit = MANAGED_DASHBOARD_UNIT_NAMES.find((candidate) => + arguments_.includes(candidate) + ); + return Promise.resolve({ + stderr: "", + stdout: unit + ? `DropInPaths=\nFragmentPath=${path.join( + unitRoot, + unit + )}\nLoadState=loaded\n` + : "", + }); + }; + + expect( + await prepareManagedDashboardUnits(release, { commandRunner, unitRoot }) + ).toMatchObject({ + changed: [...MANAGED_DASHBOARD_UNIT_NAMES], + }); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + expect(readFileSync(path.join(unitRoot, unit), "utf8")).toBe( + readFileSync(path.join(release.path, "systemd", unit), "utf8") + ); + expect(statSync(path.join(unitRoot, unit)).mode & 0o777).toBe(0o644); + } + expect(calls).toHaveLength(3); + + calls.length = 0; + expect( + await prepareManagedDashboardUnits(release, { commandRunner, unitRoot }) + ).toMatchObject({ + changed: [], + }); + expect(calls).toHaveLength(3); + }); + + it("repairs unit modes even when the managed content already matches", async () => { + const releasesRoot = temporaryRoot("mira-systemd-mode-release-"); + const unitRoot = path.join(temporaryRoot("mira-systemd-mode-user-"), "units"); + const release = await managedReleaseWithUnits(releasesRoot); + mkdirSync(unitRoot, { recursive: true }); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + writeFileSync( + path.join(unitRoot, unit), + readFileSync(path.join(release.path, "systemd", unit), "utf8"), + { mode: 0o600 } + ); + } + const commandRunner: ManagedDashboardSystemdCommandRunner = ( + _command, + arguments_ + ) => { + const unit = MANAGED_DASHBOARD_UNIT_NAMES.find((candidate) => + arguments_.includes(candidate) + ); + return Promise.resolve({ + stderr: "", + stdout: unit + ? `DropInPaths=\nFragmentPath=${path.join( + unitRoot, + unit + )}\nLoadState=loaded\n` + : "", + }); + }; + + expect( + await prepareManagedDashboardUnits(release, { commandRunner, unitRoot }) + ).toMatchObject({ + changed: [...MANAGED_DASHBOARD_UNIT_NAMES], + }); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + expect(statSync(path.join(unitRoot, unit)).mode & 0o777).toBe(0o644); + } + }); + + it("restores the installed units when daemon verification fails", async () => { + const releasesRoot = temporaryRoot("mira-systemd-rollback-release-"); + const unitRoot = path.join(temporaryRoot("mira-systemd-rollback-user-"), "units"); + const release = await managedReleaseWithUnits(releasesRoot); + mkdirSync(unitRoot, { recursive: true }); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + writeFileSync(path.join(unitRoot, unit), `old ${unit}\n`); + chmodSync(path.join(unitRoot, unit), 0o644); + } + chmodSync(path.join(unitRoot, MANAGED_DASHBOARD_UNIT_NAMES[0]!), 0o600); + let reloads = 0; + const commandRunner: ManagedDashboardSystemdCommandRunner = ( + _command, + arguments_ + ) => { + if (arguments_.includes("daemon-reload")) { + reloads += 1; + return Promise.resolve({ stderr: "", stdout: "" }); + } + return Promise.resolve({ + stderr: "", + stdout: "FragmentPath=/wrong/path\nLoadState=loaded\n", + }); + }; + + const reconcileError = await captureRejection(() => + prepareManagedDashboardUnits(release, { commandRunner, unitRoot }) + ); + expect(reconcileError).toBeInstanceOf(Error); + expect((reconcileError as Error).message).toContain( + "did not load exclusively from its managed unit path" + ); + expect(reloads).toBe(2); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + expect(readFileSync(path.join(unitRoot, unit), "utf8")).toBe(`old ${unit}\n`); + } + expect( + statSync(path.join(unitRoot, MANAGED_DASHBOARD_UNIT_NAMES[0]!)).mode & 0o777 + ).toBe(0o600); + }); + + it("restores the installed units when the release transition fails", async () => { + const releasesRoot = temporaryRoot("mira-systemd-transition-release-"); + const unitRoot = path.join( + temporaryRoot("mira-systemd-transition-user-"), + "units" + ); + const release = await managedReleaseWithUnits(releasesRoot); + mkdirSync(unitRoot, { recursive: true }); + const existingUnit = MANAGED_DASHBOARD_UNIT_NAMES[0]!; + const newlyInstalledUnit = MANAGED_DASHBOARD_UNIT_NAMES[1]!; + writeFileSync(path.join(unitRoot, existingUnit), `old ${existingUnit}\n`); + let reloads = 0; + const commandRunner: ManagedDashboardSystemdCommandRunner = ( + _command, + arguments_ + ) => { + if (arguments_.includes("daemon-reload")) { + reloads += 1; + return Promise.resolve({ stderr: "", stdout: "" }); + } + const unit = MANAGED_DASHBOARD_UNIT_NAMES.find((candidate) => + arguments_.includes(candidate) + ); + return Promise.resolve({ + stderr: "", + stdout: unit + ? `DropInPaths=\nFragmentPath=${path.join( + unitRoot, + unit + )}\nLoadState=loaded\n` + : "", + }); + }; + const prepared = await prepareManagedDashboardUnits(release, { + commandRunner, + unitRoot, + }); + await prepared.rollback(); + expect(reloads).toBe(2); + expect(readFileSync(path.join(unitRoot, existingUnit), "utf8")).toBe( + `old ${existingUnit}\n` + ); + expect(existsSync(path.join(unitRoot, newlyInstalledUnit))).toBe(false); + }); + + it("rejects releases that predate the managed unit bundle", async () => { + const releasesRoot = temporaryRoot("mira-systemd-legacy-release-"); + const releasePath = managedReleasePath(releasesRoot, COMMIT_SHA); + mkdirSync(releasePath, { recursive: true }); + await createReleaseFixture(releasePath, COMMIT_SHA); + const release = await loadManagedRelease(releasesRoot, COMMIT_SHA); + const unitRoot = path.join(temporaryRoot("mira-systemd-legacy-user-"), "units"); + const bundleError = await captureRejection(() => + prepareManagedDashboardUnits(release, { + commandRunner: rejectUnexpectedSystemctl, + unitRoot, + }) + ); + expect(bundleError).toBeInstanceOf(Error); + expect((bundleError as Error).message).toContain( + "does not contain managed systemd units" + ); + expect(existsSync(unitRoot)).toBe(false); + }); +}); diff --git a/backend/test/multiFactorAuth.test.ts b/backend/test/multiFactorAuth.test.ts index fe6b0c21e..14f6c9177 100644 --- a/backend/test/multiFactorAuth.test.ts +++ b/backend/test/multiFactorAuth.test.ts @@ -610,6 +610,13 @@ describe("Dashboard multi-factor authentication", () => { }) ) ).toBe(true); + expect( + requiresRecentMfa( + new Request("https://dashboard.example/api/job-executions/claims", { + method: "PATCH", + }) + ) + ).toBe(true); expect( requiresRecentMfa( new Request("https://dashboard.example/api/terminal/complete", { diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index dbaf63f18..c7f39bfa9 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -24,6 +24,7 @@ import { type DatabaseMigrationIdentity, } from "../src/databaseMigrations/index.ts"; import { + currentBunRuntimeIdentity, hasManagedBunRuntime, installManagedBunRuntime, } from "../src/managedBunRuntime.ts"; @@ -58,17 +59,18 @@ const FIRST_COMMIT = "a".repeat(40); const SECOND_COMMIT = "b".repeat(40); const THIRD_COMMIT = "c".repeat(40); const FOURTH_COMMIT = "d".repeat(40); +const CURRENT_BUN_RUNTIME_IDENTITY = currentBunRuntimeIdentity(); const TEST_FUTURE_MIGRATIONS: DatabaseMigrationIdentity[] = [ - { - checksum: "8".repeat(64), - name: "test-migration-8", - version: 8, - }, { checksum: "9".repeat(64), name: "test-migration-9", version: 9, }, + { + checksum: "a".repeat(64), + name: "test-migration-10", + version: 10, + }, ]; function testLiveSchemaState( @@ -140,7 +142,7 @@ async function createManagedRelease( releasesRoot: string, directoryCommit: string, manifestCommit = directoryCommit, - bunVersion = Bun.version, + bunVersion = CURRENT_BUN_RUNTIME_IDENTITY, builtAt = new Date("2026-07-25T17:00:00.000Z") ): Promise { await ensureDashboardReleaseLayout(releasesRoot); @@ -660,15 +662,15 @@ describe("Dashboard immutable release manager", () => { const candidatePath = await createManagedRelease(root, SECOND_COMMIT); await rewriteManifest(candidatePath, { migrationRegistrySha256: "c".repeat(64), - schemaMaximum: 8, + schemaMaximum: 9, schemaMinimum: 6, - schemaTarget: 8, + schemaTarget: 9, }); await activateDashboardRelease(FIRST_COMMIT, root, SCHEMA_6_OPTIONS); expect( activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS) - ).rejects.toThrow("cannot roll back after SQLite schema 8"); + ).rejects.toThrow("cannot roll back after SQLite schema 9"); expect(readlinkSync(path.join(root, "current"))).toBe(`releases/${FIRST_COMMIT}`); expect(existsSync(path.join(root, "previous"))).toBe(false); }); @@ -742,27 +744,27 @@ describe("Dashboard immutable release manager", () => { const migratedPath = await createManagedRelease(root, SECOND_COMMIT); await createManagedRelease(root, THIRD_COMMIT); await rewriteManifest(rollbackPath, { - schemaMaximum: 8, + schemaMaximum: 9, }); await rewriteManifest(migratedPath, { migrationRegistrySha256: "d".repeat(64), - schemaMaximum: 8, - schemaMinimum: 7, - schemaTarget: 8, + schemaMaximum: 9, + schemaMinimum: 8, + schemaTarget: 9, }); - let liveSchemaVersion = 7; + let liveSchemaVersion = 8; const options = { hasRuntime: () => true, readLiveSchemaState: () => testLiveSchemaState(liveSchemaVersion), }; await activateDashboardRelease(FIRST_COMMIT, root, options); await activateDashboardRelease(SECOND_COMMIT, root, options); - liveSchemaVersion = 8; + liveSchemaVersion = 9; await rollbackDashboardRelease(root, options); expect(activateDashboardRelease(THIRD_COMMIT, root, options)).rejects.toThrow( - "Activation release cannot open live SQLite schema 8" + "Activation release cannot open live SQLite schema 9" ); const state = await readDashboardReleaseState(root); expect(state.current?.commitSha).toBe(FIRST_COMMIT); @@ -774,32 +776,32 @@ describe("Dashboard immutable release manager", () => { const currentPath = await createManagedRelease(root, FIRST_COMMIT); const candidatePath = await createManagedRelease(root, SECOND_COMMIT); await rewriteManifest(currentPath, { - schemaMaximum: 8, + schemaMaximum: 9, }); await rewriteManifest(candidatePath, { migrationRegistrySha256: "d".repeat(64), - schemaMaximum: 8, - schemaMinimum: 7, - schemaTarget: 8, + schemaMaximum: 9, + schemaMinimum: 8, + schemaTarget: 9, }); await activateDashboardRelease(FIRST_COMMIT, root, { hasRuntime: () => true, - readLiveSchemaState: () => testLiveSchemaState(7), + readLiveSchemaState: () => testLiveSchemaState(8), }); expect( activateDashboardRelease(SECOND_COMMIT, root, { hasRuntime: () => true, readLiveSchemaState: () => - testLiveSchemaState(8, { - 8: { + testLiveSchemaState(9, { + 9: { ...TEST_FUTURE_MIGRATIONS[0]!, checksum: "f".repeat(64), }, }), }) ).rejects.toThrow( - "Activation release SQLite migration 8 identity does not match live history" + "Activation release SQLite migration 9 identity does not match live history" ); }); @@ -809,12 +811,12 @@ describe("Dashboard immutable release manager", () => { const candidatePath = await createManagedRelease(root, SECOND_COMMIT); await rewriteManifest(candidatePath, { migrationRegistrySha256: "d".repeat(64), - schemaMaximum: 8, - schemaMinimum: 8, - schemaTarget: 8, + schemaMaximum: 9, + schemaMinimum: 9, + schemaTarget: 9, }); - let liveSchemaVersion = 7; + let liveSchemaVersion = 8; const options = { hasRuntime: () => true, readLiveSchemaState: () => testLiveSchemaState(liveSchemaVersion), @@ -829,7 +831,7 @@ describe("Dashboard immutable release manager", () => { current: { commitSha: FIRST_COMMIT }, }); expect(activateDashboardRelease(SECOND_COMMIT, root, options)).rejects.toThrow( - "cannot roll back after SQLite schema 8" + "cannot roll back after SQLite schema 9" ); await runReleaseLifecycleCommand( @@ -837,15 +839,15 @@ describe("Dashboard immutable release manager", () => { root, options ); - liveSchemaVersion = 8; + liveSchemaVersion = 9; expect( activateDashboardRelease(SECOND_COMMIT, root, { hasRuntime: () => true, - readLiveSchemaState: () => testLiveSchemaState(9), + readLiveSchemaState: () => testLiveSchemaState(10), }) - ).rejects.toThrow("Activation release cannot open live SQLite schema 9"); + ).rejects.toThrow("Activation release cannot open live SQLite schema 10"); expect(rollbackDashboardRelease(root, options)).rejects.toThrow( - "Rollback release cannot open SQLite schema 8" + "Rollback release cannot open SQLite schema 9" ); expect(readlinkSync(path.join(root, "current"))).toBe( `releases/${SECOND_COMMIT}` @@ -857,16 +859,16 @@ describe("Dashboard immutable release manager", () => { const compatibleOldPath = await createManagedRelease(root, FIRST_COMMIT); const migratedPath = await createManagedRelease(root, SECOND_COMMIT); await rewriteManifest(compatibleOldPath, { - schemaMaximum: 8, + schemaMaximum: 9, }); await rewriteManifest(migratedPath, { migrationRegistrySha256: "d".repeat(64), - schemaMaximum: 8, - schemaMinimum: 8, - schemaTarget: 8, + schemaMaximum: 9, + schemaMinimum: 9, + schemaTarget: 9, }); - let liveSchemaVersion = 7; + let liveSchemaVersion = 8; const options = { hasRuntime: () => true, readLiveSchemaState: () => testLiveSchemaState(liveSchemaVersion), @@ -876,7 +878,7 @@ describe("Dashboard immutable release manager", () => { ...options, schemaCutoverMode: "coordinated", }); - liveSchemaVersion = 8; + liveSchemaVersion = 9; const oldCode = await rollbackDashboardRelease(root, options); expect(oldCode.current?.commitSha).toBe(FIRST_COMMIT); @@ -972,14 +974,24 @@ describe("Dashboard immutable release manager", () => { await createManagedRelease(root, SECOND_COMMIT); await activateDashboardRelease(FIRST_COMMIT, root, SCHEMA_6_OPTIONS); const lockFileDescriptor = holdTransitionLock(root); + let preparationCalls = 0; try { expect(readDashboardReleaseState(root)).rejects.toThrow( "Another managed release transition is in progress" ); expect( - activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS) + activateDashboardRelease(SECOND_COMMIT, root, { + ...SCHEMA_6_OPTIONS, + prepareReleaseTransition: () => { + preparationCalls += 1; + return Promise.resolve({ + rollback: () => Promise.resolve(), + }); + }, + }) ).rejects.toThrow("Another managed release transition is in progress"); + expect(preparationCalls).toBe(0); expect(readlinkSync(path.join(root, "current"))).toBe( `releases/${FIRST_COMMIT}` ); @@ -1119,6 +1131,7 @@ describe("Dashboard immutable release manager", () => { await activateDashboardRelease(FIRST_COMMIT, root, SCHEMA_6_OPTIONS); const originalRename = fsp.rename.bind(fsp); let hasChangedCandidate = false; + let preparationRollbacks = 0; const rename = spyOn(fsp, "rename").mockImplementation( async (oldPath, newPath) => { if (!hasChangedCandidate && newPath === path.join(root, "current")) { @@ -1134,12 +1147,24 @@ describe("Dashboard immutable release manager", () => { try { expect( - activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS) + activateDashboardRelease(SECOND_COMMIT, root, { + ...SCHEMA_6_OPTIONS, + prepareReleaseTransition: (target) => { + expect(target.commitSha).toBe(SECOND_COMMIT); + return Promise.resolve({ + rollback: () => { + preparationRollbacks += 1; + return Promise.resolve(); + }, + }); + }, + }) ).rejects.toThrow("Managed release snapshot changed while linking"); } finally { rename.mockRestore(); } + expect(preparationRollbacks).toBe(1); const state = await readDashboardReleaseState(root); expect(state.current?.commitSha).toBe(FIRST_COMMIT); expect(state.previous).toBeUndefined(); @@ -1167,15 +1192,13 @@ describe("Dashboard immutable release manager", () => { `#!/bin/sh if [ "\${1:-}" = "--revision" ]; then printf '%s\\n' '${obsoleteRuntimeIdentity}' -elif [ "\${1:-}" = "--version" ]; then - printf '%s\\n' '2.0.0' else exit 2 fi ` ); chmodSync(obsoleteRuntimeSource, 0o700); - await installManagedBunRuntime(process.execPath, Bun.version, { + await installManagedBunRuntime(process.execPath, CURRENT_BUN_RUNTIME_IDENTITY, { runtimeRoot, }); await installManagedBunRuntime(obsoleteRuntimeSource, obsoleteRuntimeIdentity, { @@ -1185,28 +1208,28 @@ fi root, FIRST_COMMIT, FIRST_COMMIT, - Bun.version, + CURRENT_BUN_RUNTIME_IDENTITY, new Date("2026-07-25T17:00:00.000Z") ); await createManagedRelease( root, SECOND_COMMIT, SECOND_COMMIT, - Bun.version, + CURRENT_BUN_RUNTIME_IDENTITY, new Date("2026-07-25T17:01:00.000Z") ); await createManagedRelease( root, THIRD_COMMIT, THIRD_COMMIT, - Bun.version, + CURRENT_BUN_RUNTIME_IDENTITY, new Date("2026-07-25T17:02:00.000Z") ); await createManagedRelease( root, FOURTH_COMMIT, FOURTH_COMMIT, - Bun.version, + CURRENT_BUN_RUNTIME_IDENTITY, new Date("2026-07-25T17:03:00.000Z") ); await activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS); @@ -1245,10 +1268,12 @@ fi removed: [FIRST_COMMIT], removedRuntimes: [obsoleteRuntimeIdentity], retained: [FOURTH_COMMIT, THIRD_COMMIT, SECOND_COMMIT], - retainedRuntimes: [Bun.version], + retainedRuntimes: [CURRENT_BUN_RUNTIME_IDENTITY], warnings: [`Skipped unverifiable release ${unverifiableCommit}`], }); - expect(hasManagedBunRuntime(Bun.version, runtimeRoot)).toBe(true); + expect(hasManagedBunRuntime(CURRENT_BUN_RUNTIME_IDENTITY, runtimeRoot)).toBe( + true + ); expect(hasManagedBunRuntime(obsoleteRuntimeIdentity, runtimeRoot)).toBe(false); expect(existsSync(managedReleasePath(root, FIRST_COMMIT))).toBe(false); expect(existsSync(managedReleasePath(root, SECOND_COMMIT))).toBe(true); diff --git a/backend/test/releaseManifest.test.ts b/backend/test/releaseManifest.test.ts index 0ce6de266..0a9a98a96 100644 --- a/backend/test/releaseManifest.test.ts +++ b/backend/test/releaseManifest.test.ts @@ -272,20 +272,50 @@ describe("Dashboard release manifest", () => { ); }); - it("verifies declared pre-root workspace package files when present", async () => { + it("does not publish obsolete backend workspace package files", async () => { const root = temporaryReleaseRoot(); writeFileSync(path.join(root, "backend", "package.json"), "{}\n"); writeFileSync(path.join(root, "backend", "bun.lock"), "backend-lock\n"); const manifest = await writeReleaseManifest(manifestOptions(root)); - expect(manifest.artifacts.map((artifact) => artifact.path)).toContain( + expect(manifest.artifacts.map((artifact) => artifact.path)).not.toContain( "backend/package.json" ); + expect(manifest.artifacts.map((artifact) => artifact.path)).not.toContain( + "backend/bun.lock" + ); await verifyReleaseArtifacts(root, manifest); + }); - rmSync(path.join(root, "backend", "package.json")); - expect(verifyReleaseArtifacts(root, manifest)).rejects.toThrow( - "Release artifact inventory does not match its manifest" + it("publishes only complete managed systemd unit bundles", async () => { + const root = temporaryReleaseRoot(); + mkdirSync(path.join(root, "systemd")); + writeFileSync( + path.join(root, "systemd", "mira-dashboard.service"), + "[Service]\nExecStart=/dashboard\n" + ); + writeFileSync( + path.join(root, "systemd", "mira-dashboard-worker.service"), + "[Service]\nExecStart=/worker\n" + ); + const manifest = await writeReleaseManifest(manifestOptions(root)); + + expect(manifest.artifacts.map((artifact) => artifact.path)).toContain( + "systemd/mira-dashboard.service" + ); + expect(manifest.artifacts.map((artifact) => artifact.path)).toContain( + "systemd/mira-dashboard-worker.service" + ); + await verifyReleaseArtifacts(root, manifest); + + const partialRoot = temporaryReleaseRoot(); + mkdirSync(path.join(partialRoot, "systemd")); + writeFileSync( + path.join(partialRoot, "systemd", "mira-dashboard.service"), + "[Service]\nExecStart=/dashboard\n" + ); + expect(writeReleaseManifest(manifestOptions(partialRoot))).rejects.toThrow( + "Managed systemd release artifacts must be complete" ); }); diff --git a/backend/test/routeAndServiceBehavior.test.ts b/backend/test/routeAndServiceBehavior.test.ts index 0cf1ee1bd..bdb52de15 100644 --- a/backend/test/routeAndServiceBehavior.test.ts +++ b/backend/test/routeAndServiceBehavior.test.ts @@ -1349,6 +1349,46 @@ describe("backend route and service behavior", () => { await import("../src/routes/jobExecutionRoutes.ts"); const missingExecutionId = "018f47a2-9b7c-7cc8-a123-456789abcdef"; + const invalidClaimsPatch = await jobExecutionRoutes[ + "/api/job-executions/claims" + ].PATCH( + new Request("https://test.local/api/job-executions/claims", { + body: JSON.stringify({ paused: "yes" }), + headers: { "Content-Type": "application/json" }, + method: "PATCH", + }) + ); + expect(invalidClaimsPatch.status).toBe(400); + + const pausedClaims = await jobExecutionRoutes["/api/job-executions/claims"].PATCH( + new Request("https://test.local/api/job-executions/claims", { + body: JSON.stringify({ paused: true }), + headers: { "Content-Type": "application/json" }, + method: "PATCH", + }) + ); + expect(pausedClaims.status).toBe(200); + try { + expect(await pausedClaims.json()).toMatchObject({ + isOk: true, + state: { paused: true }, + }); + const pausedQueue = jobExecutionRoutes["/api/job-executions"].GET( + new Request("https://test.local/api/job-executions?include=claims") + ); + expect(await pausedQueue.json()).toMatchObject({ + summary: { claimsPaused: true }, + }); + } finally { + await jobExecutionRoutes["/api/job-executions/claims"].PATCH( + new Request("https://test.local/api/job-executions/claims", { + body: JSON.stringify({ paused: false }), + headers: { "Content-Type": "application/json" }, + method: "PATCH", + }) + ); + } + const missingExecution = jobExecutionRoutes["/api/job-executions/:id"].GET( requestWithParameters(`/api/job-executions/${missingExecutionId}`, { id: missingExecutionId, diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 6802fcf03..5d266a139 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1952,7 +1952,7 @@ describe("backend service behavior", () => { await createReleaseFixture( managedReleasePath(releasesRoot, currentCommit), currentCommit, - { commitTitle: "Schema 7 dashboard release" } + { commitTitle: "Schema 8 dashboard release" } ); const previousReleasePath = managedReleasePath(releasesRoot, previousCommit); await createReleaseFixture(previousReleasePath, previousCommit, { @@ -1979,7 +1979,7 @@ describe("backend service behavior", () => { expect(getDashboardReleaseStatus()).resolves.toMatchObject({ current: { commitSha: currentCommit, - schema: { maximumCompatible: 7, target: 7 }, + schema: { maximumCompatible: 8, target: 8 }, }, previous: { commitSha: previousCommit, @@ -1987,11 +1987,11 @@ describe("backend service behavior", () => { }, rollback: { available: false, - reason: "Rollback release cannot open SQLite schema 7", + reason: "Rollback release cannot open SQLite schema 8", }, }); expect(prepareAndStartRollback(previousCommit)).rejects.toThrow( - "Previous release is not eligible for rollback: Rollback release cannot open SQLite schema 7" + "Previous release is not eligible for rollback: Rollback release cannot open SQLite schema 8" ); const response = await pullRequestRoutes[ "/api/pull-requests/releases/rollback" @@ -1999,7 +1999,7 @@ describe("backend service behavior", () => { expect(response.status).toBe(409); expect(response.json()).resolves.toMatchObject( apiErrorExpectation( - "Previous release is not eligible for rollback: Rollback release cannot open SQLite schema 7" + "Previous release is not eligible for rollback: Rollback release cannot open SQLite schema 8" ) ); expect(countRollbackExecutions()).toBe(executionCountBefore); @@ -3153,7 +3153,7 @@ printf 'scheduled\n' .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") .get(schemaBlockedRedeploy.id) ).toEqual({ - note: "Automatic redeploy fallback is not eligible: Rollback release cannot open SQLite schema 7", + note: "Automatic redeploy fallback is not eligible: Rollback release cannot open SQLite schema 8", status: "failed", }); await executeSuccessfulGuardianPath(restartCommand); diff --git a/contracts/jobs.ts b/contracts/jobs.ts index dd1f5a9bb..a4446afd2 100644 --- a/contracts/jobs.ts +++ b/contracts/jobs.ts @@ -96,6 +96,8 @@ export const jobExecutionSchema = v.strictObject({ export const jobExecutionSummarySchema = v.strictObject({ activeResourceClasses: v.array(jobResourceClassSchema), + claimsPaused: v.optional(v.boolean()), + claimsPausedAt: v.optional(v.string()), oldestQueuedAgeMs: v.optional(finiteNumberSchema), oldestQueuedAt: v.optional(v.string()), queued: finiteNumberSchema, @@ -120,6 +122,20 @@ export const jobExecutionCancelResponseSchema = v.strictObject({ isOk: successLiteralSchema, }); +export const jobWorkerClaimsPatchSchema = strictJsonObjectSchema({ + paused: v.boolean(), +}); + +export const jobWorkerClaimsStateSchema = v.strictObject({ + paused: v.boolean(), + updatedAt: v.string(), +}); + +export const jobWorkerClaimsMutationResponseSchema = v.strictObject({ + isOk: successLiteralSchema, + state: jobWorkerClaimsStateSchema, +}); + export const scheduledJobRunSchema = v.strictObject({ cancelRequestedAt: v.optional(v.string()), cancellable: v.boolean(), @@ -204,6 +220,11 @@ export type JobExecutionResponse = v.InferOutput; +export type JobWorkerClaimsPatch = v.InferOutput; +export type JobWorkerClaimsState = v.InferOutput; +export type JobWorkerClaimsMutationResponse = v.InferOutput< + typeof jobWorkerClaimsMutationResponseSchema +>; export type ScheduledJobScheduleType = v.InferOutput< typeof scheduledJobScheduleTypeSchema >; @@ -278,6 +299,16 @@ export function parseJobExecutionCancelResponse( return parseContract(jobExecutionCancelResponseSchema, value, "response"); } +export function parseJobWorkerClaimsPatch(value: unknown): JobWorkerClaimsPatch { + return parseContract(jobWorkerClaimsPatchSchema, value); +} + +export function parseJobWorkerClaimsMutationResponse( + value: unknown +): JobWorkerClaimsMutationResponse { + return parseContract(jobWorkerClaimsMutationResponseSchema, value, "response"); +} + /** * Parses one scheduled run, including its bounded public output object. * @param value Value to process. diff --git a/contracts/moltbook.ts b/contracts/moltbook.ts index 2b7798520..49350080f 100644 --- a/contracts/moltbook.ts +++ b/contracts/moltbook.ts @@ -25,7 +25,7 @@ export const moltbookHomeSchema = v.strictObject({ }); const moltbookAuthorSchema = v.object({ - avatar_url: v.optional(v.string()), + avatar_url: v.optional(v.nullable(v.string())), display_name: v.optional(v.string()), name: trimmedNonEmptyStringSchema, }); @@ -63,7 +63,7 @@ export const moltbookFeedSchema = v.object({ }); export const moltbookProfileSchema = v.object({ - avatar_url: v.optional(v.string()), + avatar_url: v.optional(v.nullable(v.string())), comments_count: finiteNumberSchema, description: v.string(), display_name: v.string(), @@ -165,9 +165,14 @@ export function parseMoltbookFeed(value: unknown, path = "moltbookFeed"): Moltbo */ export function moltbookPostFromPayload(post: MoltbookFeedPostPayload): MoltbookPost { return { - author: post.author ?? { - name: post.author_name ?? "unknown", - }, + author: post.author + ? { + ...post.author, + avatar_url: post.author.avatar_url ?? undefined, + } + : { + name: post.author_name ?? "unknown", + }, comment_count: post.comment_count ?? 0, content: post.content ?? post.content_preview ?? "", created_at: post.created_at, diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 148a0b5c1..fd13e07bb 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -141,21 +141,22 @@ Create body: ## Jobs And Cron -| Method | Path | Purpose | -| ------- | -------------------------------- | ------------------------------------------------------------------------------------- | -| `GET` | `/api/jobs` | Lists Dashboard scheduled jobs. | -| `GET` | `/api/jobs/:id` | Reads a scheduled job. | -| `PATCH` | `/api/jobs/:id` | Updates scheduled job settings and intentional-disable metadata. | -| `POST` | `/api/jobs/:id/run` | Queues a scheduled job and returns `202`. | -| `GET` | `/api/job-executions` | Lists recent executions plus queue/worker summary. | -| `GET` | `/api/job-executions/:id` | Reads one execution, including its persisted progress/result output snapshot. | -| `POST` | `/api/job-executions/:id/cancel` | Cancels queued work or requests cooperative cancellation of a running execution. | -| `GET` | `/api/jobs/:id/runs` | Lists job run history. | -| `GET` | `/api/cron/jobs` | Lists OpenClaw cron jobs and open linked tasks. | -| `POST` | `/api/cron/jobs/:id/run` | Runs an OpenClaw cron job. | -| `POST` | `/api/cron/jobs/:id/toggle` | Enables/disables an OpenClaw cron job and updates its Dashboard-owned disable intent. | -| `POST` | `/api/cron/jobs/:id/update` | Updates an OpenClaw cron job patch. | -| `POST` | `/api/cron/jobs/:id/delete` | Deletes an OpenClaw cron job. | +| Method | Path | Purpose | +| ------- | -------------------------------- | -------------------------------------------------------------------------------------- | +| `GET` | `/api/jobs` | Lists Dashboard scheduled jobs. | +| `GET` | `/api/jobs/:id` | Reads a scheduled job. | +| `PATCH` | `/api/jobs/:id` | Updates scheduled job settings and intentional-disable metadata. | +| `POST` | `/api/jobs/:id/run` | Queues a scheduled job and returns `202`. | +| `GET` | `/api/job-executions` | Lists recent executions plus queue/worker summary; `?include=claims` adds pause state. | +| `PATCH` | `/api/job-executions/claims` | Pauses/resumes new worker claims; running work is not cancelled. | +| `GET` | `/api/job-executions/:id` | Reads one execution, including its persisted progress/result output snapshot. | +| `POST` | `/api/job-executions/:id/cancel` | Cancels queued work or requests cooperative cancellation of a running execution. | +| `GET` | `/api/jobs/:id/runs` | Lists job run history. | +| `GET` | `/api/cron/jobs` | Lists OpenClaw cron jobs and open linked tasks. | +| `POST` | `/api/cron/jobs/:id/run` | Runs an OpenClaw cron job. | +| `POST` | `/api/cron/jobs/:id/toggle` | Enables/disables an OpenClaw cron job and updates its Dashboard-owned disable intent. | +| `POST` | `/api/cron/jobs/:id/update` | Updates an OpenClaw cron job patch. | +| `POST` | `/api/cron/jobs/:id/delete` | Deletes an OpenClaw cron job. | When a Dashboard job or OpenClaw cron job is intentionally disabled, its update body may include `disableIntent: { mode, comment, until? }`. `mode` is `until` @@ -173,6 +174,10 @@ restart does not cancel the action. Poll the execution detail endpoint for its bounded progress/output snapshot, and use the explicit cancel endpoint when a queued or running action should stop. +The claims mutation requires recent MFA. Its `{ paused: boolean }` state is +durable across worker restarts. Paused executions remain queued, while an +already-running execution finishes cooperatively. + ## OpenClaw Config | Method | Path | Purpose | diff --git a/docs/architecture/database.md b/docs/architecture/database.md index 1de9be2c9..363054b9d 100644 --- a/docs/architecture/database.md +++ b/docs/architecture/database.md @@ -105,6 +105,7 @@ edit a released migration. Add the next numbered file instead. | `openclaw_cron_job_metadata` | Disable intent and Dashboard metadata for OpenClaw cron jobs. | Keyed operator intent; removed explicitly when intent is cleared. | | `job_executions` | Persistent execution queue with leases, heartbeats, and cancellation. | Terminal: 90 days/20,000 rows; queued/running rows retained. | | `job_workers` | Worker capacity and liveness heartbeats. | Stale heartbeats removed after 24 hours. | +| `job_worker_control` | Singleton operator pause state for new worker claims. | Updated only by the recent-MFA worker control. | | `chat_runtime_snapshots` | Durable OpenClaw chat replay/session snapshots. | Per-scope live cap plus global 30-day/200-row maintenance safety net. | | `chat_runtime_snapshot_events` | Ordered durable replay events for those snapshots. | Follows retained snapshots; orphan rows are removed. | | `docker_managed_services` | Docker updater managed service inventory. | Reconciled with current Compose inventory. | diff --git a/docs/operations/scheduler-cache-backups.md b/docs/operations/scheduler-cache-backups.md index fac732a8c..2654bf775 100644 --- a/docs/operations/scheduler-cache-backups.md +++ b/docs/operations/scheduler-cache-backups.md @@ -18,6 +18,7 @@ Dashboard-local scheduled jobs are stored in SQLite: | `scheduled_job_execution_policies` | Resource class and timeout per job. | | `job_executions` | Persistent queue, lease, heartbeat, and cancellation. | | `job_workers` | Worker capacity and liveness heartbeat. | +| `job_worker_control` | Singleton operator pause state for new execution claims. | | `openclaw_cron_job_metadata` | Dashboard-owned metadata for external OpenClaw cron jobs. | Supported schedule shapes: @@ -42,12 +43,20 @@ this avoids repeating backup, update, or other non-idempotent side effects after a worker crash. `GET /api/job-executions` exposes queue depth, oldest wait, resource classes, -and worker liveness. `GET /api/job-executions/:id` includes the bounded -persisted output snapshot used for stdout/stderr and incremental progress. +and worker liveness; `?include=claims` also includes the persistent pause state. +`GET /api/job-executions/:id` includes the bounded persisted output snapshot +used for stdout/stderr and incremental progress. HTTP waiters are observers only: disconnecting a request or restarting the web service never writes a cancellation request. Cancellation is explicit through `POST /api/job-executions/:id/cancel`. +The Jobs page can pause or resume new worker claims through +`PATCH /api/job-executions/claims`. Pausing is persistent across web/worker +restarts and leaves queued work queued; an execution that is already running is +allowed to finish. The claim transaction checks the singleton pause row under +the same SQLite writer lock used to select work, so a completed pause request +cannot race with a later claim. + Use the Jobs page to inspect definitions and run history before editing the database manually. @@ -123,6 +132,14 @@ External cache refreshes may require these env vars: If a page shows stale provider data, check the cache entry timestamp and the latest scheduled job run before debugging the frontend. +The **Cache refresh** observability card is different from cached provider +data. Its request/coalescing/failure/duration counters are runtime telemetry +since the current worker process started, not historical totals and not SQLite +rows. Production worker and web are separate processes, so the worker +atomically mirrors its in-memory counters to an owner-only snapshot below the +user's reboot-volatile `XDG_RUNTIME_DIR`; the web process samples that snapshot. +Worker restart resets every counter. + ### Status And Heartbeat Projections Dashboard exposes two intentionally different aggregate cache endpoints: diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index 9bfe7b67b..0bb4c17a5 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -247,11 +247,11 @@ Healthy response shape: { "checks": { "database": { - "currentSchemaVersion": 6, - "maximumCompatibleSchemaVersion": 6, + "currentSchemaVersion": 8, + "maximumCompatibleSchemaVersion": 8, "minimumCompatibleSchemaVersion": 6, "ready": true, - "targetSchemaVersion": 6 + "targetSchemaVersion": 8 }, "frontend": { "ready": true }, "release": { diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 40670f9c3..d034e7904 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -86,14 +86,17 @@ The Dashboard worker owns the deployment: 3. Install frozen frontend and backend dependencies. 4. Run `deploy:prepare` against the stable production database. 5. Verify the release manifest, component identities, schema contract, and - every checksummed artifact. + every checksummed artifact, including the complete web/worker systemd unit + bundle. 6. Copy only declared artifacts to a hidden directory and atomically publish it as `releases/`. 7. Persist a unique cutover-snapshot id and start a detached guardian. 8. Require the scheduling execution, snapshot id, deployment row, and release lock to be durably consistent. Then stop web and worker, create and - restore-verify the exact SQLite cutover snapshot, and atomically switch - `current` while retaining the old release as `previous`. + restore-verify the exact SQLite cutover snapshot, reconcile changed tracked + units into `~/.config/systemd/user`, run `systemctl --user daemon-reload`, + verify both loaded fragment paths, and atomically switch `current` while + retaining the old release as `previous`. 9. Start web and worker. Unsafe HTTP requests, explicit user-activity touches, Gateway WebSockets, and worker execution claims remain paused while the deployment row is `verifying`. @@ -132,11 +135,24 @@ remain available for restart and rollback. Interrupted `.retired-*` runtime cleanup is completed by the next prune, while missing runtimes for retained releases fail the operation before an old release is removed. -The release parser temporarily permits the obsolete `backend/package.json` and -`backend/bun.lock` artifacts so the single managed rollback slot created before -the root-package consolidation remains verifiable. Neither file is produced or -required by new releases. Remove `PRE_ROOT_WORKSPACE_RELEASE_ARTIFACTS` after -both `current` and `previous` were built from the consolidated root package. +Managed releases carry checksummed copies of +`systemd/mira-dashboard.service` and +`systemd/mira-dashboard-worker.service`. Activation, restore, and rollback +install a changed complete pair with mode `0644`, reload the user manager, and +verify that both units loaded from `~/.config/systemd/user`. If reconciliation +fails, the pre-operation files are restored before the release transition +returns an error. New VPS provisioning must still perform the initial install +and `enable --now`; ordinary deployments thereafter update the installed unit +definitions automatically. + +The first rollout of this unit-bundle contract is manual and supervised because +its rollback release predates the bundle and is deliberately not special-cased +in the permanent lifecycle code. Preserve the installed units and use the +previous release lifecycle during recovery if that first cutover fails. After +the first successful release, later activation targets carry the bundle, but +the immediately previous pre-bundle slot remains manual-only until a second +bundled release rotates it out. Automatic rollback is fully available again +once both managed slots contain verified bundles. ## Restart And Smoke Test @@ -224,9 +240,7 @@ test ! -L "$CURRENT_BUN" [[ "$(realpath --canonicalize-existing "$CURRENT_BUN")" == "$CURRENT_BUN" ]] [[ "$(stat --format='%h' -- "$CURRENT_BUN")" == "1" ]] CURRENT_BUN_REVISION="$("$CURRENT_BUN" --revision)" -CURRENT_BUN_VERSION="$("$CURRENT_BUN" --version)" -[[ "$CURRENT_BUN_REVISION" == "$CURRENT_BUN_ID" || - "$CURRENT_BUN_VERSION" == "$CURRENT_BUN_ID" ]] +[[ "$CURRENT_BUN_REVISION" == "$CURRENT_BUN_ID" ]] STATUS="$( env NODE_ENV=production \ "$CURRENT_BUN" "$CURRENT_LIFECYCLE" status diff --git a/frontend/src/components/features/jobs/JobExecutionQueueCard.tsx b/frontend/src/components/features/jobs/JobExecutionQueueCard.tsx index 83e17368b..ec8b557c9 100644 --- a/frontend/src/components/features/jobs/JobExecutionQueueCard.tsx +++ b/frontend/src/components/features/jobs/JobExecutionQueueCard.tsx @@ -1,8 +1,12 @@ -import { Activity, Clock3, Cpu, Layers3, XCircle } from "lucide-react"; +import { Activity, Clock3, Cpu, Layers3, Pause, Play, XCircle } from "lucide-react"; import type { ComponentProps } from "react"; import type { JobExecution } from "../../../../../contracts/jobs"; -import { useCancelJobExecution, useJobExecutions } from "../../../hooks"; +import { + useCancelJobExecution, + useJobExecutions, + useSetJobWorkerClaimsPaused, +} from "../../../hooks"; import { messageFromError } from "../../../lib/errorMessage"; import { cn } from "../../../utils/cn"; import { formatDate, formatDuration } from "../../../utils/format"; @@ -36,7 +40,9 @@ export function JobExecutionQueueCard({ }: JobExecutionQueueCardProperties = {}) { const queue = useJobExecutions(); const cancelExecution = useCancelJobExecution(); + const setClaimsPaused = useSetJobWorkerClaimsPaused(); const summary = queue.data?.summary; + const claimsPaused = summary?.claimsPaused === true; const activeExecutions = (queue.data?.executions ?? []).filter( (execution) => execution.status === "queued" || execution.status === "running" ); @@ -59,6 +65,9 @@ export function JobExecutionQueueCard({ if (queue.isLoading) { workerBadgeVariant = "default"; workerBadgeLabel = "Loading worker"; + } else if (claimsPaused) { + workerBadgeVariant = "warning"; + workerBadgeLabel = "Worker paused"; } else if (summary?.workerOnline && summary.running) { workerBadgeVariant = "info"; workerBadgeLabel = "Worker active"; @@ -66,6 +75,10 @@ export function JobExecutionQueueCard({ workerBadgeVariant = "success"; workerBadgeLabel = "Worker idle"; } + let claimsButtonLabel = claimsPaused ? "Resume worker" : "Pause worker"; + if (setClaimsPaused.isPending) { + claimsButtonLabel = "Saving..."; + } return ( @@ -76,7 +89,26 @@ export function JobExecutionQueueCard({ Persistent worker queue · global concurrency 1

- {workerBadgeLabel} +
+ {workerBadgeLabel} + +
{queue.error ? ( @@ -93,6 +125,20 @@ export function JobExecutionQueueCard({ )} ) : undefined} + {setClaimsPaused.error ? ( + + {messageFromError( + setClaimsPaused.error, + "Failed to update worker claim state" + )} + + ) : undefined} + {claimsPaused ? ( + + New executions remain queued. Any running execution is allowed to + finish. + + ) : undefined}
diff --git a/frontend/src/hooks/index.ts b/frontend/src/hooks/index.ts index 5acd3913f..bc50e14fa 100644 --- a/frontend/src/hooks/index.ts +++ b/frontend/src/hooks/index.ts @@ -84,6 +84,7 @@ export { jobExecutionKeys, useCancelJobExecution, useJobExecutions, + useSetJobWorkerClaimsPaused, } from "./useJobExecutions"; export { logKeys, useDashboardLogContent, useLogContent, useLogFiles } from "./useLogs"; export { useMetrics } from "./useMetrics"; diff --git a/frontend/src/hooks/useJobExecutions.ts b/frontend/src/hooks/useJobExecutions.ts index 60ebbb09b..98c4129bf 100644 --- a/frontend/src/hooks/useJobExecutions.ts +++ b/frontend/src/hooks/useJobExecutions.ts @@ -8,9 +8,10 @@ import { import { parseJobExecutionCancelResponse, parseJobExecutionsResponse, + parseJobWorkerClaimsMutationResponse, } from "../../../contracts/jobs"; import { refreshPolicy } from "../lib/refreshPolicy"; -import { apiFetchParsed, apiPostParsed } from "./useApi"; +import { apiFetchParsed, apiPatchParsed, apiPostParsed } from "./useApi"; export const jobExecutionKeys = { all: ["job-executions"] as const, @@ -43,7 +44,8 @@ export async function refreshJobExecutionQueueWhilePending( export function useJobExecutions() { return useQuery({ queryKey: jobExecutionKeys.list(), - queryFn: () => apiFetchParsed("/job-executions", parseJobExecutionsResponse), + queryFn: () => + apiFetchParsed("/job-executions?include=claims", parseJobExecutionsResponse), refetchInterval: refreshPolicy.active, refetchIntervalInBackground: false, staleTime: 500, @@ -69,3 +71,18 @@ export function useCancelJobExecution() { }, }); } + +export function useSetJobWorkerClaimsPaused() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (paused: boolean) => + apiPatchParsed( + "/job-executions/claims", + parseJobWorkerClaimsMutationResponse, + { paused } + ), + onSuccess: () => { + void queryClient.invalidateQueries({ queryKey: jobExecutionKeys.all }); + }, + }); +} diff --git a/frontend/src/test/componentBehavior.test.tsx b/frontend/src/test/componentBehavior.test.tsx index 1337ef0da..7d23f4d26 100644 --- a/frontend/src/test/componentBehavior.test.tsx +++ b/frontend/src/test/componentBehavior.test.tsx @@ -3417,6 +3417,7 @@ describe("shared component helpers", () => { }); it("shows queue pressure and cancels an active job execution", async () => { + let claimsPaused = false; let executionStatus: "cancelled" | "queued" = "queued"; const fetchMock = jest.fn((input: RequestInfo | URL, init?: RequestInit) => { return Promise.try(() => { @@ -3443,11 +3444,12 @@ describe("shared component helpers", () => { triggerType: "manual", }; - if (url === "/api/job-executions" && method === "GET") { + if (url === "/api/job-executions?include=claims" && method === "GET") { return Response.json({ executions: [execution], summary: { activeResourceClasses: [], + claimsPaused, oldestQueuedAt: executionStatus === "queued" ? execution.queuedAt @@ -3461,6 +3463,19 @@ describe("shared component helpers", () => { }); } + if (url === "/api/job-executions/claims" && method === "PATCH") { + claimsPaused = ( + JSON.parse(requestBodyText(init?.body)) as { paused: boolean } + ).paused; + return Response.json({ + isOk: true, + state: { + paused: claimsPaused, + updatedAt: "2026-07-30T08:00:00.000Z", + }, + }); + } + if ( url === "/api/job-executions/execution-1/cancel" && method === "POST" @@ -3493,6 +3508,26 @@ describe("shared component helpers", () => { expect(screen.getByText("Worker idle")).toBeInTheDocument(); expect(screen.getByText("Active class").querySelector("svg")).not.toBeNull(); + await userEvent.click( + screen.getByRole("button", { name: "Pause worker claims" }) + ); + await waitFor(() => { + expect(claimsPaused).toBe(true); + expect(screen.getByText("Worker paused")).toBeInTheDocument(); + expect( + screen.getByText( + "New executions remain queued. Any running execution is allowed to finish." + ) + ).toBeInTheDocument(); + }); + await userEvent.click( + screen.getByRole("button", { name: "Resume worker claims" }) + ); + await waitFor(() => { + expect(claimsPaused).toBe(false); + expect(screen.getByText("Worker idle")).toBeInTheDocument(); + }); + await userEvent.click(screen.getByRole("button", { name: "Cancel Host backup" })); await waitFor(() => { diff --git a/frontend/src/test/contracts.test.ts b/frontend/src/test/contracts.test.ts index 507a964cb..918ae825c 100644 --- a/frontend/src/test/contracts.test.ts +++ b/frontend/src/test/contracts.test.ts @@ -15,6 +15,7 @@ import { parseExecRequest } from "../../../contracts/exec"; import { parseFileContent, parseFilesResponse } from "../../../contracts/files"; import { parseJobExecutionsResponse, + parseJobWorkerClaimsPatch, parseScheduledJobsResponse, parseScheduledJobUpdateRequest, } from "../../../contracts/jobs"; @@ -22,6 +23,11 @@ import { parseLogRotationRunResult, parseLogRotationStatus, } from "../../../contracts/logRotation"; +import { + moltbookPostFromPayload, + parseMoltbookFeed, + parseMoltbookProfile, +} from "../../../contracts/moltbook"; import { parseNotificationsResponse } from "../../../contracts/notifications"; import { parseOpenClawConfig, @@ -43,6 +49,57 @@ function captureContractError(operation: () => unknown): ContractValidationError } describe("shared runtime contracts", () => { + it("accepts provider-null Moltbook avatars and normalizes feed display data", () => { + const feed = parseMoltbookFeed({ + hasMore: false, + posts: [ + { + author: { + avatar_url: null, + display_name: "Raymond", + name: "raymond", + }, + created_at: "2026-07-30T08:00:00.000Z", + id: "post-1", + submolt_name: "dashboard", + title: "Null avatar", + }, + ], + }); + expect(moltbookPostFromPayload(feed.posts[0]!)).toMatchObject({ + author: { + avatar_url: undefined, + display_name: "Raymond", + name: "raymond", + }, + id: "post-1", + }); + expect( + parseMoltbookProfile({ + avatar_url: null, + comments_count: 0, + description: "", + display_name: "Mira", + follower_count: 0, + following_count: 0, + karma: 0, + name: "mira", + posts_count: 0, + }).avatar_url + ).toBeNull(); + }); + + it("keeps the worker claims mutation body strict", () => { + expect(parseJobWorkerClaimsPatch({ paused: true })).toEqual({ + paused: true, + }); + expect( + captureContractError(() => + parseJobWorkerClaimsPatch({ pause: true }) + ).issues.map((issue) => issue.path) + ).toEqual(["body.paused", "body.pause"]); + }); + it("normalizes valid task input without losing intentional body whitespace", () => { expect( parseCreateTaskRequest({ diff --git a/frontend/src/test/frontendBehavior.test.tsx b/frontend/src/test/frontendBehavior.test.tsx index 1b181f125..c19a0e6d5 100644 --- a/frontend/src/test/frontendBehavior.test.tsx +++ b/frontend/src/test/frontendBehavior.test.tsx @@ -4075,7 +4075,7 @@ describe("Mira Dashboard frontend behavior", () => { author: { name: "raymond", display_name: "Raymond", - avatar_url: "/avatar.png", + avatar_url: null, }, created_at: "2026-06-23T08:30:00.000Z", submolt_name: "dashboard", @@ -4189,7 +4189,7 @@ describe("Mira Dashboard frontend behavior", () => { author: { name: "raymond", display_name: "Raymond", - avatar_url: "/avatar.png", + avatar_url: undefined, }, upvotes: 0, you_follow_author: true, diff --git a/frontend/src/test/pageBehavior.test.tsx b/frontend/src/test/pageBehavior.test.tsx index afa68b3ca..4208d5fba 100644 --- a/frontend/src/test/pageBehavior.test.tsx +++ b/frontend/src/test/pageBehavior.test.tsx @@ -1508,7 +1508,7 @@ function apiResponse(url: string, method: string, init?: RequestInit) { }); } - if (url === "/api/job-executions") { + if (url === "/api/job-executions?include=claims") { return Response.json({ executions: [], summary: { diff --git a/scripts/runManagedDashboardRelease.sh b/scripts/runManagedDashboardRelease.sh index 940423fe1..7ffd77f53 100755 --- a/scripts/runManagedDashboardRelease.sh +++ b/scripts/runManagedDashboardRelease.sh @@ -76,8 +76,7 @@ if [[ "$(/usr/bin/stat --format='%h' -- "$runtime_path")" != "1" ]]; then exit 78 fi runtime_revision="$("$runtime_path" --revision)" -runtime_version="$("$runtime_path" --version)" -if [[ "$runtime_revision" != "$bun_version" && "$runtime_version" != "$bun_version" ]]; then +if [[ "$runtime_revision" != "$bun_version" ]]; then echo "Managed Dashboard Bun runtime version does not match the release" >&2 exit 78 fi From 6c7ceb6ccc6a0f8779ea486553178bfc5c51ab57 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Thu, 30 Jul 2026 03:28:22 +0200 Subject: [PATCH 2/3] fix: complete deployment bootstrap hardening --- backend/src/lib/systemdProperties.ts | 20 + backend/src/managedBunRuntime.ts | 9 +- backend/src/managedDashboardSystemd.ts | 13 +- backend/src/releaseDeployment.ts | 13 +- backend/src/services/cacheRefreshMetrics.ts | 7 +- .../src/services/pullRequestPreviewHost.ts | 8 +- backend/src/services/pullRequests.ts | 2 +- backend/src/services/scheduledJobs.ts | 20 +- backend/test/databaseOverview.test.ts | 2 +- backend/test/jobExecutionQueue.test.ts | 24 ++ backend/test/managedBunRuntime.test.ts | 19 +- backend/test/managedDashboardSystemd.test.ts | 14 +- backend/test/productionBootstrap.test.ts | 304 ++++++++++++++ backend/test/releaseManager.test.ts | 32 +- backend/test/releaseManifest.test.ts | 9 +- backend/test/routeAndServiceBehavior.test.ts | 26 +- backend/test/support/rejections.ts | 15 + docs/operations/scheduler-cache-backups.md | 5 +- docs/setup/new-vps.md | 148 +++---- docs/setup/production-deploy.md | 6 +- frontend/src/test/componentBehavior.test.tsx | 27 +- frontend/src/test/pageBehavior.test.tsx | 2 + package.json | 1 + scripts/bootstrapProduction.sh | 43 ++ scripts/productionBootstrap.ts | 385 ++++++++++++++++++ scripts/runManagedDashboardRelease.sh | 2 +- 26 files changed, 970 insertions(+), 186 deletions(-) create mode 100644 backend/src/lib/systemdProperties.ts create mode 100644 backend/test/productionBootstrap.test.ts create mode 100644 backend/test/support/rejections.ts create mode 100755 scripts/bootstrapProduction.sh create mode 100644 scripts/productionBootstrap.ts diff --git a/backend/src/lib/systemdProperties.ts b/backend/src/lib/systemdProperties.ts new file mode 100644 index 000000000..97d4eedb7 --- /dev/null +++ b/backend/src/lib/systemdProperties.ts @@ -0,0 +1,20 @@ +/** + * Parses the newline-delimited `key=value` format emitted by `systemctl show`. + * Blank lines are ignored, missing separators produce an empty value, and + * additional separators remain part of the value. + * @param output Bounded `systemctl show` output. + * @returns Parsed systemd properties. + */ +export function parseSystemdProperties(output: string): Map { + return new Map( + output + .split("\n") + .filter(Boolean) + .map((line): [string, string] => { + const separator = line.indexOf("="); + return separator === -1 + ? [line, ""] + : [line.slice(0, separator), line.slice(separator + 1)]; + }) + ); +} diff --git a/backend/src/managedBunRuntime.ts b/backend/src/managedBunRuntime.ts index a44b0d164..0d6ae89bf 100644 --- a/backend/src/managedBunRuntime.ts +++ b/backend/src/managedBunRuntime.ts @@ -8,7 +8,7 @@ import { resolveDashboardProjectPaths } from "./lib/dashboardPaths.ts"; import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; const BUN_RUNTIME_VERSION_PATTERN = - /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*)|(?:\d*[A-Za-z-][\dA-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|(?:\d*[A-Za-z-][\dA-Za-z-]*)))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u; + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*)|(?:\d*[A-Za-z-][\dA-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|(?:\d*[A-Za-z-][\dA-Za-z-]*)))*)?\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*$/u; const BUN_RUNTIME_VERSION_MAX_LENGTH = 64; const RETIRED_RUNTIME_DIRECTORY_PATTERN = /^\.retired-[\da-f]{8}-[\da-f]{4}-7[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; @@ -26,7 +26,8 @@ export interface ManagedBunRuntimePruneResult { } /** - * Accepts only bounded, complete semantic versions that are safe as path segments. + * Accepts only bounded, revision-qualified semantic versions that are safe as + * path segments. * @param value Candidate Bun version. * @returns Whether the candidate is a strict Bun runtime version. */ @@ -40,7 +41,9 @@ export function isBunRuntimeVersion(value: string): boolean { function assertBunRuntimeVersion(value: string): string { if (!isBunRuntimeVersion(value)) { - throw new TypeError("Managed Bun runtime version must be valid semver"); + throw new TypeError( + "Managed Bun runtime version must be revision-qualified semver" + ); } return value; } diff --git a/backend/src/managedDashboardSystemd.ts b/backend/src/managedDashboardSystemd.ts index cf3cf655c..440d50e1c 100644 --- a/backend/src/managedDashboardSystemd.ts +++ b/backend/src/managedDashboardSystemd.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { guardedPath, writeTextNoFollowAnchoredGuarded } from "./lib/guardedOps.ts"; import { runProcess } from "./lib/processes.ts"; import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; +import { parseSystemdProperties } from "./lib/systemdProperties.ts"; import { MANAGED_DASHBOARD_UNIT_ARTIFACTS, MANAGED_DASHBOARD_UNIT_NAMES, @@ -136,17 +137,7 @@ async function reloadAndVerifyUnits( "--property=LoadState", "--no-pager", ]); - const properties = new Map( - result.stdout - .split("\n") - .filter(Boolean) - .map((line) => { - const separator = line.indexOf("="); - return separator === -1 - ? [line, ""] - : [line.slice(0, separator), line.slice(separator + 1)]; - }) - ); + const properties = parseSystemdProperties(result.stdout); if ( (properties.get("DropInPaths") ?? "") !== "" || properties.get("LoadState") !== "loaded" || diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index 8ab11fc53..f3626cdc2 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -6,6 +6,7 @@ import { writeCliError, writeCliOutput } from "./lib/cliOutput.ts"; import { resolveDashboardProjectPaths } from "./lib/dashboardPaths.ts"; import { runProcess } from "./lib/processes.ts"; import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; +import { parseSystemdProperties } from "./lib/systemdProperties.ts"; import { bunExecutableRuntimeIdentity, installManagedBunRuntime, @@ -278,17 +279,7 @@ export function assertManagedDashboardUnitProperties( `MIRA_DASHBOARD_PROJECT_ROOT=${contract.projectRoot}`, ]; const expectedWorkingDirectory = `${contract.releaseRoot}/backend`; - const actual = new Map( - properties - .split("\n") - .filter(Boolean) - .map((line) => { - const separator = line.indexOf("="); - return separator === -1 - ? [line, ""] - : [line.slice(0, separator), line.slice(separator + 1)]; - }) - ); + const actual = parseSystemdProperties(properties); if (actual.get("WorkingDirectory") !== expectedWorkingDirectory) { throw new Error( `${unit} must run from managed current/backend before Dashboard deployment` diff --git a/backend/src/services/cacheRefreshMetrics.ts b/backend/src/services/cacheRefreshMetrics.ts index f30a2c5dd..eaff37b4f 100644 --- a/backend/src/services/cacheRefreshMetrics.ts +++ b/backend/src/services/cacheRefreshMetrics.ts @@ -39,6 +39,7 @@ const emptyCacheRefreshMetrics = (): CacheRefreshMetrics => ({ let cacheRefreshMetricsState = emptyCacheRefreshMetrics(); let activeSession: | { + directoryValidated: boolean; instanceId: string; snapshotPath: string | undefined; startedAt: string; @@ -109,7 +110,6 @@ function writeSnapshot( snapshot: CacheRefreshMetricsSnapshot ): void { const directoryPath = path.dirname(snapshotPath); - ensurePrivateRuntimeDirectory(directoryPath); const temporaryPath = path.join( directoryPath, `.${SNAPSHOT_FILE_NAME}.${process.pid}.${Bun.randomUUIDv7()}.tmp` @@ -133,6 +133,10 @@ function writeSnapshot( function publishSnapshot(): void { if (!activeSession?.snapshotPath) return; + if (!activeSession.directoryValidated) { + ensurePrivateRuntimeDirectory(path.dirname(activeSession.snapshotPath)); + activeSession.directoryValidated = true; + } writeSnapshot(activeSession.snapshotPath, { instanceId: activeSession.instanceId, metrics: metricsSnapshot(), @@ -212,6 +216,7 @@ export function startCacheRefreshMetricsSession( if (activeSession) return; cacheRefreshMetricsState = emptyCacheRefreshMetrics(); activeSession = { + directoryValidated: false, instanceId: Bun.randomUUIDv7(), snapshotPath: options.snapshotPath ?? diff --git a/backend/src/services/pullRequestPreviewHost.ts b/backend/src/services/pullRequestPreviewHost.ts index f165710ce..aafd1f01e 100644 --- a/backend/src/services/pullRequestPreviewHost.ts +++ b/backend/src/services/pullRequestPreviewHost.ts @@ -33,6 +33,7 @@ import { resolveDashboardProjectPaths } from "../lib/dashboardPaths.ts"; import { errorMessage } from "../lib/errors.ts"; import { runProcess } from "../lib/processes.ts"; import { createStructuredLogger } from "../lib/structuredLogger.ts"; +import { parseSystemdProperties } from "../lib/systemdProperties.ts"; import { hasLineBreakOrNullByte } from "../lib/values.ts"; import { isPullRequestPreviewAuthorAllowed, @@ -1293,12 +1294,7 @@ async function startPreviewGatewayProxyUnit( * @returns Parsed the bounded systemctl property format used for preview status. */ export function parsePreviewUnitState(output: string): SystemdUnitState { - const properties = new Map(); - for (const line of output.split("\n")) { - const separator = line.indexOf("="); - if (separator <= 0) continue; - properties.set(line.slice(0, separator), line.slice(separator + 1)); - } + const properties = parseSystemdProperties(output); return { activeState: properties.get("ActiveState") || undefined, result: properties.get("Result") || undefined, diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index dbb3108d2..61f4af32e 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -2118,7 +2118,7 @@ function releaseCutoverShellFunctions(): string[] { "resolve_release_bun() {", ' release_root="$1"', ' bun_version="$(/usr/bin/jq --exit-status --raw-output \'.bunVersion | select(type == "string" and length > 0 and length <= 64)\' "$release_root/release-manifest.json")" || return 1', - String.raw` [[ "$bun_version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*))(\.((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*)))*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]] || return 1`, + String.raw` [[ "$bun_version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*))(\.((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*)))*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)$ ]] || return 1`, ' runtime_path="$project_root/production/runtimes/bun/$bun_version/bun"', ' [ -f "$runtime_path" ] && [ -x "$runtime_path" ] && [ ! -L "$runtime_path" ] || return 1', ' [ "$(/usr/bin/realpath --canonicalize-existing "$runtime_path")" = "$runtime_path" ] || return 1', diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index b5eb86d9e..de9a98d7e 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -13,6 +13,7 @@ import { errorMessage } from "../lib/errors.ts"; import { isJobResourceClass, withJobResourceClass } from "../lib/jobResources.ts"; import { runWithLogContext } from "../lib/logContext.ts"; import { createStructuredLogger } from "../lib/structuredLogger.ts"; +import { parseSystemdProperties } from "../lib/systemdProperties.ts"; import { parseJobDisableIntent } from "./jobDisableIntent.ts"; import { claimNextJobExecution, @@ -1286,17 +1287,8 @@ function readSystemdUnitState(unit: string): SystemdUnitState { unit, }); } - const properties = new Map( - new TextDecoder() - .decode(result.stdout) - .trim() - .split("\n") - .map((line) => { - const separator = line.indexOf("="); - return separator === -1 - ? [line, ""] - : [line.slice(0, separator), line.slice(separator + 1)]; - }) + const properties = parseSystemdProperties( + new TextDecoder().decode(result.stdout).trim() ); if (properties.get("LoadState") === "not-found") { return "missing"; @@ -1512,15 +1504,15 @@ function executorTick(): void { } scheduledJobRuntimeState.isExecutorTickRunning = true; try { - if (getJobWorkerClaimsState().paused) { - return; - } // The in-memory pause is lost when the deployment restarts this worker. // Keep replacement workers idle until the detached guardian records a // terminal deployment status in the shared database. if (hasPendingDeploymentCutover()) { return; } + if (getJobWorkerClaimsState().paused) { + return; + } const execution = claimNextJobExecution( scheduledJobRuntimeState.workerId, executorCapacity diff --git a/backend/test/databaseOverview.test.ts b/backend/test/databaseOverview.test.ts index 2d8446946..1896f7cf3 100644 --- a/backend/test/databaseOverview.test.ts +++ b/backend/test/databaseOverview.test.ts @@ -237,7 +237,7 @@ describe("database overview service", () => { backup: { count: 0, current: false, reviewAgeHours: 48 }, foreignKeysEnabled: true, journalMode: "wal", - migrations: { applied: 7, current: true, latest: 7 }, + migrations: { applied: 8, current: true, latest: 8 }, permissions: { secure: true }, status: "review", walAutoCheckpointPages: 1000, diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index da72b32ef..4ee543e18 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -634,6 +634,30 @@ printf 'LoadState=loaded\nActiveState=active\n' finishJobExecution(queued.id, workerId, "success", undefined, {}); }); + it("reconciles orphaned deployment cutovers while new claims are paused", () => { + const deploymentId = createVerifyingDeployment( + "2026-07-26T03:00:00.000Z", + "c0ffee12" + ); + setJobWorkerClaimsPaused(true, "2026-07-30T08:00:00.000Z"); + + startScheduledJobExecutor(); + + expect( + database + .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") + .get(deploymentId) + ).toEqual({ + note: "Interrupted deployment cutover cannot be recovered because it lacks a persisted full candidate SHA", + status: "failed", + }); + expect( + database + .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") + .get(deploymentId) + ).toBeNull(); + }); + it("prioritizes interactive work and enforces global capacity", () => { const heavyJobId = createScheduledTestJob("host-heavy", "Heavy test job"); const interactiveJobId = createScheduledTestJob( diff --git a/backend/test/managedBunRuntime.test.ts b/backend/test/managedBunRuntime.test.ts index efe75db8f..043e78236 100644 --- a/backend/test/managedBunRuntime.test.ts +++ b/backend/test/managedBunRuntime.test.ts @@ -60,10 +60,11 @@ afterEach(() => { }); describe("managed Bun runtimes", () => { - it("accepts bounded semver versions across Bun majors", () => { - expect(isBunRuntimeVersion("1.3.14")).toBe(true); - expect(isBunRuntimeVersion("2.0.0")).toBe(true); + it("accepts only bounded revision-qualified identities across Bun majors", () => { + expect(isBunRuntimeVersion("1.3.14+0d9b296af")).toBe(true); expect(isBunRuntimeVersion("2.0.0-canary.1+build.2")).toBe(true); + expect(isBunRuntimeVersion("1.3.14")).toBe(false); + expect(isBunRuntimeVersion("2.0.0")).toBe(false); expect(isBunRuntimeVersion("1.3")).toBe(false); expect(isBunRuntimeVersion("1.3.14foo")).toBe(false); expect(isBunRuntimeVersion("01.3.14")).toBe(false); @@ -112,7 +113,11 @@ describe("managed Bun runtimes", () => { const root = temporaryRoot("mira-managed-bun-invalid"); const runtimeRoot = path.join(root, "runtimes"); const source = path.join(root, "bun-source"); + const versionOnlySource = path.join(root, "bun-version-only"); writeFakeBun(source, "2.0.0", "2.0.0+feedface"); + writeFakeBun(versionOnlySource, "2.0.0", "2.0.0"); + + expect(bunExecutableRuntimeIdentity(versionOnlySource)).toBeUndefined(); const installError = await installManagedBunRuntime(source, "1.3.14", { runtimeRoot, @@ -122,13 +127,13 @@ describe("managed Bun runtimes", () => { ); expect(installError).toBeInstanceOf(Error); expect((installError as Error).message).toContain( - "does not report expected version 1.3.14" + "must be revision-qualified semver" ); expect(() => requireManagedBunRuntime("2.0.0", runtimeRoot)).toThrow( - "Managed Bun runtime 2.0.0 is not available" + "must be revision-qualified semver" ); expect(() => managedBunRuntimeExecutablePath("../2.0.0", runtimeRoot)).toThrow( - "must be valid semver" + "must be revision-qualified semver" ); }); @@ -280,7 +285,7 @@ describe("managed Bun runtimes", () => { }); expect(versionOnly.exitCode).toBe(78); expect(new TextDecoder().decode(versionOnly.stderr)).toContain( - "runtime version does not match" + "release manifest has no valid Bun runtime" ); const rejected = Bun.spawnSync({ diff --git a/backend/test/managedDashboardSystemd.test.ts b/backend/test/managedDashboardSystemd.test.ts index e81be77a1..a129f3bea 100644 --- a/backend/test/managedDashboardSystemd.test.ts +++ b/backend/test/managedDashboardSystemd.test.ts @@ -18,6 +18,7 @@ import { } from "../src/managedDashboardSystemd.ts"; import { MANAGED_DASHBOARD_UNIT_NAMES } from "../src/managedDashboardUnitPolicy.ts"; import { loadManagedRelease, managedReleasePath } from "../src/releaseManager.ts"; +import { captureRejection } from "./support/rejections.ts"; import { createReleaseFixture } from "./support/releaseFixture.ts"; const COMMIT_SHA = "a".repeat(40); @@ -45,15 +46,6 @@ async function managedReleaseWithUnits(releasesRoot: string) { return loadManagedRelease(releasesRoot, COMMIT_SHA); } -async function captureRejection(operation: () => Promise): Promise { - try { - await operation(); - } catch (error) { - return error; - } - throw new Error("Expected operation to reject"); -} - afterEach(() => { for (const root of temporaryRoots.splice(0)) { rmSync(root, { force: true, recursive: true }); @@ -96,7 +88,7 @@ describe("managed Dashboard systemd reconciliation", () => { ); expect(statSync(path.join(unitRoot, unit)).mode & 0o777).toBe(0o644); } - expect(calls).toHaveLength(3); + expect(calls).toHaveLength(MANAGED_DASHBOARD_UNIT_NAMES.length + 1); calls.length = 0; expect( @@ -104,7 +96,7 @@ describe("managed Dashboard systemd reconciliation", () => { ).toMatchObject({ changed: [], }); - expect(calls).toHaveLength(3); + expect(calls).toHaveLength(MANAGED_DASHBOARD_UNIT_NAMES.length + 1); }); it("repairs unit modes even when the managed content already matches", async () => { diff --git a/backend/test/productionBootstrap.test.ts b/backend/test/productionBootstrap.test.ts new file mode 100644 index 000000000..29b42b398 --- /dev/null +++ b/backend/test/productionBootstrap.test.ts @@ -0,0 +1,304 @@ +import { afterEach, describe, expect, it, jest } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + bootstrapProductionDashboard, + initializeProductionBootstrapDatabase, + type ProductionBootstrapCommandRunner, + runProductionBootstrapCommand, +} from "../../scripts/productionBootstrap.ts"; +import { dashboardProjectPaths } from "../src/lib/dashboardPaths.ts"; +import { MANAGED_DASHBOARD_UNIT_NAMES } from "../src/managedDashboardUnitPolicy.ts"; +import { captureRejection } from "./support/rejections.ts"; + +const COMMIT_SHA = "a".repeat(40); +const OTHER_COMMIT_SHA = "b".repeat(40); +const temporaryRoots: string[] = []; + +function temporaryProjectRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), "mira-production-bootstrap-")); + temporaryRoots.push(root); + const paths = dashboardProjectPaths(root); + mkdirSync(paths.productionCheckoutRoot, { recursive: true }); + return root; +} + +function commandRunner( + sourceRoot: string, + calls: string[], + statusOutput = "" +): ProductionBootstrapCommandRunner { + return (command, arguments_, options) => { + calls.push(`${command} ${arguments_.join(" ")}`); + if (command === "git" && arguments_[1] === "--show-toplevel") { + return Promise.resolve({ stderr: "", stdout: `${sourceRoot}\n` }); + } + if (command === "git" && arguments_[0] === "status") { + return Promise.resolve({ stderr: "", stdout: statusOutput }); + } + if (command === "git" && arguments_[1] === "--verify") { + return Promise.resolve({ stderr: "", stdout: `${COMMIT_SHA}\n` }); + } + if (command === "/usr/bin/systemctl" && arguments_[1] === "is-enabled") { + return Promise.resolve({ stderr: "", stdout: "enabled\n" }); + } + if (command === "/usr/bin/systemctl" && arguments_[1] === "show") { + return Promise.resolve({ + stderr: "", + stdout: "ActiveState=active\nResult=success\nSubState=running\n", + }); + } + expect(options.cwd).toBeUndefined(); + return Promise.resolve({ stderr: "", stdout: "" }); + }; +} + +afterEach(() => { + for (const root of temporaryRoots) { + rmSync(root, { force: true, recursive: true }); + } + temporaryRoots.length = 0; +}); + +describe("production bootstrap", () => { + it("runs bounded child commands and initializes the production database", async () => { + expect( + await runProductionBootstrapCommand("/usr/bin/printf", ["ready"], { + timeoutMs: 5000, + }) + ).toEqual({ stderr: "", stdout: "ready" }); + const commandError = await captureRejection(() => + runProductionBootstrapCommand("/usr/bin/false", [], { + timeoutMs: 5000, + }) + ); + expect(commandError).toBeInstanceOf(Error); + expect((commandError as Error).message).toContain( + "/usr/bin/false failed with exit code 1" + ); + + await initializeProductionBootstrapDatabase(); + }); + + it("initializes, stages, activates, enables, and verifies a blank host", async () => { + const root = temporaryProjectRoot(); + const paths = dashboardProjectPaths(root); + const calls: string[] = []; + const progress: string[] = []; + const lifecycle: string[] = []; + let databaseInitialized = false; + + const result = await bootstrapProductionDashboard({ + activateRelease: (commitSha) => { + expect(commitSha).toBe(COMMIT_SHA); + lifecycle.push("activate"); + return Promise.resolve(); + }, + commandRunner: commandRunner(paths.productionCheckoutRoot, calls), + environment: { + MIRA_DASHBOARD_PROJECT_ROOT: root, + NODE_ENV: "production", + }, + initializeDatabase: () => { + databaseInitialized = true; + lifecycle.push("database"); + return Promise.resolve(); + }, + onProgress: (message) => { + progress.push(message); + }, + paths, + readReleaseSlots: () => Promise.resolve({}), + serviceStabilizationMs: 0, + stageRelease: (commitSha) => { + expect(databaseInitialized).toBe(true); + expect(commitSha).toBe(COMMIT_SHA); + lifecycle.push("stage"); + return Promise.resolve({ + commitSha, + path: path.join(paths.productionReleasesRoot, "releases", commitSha), + }); + }, + }); + + expect(result).toMatchObject({ + commitSha: COMMIT_SHA, + databasePath: paths.productionDatabasePath, + services: MANAGED_DASHBOARD_UNIT_NAMES.map((name) => ({ + activeState: "active", + enabled: true, + name, + subState: "running", + })), + }); + expect(lifecycle).toEqual(["database", "stage", "activate"]); + expect(progress).toEqual([ + "Preparing managed production directories", + "Verifying clean production checkout", + "Initializing and verifying production SQLite", + "Staging the initial managed release", + "Activating release and reconciling managed systemd units", + "Enabling and starting Dashboard services", + "Dashboard production bootstrap completed", + ]); + expect(calls).toContain( + `/usr/bin/systemctl --user enable --now ${MANAGED_DASHBOARD_UNIT_NAMES.join( + " " + )}` + ); + for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { + expect(calls).toContain(`/usr/bin/systemctl --user is-enabled ${unit}`); + expect(calls).toContain( + `/usr/bin/systemctl --user show ${unit} --property=ActiveState --property=Result --property=SubState --no-pager` + ); + } + for (const directory of [ + paths.developmentWorktreeRoot, + paths.productionReleasesRoot, + paths.productionBunRuntimeRoot, + ]) { + expect(statSync(directory).mode & 0o777).toBe(0o755); + } + for (const directory of [ + paths.productionStateRoot, + paths.productionOpenClawHome, + ]) { + expect(statSync(directory).mode & 0o777).toBe(0o700); + } + }); + + it("rejects non-production mode and a dirty checkout", async () => { + const root = temporaryProjectRoot(); + const paths = dashboardProjectPaths(root); + + const environmentError = await captureRejection(() => + bootstrapProductionDashboard({ + environment: { NODE_ENV: "development" }, + paths, + }) + ); + expect(environmentError).toBeInstanceOf(Error); + expect((environmentError as Error).message).toContain( + "requires NODE_ENV=production" + ); + + const dirtyCheckoutError = await captureRejection(() => + bootstrapProductionDashboard({ + commandRunner: commandRunner( + paths.productionCheckoutRoot, + [], + " M package.json\n" + ), + environment: { NODE_ENV: "production" }, + paths, + }) + ); + expect(dirtyCheckoutError).toBeInstanceOf(Error); + expect((dirtyCheckoutError as Error).message).toContain( + "requires a clean production checkout" + ); + }); + + it("refuses to replace an already managed different release", async () => { + const root = temporaryProjectRoot(); + const paths = dashboardProjectPaths(root); + const initializeDatabase = jest.fn(() => Promise.resolve()); + const stageRelease = jest.fn(() => + Promise.resolve({ commitSha: COMMIT_SHA, path: "unexpected" }) + ); + + const existingReleaseError = await captureRejection(() => + bootstrapProductionDashboard({ + commandRunner: commandRunner(paths.productionCheckoutRoot, []), + environment: { NODE_ENV: "production" }, + initializeDatabase, + paths, + readReleaseSlots: () => Promise.resolve({ current: OTHER_COMMIT_SHA }), + stageRelease, + }) + ); + expect(existingReleaseError).toBeInstanceOf(Error); + expect((existingReleaseError as Error).message).toContain( + "use the normal deployment path" + ); + expect(initializeDatabase).not.toHaveBeenCalled(); + expect(stageRelease).not.toHaveBeenCalled(); + }); + + it("fails closed for invalid slots, staged identity, and service state", async () => { + const invalidDelayError = await captureRejection(() => + bootstrapProductionDashboard({ + environment: { NODE_ENV: "production" }, + paths: dashboardProjectPaths(temporaryProjectRoot()), + serviceStabilizationMs: -1, + }) + ); + expect(invalidDelayError).toBeInstanceOf(RangeError); + + const slotRoot = temporaryProjectRoot(); + const slotPaths = dashboardProjectPaths(slotRoot); + const invalidSlotsError = await captureRejection(() => + bootstrapProductionDashboard({ + commandRunner: commandRunner(slotPaths.productionCheckoutRoot, []), + environment: { NODE_ENV: "production" }, + paths: slotPaths, + readReleaseSlots: () => Promise.resolve({ previous: OTHER_COMMIT_SHA }), + }) + ); + expect((invalidSlotsError as Error).message).toContain( + "previous release without current" + ); + + const stagedRoot = temporaryProjectRoot(); + const stagedPaths = dashboardProjectPaths(stagedRoot); + const stagedIdentityError = await captureRejection(() => + bootstrapProductionDashboard({ + commandRunner: commandRunner(stagedPaths.productionCheckoutRoot, []), + environment: { NODE_ENV: "production" }, + initializeDatabase: () => Promise.resolve(), + paths: stagedPaths, + readReleaseSlots: () => Promise.resolve({}), + stageRelease: () => + Promise.resolve({ + commitSha: OTHER_COMMIT_SHA, + path: "unexpected", + }), + }) + ); + expect((stagedIdentityError as Error).message).toContain( + "staged an unexpected release" + ); + + const serviceRoot = temporaryProjectRoot(); + const servicePaths = dashboardProjectPaths(serviceRoot); + const baseRunner = commandRunner(servicePaths.productionCheckoutRoot, []); + const disabledServiceRunner: ProductionBootstrapCommandRunner = ( + command, + arguments_, + options + ) => { + if (command === "/usr/bin/systemctl" && arguments_[1] === "is-enabled") { + return Promise.resolve({ stderr: "", stdout: "disabled\n" }); + } + return baseRunner(command, arguments_, options); + }; + const disabledServiceError = await captureRejection(() => + bootstrapProductionDashboard({ + activateRelease: () => Promise.resolve(), + commandRunner: disabledServiceRunner, + environment: { NODE_ENV: "production" }, + initializeDatabase: () => Promise.resolve(), + paths: servicePaths, + readReleaseSlots: () => Promise.resolve({}), + serviceStabilizationMs: 0, + stageRelease: (commitSha) => + Promise.resolve({ commitSha, path: "release" }), + }) + ); + expect((disabledServiceError as Error).message).toContain( + "was not persistently enabled" + ); + }); +}); diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index c7f39bfa9..aeabdf5cc 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -52,6 +52,7 @@ import { RELEASE_MANIFEST_FILE_NAME, writeReleaseManifest, } from "../src/releaseManifest.ts"; +import { captureRejection } from "./support/rejections.ts"; import { createReleaseFixture } from "./support/releaseFixture.ts"; const temporaryRoots: string[] = []; @@ -703,24 +704,29 @@ describe("Dashboard immutable release manager", () => { expect(restoredRegistryState.previous).toBeUndefined(); const runtimeRoot = temporaryReleasesRoot(); - await createManagedRelease(runtimeRoot, FIRST_COMMIT, FIRST_COMMIT, "0.0.0"); + await createManagedRelease( + runtimeRoot, + FIRST_COMMIT, + FIRST_COMMIT, + "0.0.0+missing" + ); const incompatibleRelease = await loadManagedRelease(runtimeRoot, FIRST_COMMIT); expect(() => assertDashboardReleaseRuntimeAvailable(incompatibleRelease)).toThrow( - "requires unavailable managed Bun runtime 0.0.0" + "requires unavailable managed Bun runtime 0.0.0+missing" ); expect( activateDashboardRelease(FIRST_COMMIT, runtimeRoot, { ...SCHEMA_6_OPTIONS, hasRuntime: () => false, }) - ).rejects.toThrow("requires unavailable managed Bun runtime 0.0.0"); + ).rejects.toThrow("requires unavailable managed Bun runtime 0.0.0+missing"); const cachedMajorRuntimeRoot = temporaryReleasesRoot(); await createManagedRelease( cachedMajorRuntimeRoot, FIRST_COMMIT, FIRST_COMMIT, - "1.3.14" + "1.3.14+cached" ); const cachedMajorRuntimeRelease = await loadManagedRelease( cachedMajorRuntimeRoot, @@ -728,14 +734,14 @@ describe("Dashboard immutable release manager", () => { ); expect(() => assertDashboardReleaseRuntimeAvailable(cachedMajorRuntimeRelease, { - hasRuntime: (version) => version === "1.3.14", + hasRuntime: (version) => version === "1.3.14+cached", }) ).not.toThrow(); expect(() => assertDashboardReleaseRuntimeAvailable(cachedMajorRuntimeRelease, { hasRuntime: () => false, }) - ).toThrow("requires unavailable managed Bun runtime 1.3.14"); + ).toThrow("requires unavailable managed Bun runtime 1.3.14+cached"); }); it("checks the effective live schema after a code-only rollback", async () => { @@ -977,10 +983,14 @@ describe("Dashboard immutable release manager", () => { let preparationCalls = 0; try { - expect(readDashboardReleaseState(root)).rejects.toThrow( + const statusError = await captureRejection(() => + readDashboardReleaseState(root) + ); + expect(statusError).toBeInstanceOf(Error); + expect((statusError as Error).message).toContain( "Another managed release transition is in progress" ); - expect( + const activationError = await captureRejection(() => activateDashboardRelease(SECOND_COMMIT, root, { ...SCHEMA_6_OPTIONS, prepareReleaseTransition: () => { @@ -990,7 +1000,11 @@ describe("Dashboard immutable release manager", () => { }); }, }) - ).rejects.toThrow("Another managed release transition is in progress"); + ); + expect(activationError).toBeInstanceOf(Error); + expect((activationError as Error).message).toContain( + "Another managed release transition is in progress" + ); expect(preparationCalls).toBe(0); expect(readlinkSync(path.join(root, "current"))).toBe( `releases/${FIRST_COMMIT}` diff --git a/backend/test/releaseManifest.test.ts b/backend/test/releaseManifest.test.ts index 0a9a98a96..ce178b179 100644 --- a/backend/test/releaseManifest.test.ts +++ b/backend/test/releaseManifest.test.ts @@ -28,6 +28,7 @@ import { verifyReleaseArtifacts, writeReleaseManifest, } from "../src/releaseManifest.ts"; +import { captureRejection } from "./support/rejections.ts"; const temporaryRoots: string[] = []; const TEST_COMMIT = "a".repeat(40); @@ -314,7 +315,11 @@ describe("Dashboard release manifest", () => { path.join(partialRoot, "systemd", "mira-dashboard.service"), "[Service]\nExecStart=/dashboard\n" ); - expect(writeReleaseManifest(manifestOptions(partialRoot))).rejects.toThrow( + const partialBundleError = await captureRejection(() => + writeReleaseManifest(manifestOptions(partialRoot)) + ); + expect(partialBundleError).toBeInstanceOf(Error); + expect((partialBundleError as Error).message).toContain( "Managed systemd release artifacts must be complete" ); }); @@ -574,7 +579,7 @@ describe("Dashboard release manifest", () => { `${JSON.stringify( { ...manifest, - bunVersion: "0.0.0", + bunVersion: "0.0.0+other", }, undefined, 2 diff --git a/backend/test/routeAndServiceBehavior.test.ts b/backend/test/routeAndServiceBehavior.test.ts index bdb52de15..d806dd723 100644 --- a/backend/test/routeAndServiceBehavior.test.ts +++ b/backend/test/routeAndServiceBehavior.test.ts @@ -1360,15 +1360,17 @@ describe("backend route and service behavior", () => { ); expect(invalidClaimsPatch.status).toBe(400); - const pausedClaims = await jobExecutionRoutes["/api/job-executions/claims"].PATCH( - new Request("https://test.local/api/job-executions/claims", { - body: JSON.stringify({ paused: true }), - headers: { "Content-Type": "application/json" }, - method: "PATCH", - }) - ); - expect(pausedClaims.status).toBe(200); try { + const pausedClaims = await jobExecutionRoutes[ + "/api/job-executions/claims" + ].PATCH( + new Request("https://test.local/api/job-executions/claims", { + body: JSON.stringify({ paused: true }), + headers: { "Content-Type": "application/json" }, + method: "PATCH", + }) + ); + expect(pausedClaims.status).toBe(200); expect(await pausedClaims.json()).toMatchObject({ isOk: true, state: { paused: true }, @@ -1380,13 +1382,19 @@ describe("backend route and service behavior", () => { summary: { claimsPaused: true }, }); } finally { - await jobExecutionRoutes["/api/job-executions/claims"].PATCH( + const resumedClaims = await jobExecutionRoutes[ + "/api/job-executions/claims" + ].PATCH( new Request("https://test.local/api/job-executions/claims", { body: JSON.stringify({ paused: false }), headers: { "Content-Type": "application/json" }, method: "PATCH", }) ); + expect(resumedClaims.status).toBe(200); + expect(await resumedClaims.json()).toMatchObject({ + state: { paused: false }, + }); } const missingExecution = jobExecutionRoutes["/api/job-executions/:id"].GET( diff --git a/backend/test/support/rejections.ts b/backend/test/support/rejections.ts new file mode 100644 index 000000000..55b0e081c --- /dev/null +++ b/backend/test/support/rejections.ts @@ -0,0 +1,15 @@ +/** + * Awaits an operation and returns its rejection for explicit assertions. + * @param operation Asynchronous operation expected to reject. + * @returns Captured rejection. + */ +export async function captureRejection( + operation: () => Promise +): Promise { + try { + await operation(); + } catch (error) { + return error; + } + throw new Error("Expected operation to reject"); +} diff --git a/docs/operations/scheduler-cache-backups.md b/docs/operations/scheduler-cache-backups.md index 2654bf775..89894b911 100644 --- a/docs/operations/scheduler-cache-backups.md +++ b/docs/operations/scheduler-cache-backups.md @@ -138,7 +138,10 @@ since the current worker process started, not historical totals and not SQLite rows. Production worker and web are separate processes, so the worker atomically mirrors its in-memory counters to an owner-only snapshot below the user's reboot-volatile `XDG_RUNTIME_DIR`; the web process samples that snapshot. -Worker restart resets every counter. +The managed processes are systemd user services, so their user manager supplies +`XDG_RUNTIME_DIR`; new-host bootstrap enables linger so `/run/user/` is +also created for boot-time starts without an interactive login. Worker restart +resets every counter. ### Status And Heartbeat Projections diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index 0bb4c17a5..ffab3f022 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -38,87 +38,19 @@ git clone https://github.com/rajohan/Mira-Dashboard.git \ cd /home/ubuntu/projects/mira-dashboard/production/checkout ``` -Select the repository runtime channel, then install dependencies: +Select the repository runtime channel: ```bash bun upgrade --canary bun --revision -bun install --frozen-lockfile ``` -Create the managed runtime roots: - -```bash -install -d -m 0755 \ - /home/ubuntu/projects/mira-dashboard/development/worktrees \ - /home/ubuntu/projects/mira-dashboard/production/releases \ - /home/ubuntu/projects/mira-dashboard/production/runtimes -install -d -m 0700 \ - /home/ubuntu/projects/mira-dashboard/development/state \ - /home/ubuntu/projects/mira-dashboard/production/state -``` - -## Publish The Initial Managed Release - -Build, preflight, checksum, and publish the checked-out commit from an isolated -worktree: - -```bash -export MIRA_DASHBOARD_PROJECT_ROOT=/home/ubuntu/projects/mira-dashboard -cd "$MIRA_DASHBOARD_PROJECT_ROOT/production/checkout" -CANDIDATE_SHA="$(git rev-parse HEAD)" - -# A new host has no live database to preflight yet. Initialize and migrate the -# empty state database once from this exact checked-out candidate. -( - cd backend - env \ - NODE_ENV=production \ - bun -e ' - const { database } = await import("./src/database.ts"); - try { - const result = database.query("PRAGMA quick_check").get(); - if (!result || Object.values(result)[0] !== "ok") { - throw new Error("Fresh Dashboard database failed quick_check"); - } - } finally { - database.close(); - } - ' -) - -env \ - NODE_ENV=production \ - bun backend/src/releaseDeployment.ts stage "$CANDIDATE_SHA" -``` - -Activate it before installing/starting the managed systemd units: - -```bash -export MIRA_DASHBOARD_PROJECT_ROOT=/home/ubuntu/projects/mira-dashboard -cd "$MIRA_DASHBOARD_PROJECT_ROOT/production/checkout" -CANDIDATE_SHA="$(git rev-parse HEAD)" -env \ - NODE_ENV=production \ - bun backend/src/releaseLifecycle.ts activate "$CANDIDATE_SHA" -``` - -The one-shot initialization creates the fresh database in WAL mode and applies -the immutable migration registry. The staging command then installs frozen -dependencies, runs the normal database-aware `deploy:prepare`, and atomically -publishes only manifest-declared artifacts. It also copies the exact verified -Bun executable into `production/runtimes/bun//bun`; the managed units -select that runtime from the active release manifest, including across future -major-version upgrades and rollback. -The control checkout is not a production runtime directory. See -[Production deploy](production-deploy.md) for the release/state layout, -automatic rollback, retention, and recovery contract. - ## Configure Secrets Dashboard reads production secrets through Doppler project/config -`rajohan/prd`. Do not start `serverStart.js` manually to test them; the managed -systemd units below own the only production web and worker processes. +`rajohan/prd`. Configure and verify them before running the production +bootstrap because that command starts both managed services. Do not start +`serverStart.js` manually to test them. See [Secrets and environment](secrets-and-env.md) for the full list. The minimum production setup normally needs: @@ -129,12 +61,50 @@ minimum production setup normally needs: - one stable HTTPS hostname configured through `MIRA_DASHBOARD_WEBAUTHN_RP_ID` and `MIRA_DASHBOARD_WEBAUTHN_ORIGINS` for security keys; -- separate minimum-scope `MIRA_DASHBOARD_AUTOMATION_CREDENTIALS` entries for - heartbeat, task tracking, and report producers; - `MIRA_GITHUB_TOKEN` for Dashboard PR operations; - optional provider keys for Moltbook, ElevenLabs, OpenRouter, and Synthetic health checks depending on enabled Dashboard features. +The automation credential hashes are intentionally added immediately after the +initial bootstrap by following the provisioning section below. Their temporary +absence does not block service startup; it only leaves those local API callers +unauthorized until both services are restarted with the generated hashes. + +## Run The Production Bootstrap + +Run one command from the clean production checkout as the `ubuntu` user: + +```bash +cd /home/ubuntu/projects/mira-dashboard/production/checkout +bun run deploy:bootstrap +``` + +The command performs the complete first managed activation: + +1. enables systemd linger through `sudo loginctl` if it is not already enabled; +2. installs frozen control-checkout dependencies; +3. creates the production release, runtime, state, and development-worktree + directories with their required modes; +4. verifies that the production checkout is clean and resolves its exact full SHA; +5. initializes SQLite in WAL mode, applies every immutable migration, and runs + `PRAGMA quick_check`; +6. stages and preflights the SHA from an isolated worktree, then caches its + exact revision-qualified Bun executable; +7. activates the release, atomically installs and verifies both tracked + systemd unit files, and reloads the user manager; +8. enables and starts both services, waits briefly, and verifies that both + remain enabled and running. + +Run the command as the managed user, not with `sudo`; only its one +`loginctl enable-linger` child needs root. A normal sudo prompt may appear on a +host without passwordless sudo. Re-running the same checked-out SHA is safe and +repairs missing unit/runtime state. The command refuses to replace a different +existing current release; use the normal Dashboard deployment path for that. + +The control checkout is not a production runtime directory. See +[Production deploy](production-deploy.md) for the release/state layout, +automatic rollback, retention, and recovery contract. + ## Provision Local OpenClaw API Callers The Dashboard does not trust localhost as an identity. From the Dashboard @@ -155,6 +125,13 @@ the corresponding ids, SHA-256 validator hashes, and minimum scopes. Combine those four printed objects into the JSON array supplied through the Doppler secret `MIRA_DASHBOARD_AUTOMATION_CREDENTIALS`. +After adding or replacing that Doppler value, restart both managed services so +they load the new validator hashes: + +```bash +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +``` + Do not copy the full token files into Doppler, SQLite, shell history, prompts, cron payloads, reports, or host backups. A replacement host gets newly generated tokens and an updated hash-only Doppler array. See @@ -162,35 +139,20 @@ generated tokens and an updated hash-only Doppler array. See for the exact file names, scopes, wrapper behavior, rotation, and denied-route tests. -## Create The Systemd User Services - -Run this section from an interactive shell as the `ubuntu` user. Use `sudo` only -for the explicit `loginctl` command; the install and `systemctl --user` commands -must target `ubuntu`'s user manager. +## Managed Systemd User Services -Install the tracked web and worker units: - -```bash -cd /home/ubuntu/projects/mira-dashboard/production/checkout -install -d -m 0755 /home/ubuntu/.config/systemd/user -install -m 0644 systemd/mira-dashboard.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard.service -install -m 0644 systemd/mira-dashboard-worker.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service -``` +`deploy:bootstrap` installs, enables, starts, and verifies the tracked web and +worker units. No separate unit-file installation or `daemon-reload` is needed. The web role owns HTTP, WebSocket, and the Gateway bridge. The worker role owns scheduled-job registration, queue claims, cache startup seeds, and action execution. Both units have explicit CPU, IO, memory, and task guardrails. Heavy worker children are additionally placed in transient resource-class scopes. -Enable and start both: +Optional status checks: ```bash -sudo loginctl enable-linger ubuntu loginctl show-user ubuntu -p Linger -systemctl --user daemon-reload -systemctl --user enable --now mira-dashboard.service mira-dashboard-worker.service systemctl --user status mira-dashboard.service --no-pager systemctl --user status mira-dashboard-worker.service --no-pager ``` diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index d034e7904..b701426df 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -141,9 +141,9 @@ Managed releases carry checksummed copies of install a changed complete pair with mode `0644`, reload the user manager, and verify that both units loaded from `~/.config/systemd/user`. If reconciliation fails, the pre-operation files are restored before the release transition -returns an error. New VPS provisioning must still perform the initial install -and `enable --now`; ordinary deployments thereafter update the installed unit -definitions automatically. +returns an error. On a new VPS, `deploy:bootstrap` performs the first activation +and unit installation, then enables and starts both services. Ordinary +deployments thereafter update the installed unit definitions automatically. The first rollout of this unit-bundle contract is manual and supervised because its rollback release predates the bundle and is deliberately not special-cased diff --git a/frontend/src/test/componentBehavior.test.tsx b/frontend/src/test/componentBehavior.test.tsx index 7d23f4d26..3c075f584 100644 --- a/frontend/src/test/componentBehavior.test.tsx +++ b/frontend/src/test/componentBehavior.test.tsx @@ -3418,9 +3418,12 @@ describe("shared component helpers", () => { it("shows queue pressure and cancels an active job execution", async () => { let claimsPaused = false; + let delayNextPause = true; let executionStatus: "cancelled" | "queued" = "queued"; + let failNextClaimsUpdate = false; + const pauseMutationGate = Promise.withResolvers(); const fetchMock = jest.fn((input: RequestInfo | URL, init?: RequestInit) => { - return Promise.try(() => { + return Promise.try(async () => { const url = requestUrl(input); const method = init?.method ?? "GET"; const execution = { @@ -3464,9 +3467,18 @@ describe("shared component helpers", () => { } if (url === "/api/job-executions/claims" && method === "PATCH") { - claimsPaused = ( + const requestedPause = ( JSON.parse(requestBodyText(init?.body)) as { paused: boolean } ).paused; + if (failNextClaimsUpdate) { + failNextClaimsUpdate = false; + return Response.json({}, { status: 500 }); + } + if (requestedPause && delayNextPause) { + delayNextPause = false; + await pauseMutationGate.promise; + } + claimsPaused = requestedPause; return Response.json({ isOk: true, state: { @@ -3511,6 +3523,11 @@ describe("shared component helpers", () => { await userEvent.click( screen.getByRole("button", { name: "Pause worker claims" }) ); + expect(await screen.findByText("Saving...")).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: "Pause worker claims" }) + ).toBeDisabled(); + pauseMutationGate.resolve(); await waitFor(() => { expect(claimsPaused).toBe(true); expect(screen.getByText("Worker paused")).toBeInTheDocument(); @@ -3527,6 +3544,12 @@ describe("shared component helpers", () => { expect(claimsPaused).toBe(false); expect(screen.getByText("Worker idle")).toBeInTheDocument(); }); + failNextClaimsUpdate = true; + await userEvent.click( + screen.getByRole("button", { name: "Pause worker claims" }) + ); + expect(await screen.findByText("HTTP 500")).toBeInTheDocument(); + expect(claimsPaused).toBe(false); await userEvent.click(screen.getByRole("button", { name: "Cancel Host backup" })); diff --git a/frontend/src/test/pageBehavior.test.tsx b/frontend/src/test/pageBehavior.test.tsx index 4208d5fba..f78dda779 100644 --- a/frontend/src/test/pageBehavior.test.tsx +++ b/frontend/src/test/pageBehavior.test.tsx @@ -1513,6 +1513,8 @@ function apiResponse(url: string, method: string, init?: RequestInit) { executions: [], summary: { activeResourceClasses: [], + claimsPaused: false, + claimsPausedAt: undefined, queued: 0, running: 0, workerCapacity: 1, diff --git a/package.json b/package.json index daa3b725d..076035b29 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "build": "bun run build:frontend && bun run build:backend", "build:frontend": "bun node_modules/typescript/bin/tsc -p tsconfig.app.json --noEmit && bun scripts/buildFrontend.ts", "build:backend": "bun node_modules/typescript/bin/tsc -p tsconfig.node.json --noEmit && bun scripts/buildBackend.ts", + "deploy:bootstrap": "bash scripts/bootstrapProduction.sh", "deploy:prepare": "bun run build:frontend && bun run deploy:prepare:backend && bun run release:manifest", "deploy:prepare:backend": "bun run build:backend && bun --cwd backend dist/databasePreflight.js", "auth:reset-password": "NODE_ENV=production doppler run --config prd --project rajohan --preserve-env=NODE_ENV,MIRA_DASHBOARD_PROJECT_ROOT -- bun --cwd backend dist/resetDashboardPassword.js", diff --git a/scripts/bootstrapProduction.sh b/scripts/bootstrapProduction.sh new file mode 100755 index 000000000..297c9901f --- /dev/null +++ b/scripts/bootstrapProduction.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +repository_root="$( + cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + pwd -P +)" +dashboard_project_root="${MIRA_DASHBOARD_PROJECT_ROOT:-/home/ubuntu/projects/mira-dashboard}" +expected_checkout="$dashboard_project_root/production/checkout" + +if [[ "$(id -u)" == "0" ]]; then + echo "Run Dashboard bootstrap as the managed system user, not root" >&2 + exit 1 +fi +if [[ "$repository_root" != "$expected_checkout" ]]; then + echo "Dashboard bootstrap checkout must be $expected_checkout" >&2 + exit 1 +fi + +managed_user="$(id -un)" +linger_state="$(/usr/bin/loginctl show-user "$managed_user" --property=Linger --value 2>/dev/null || true)" +if [[ "$linger_state" != "yes" ]]; then + echo "Enabling persistent systemd user services for $managed_user" + /usr/bin/sudo /usr/bin/loginctl enable-linger "$managed_user" +fi +linger_state="$(/usr/bin/loginctl show-user "$managed_user" --property=Linger --value)" +if [[ "$linger_state" != "yes" ]]; then + echo "systemd linger was not enabled for $managed_user" >&2 + exit 1 +fi + +bootstrap_bun="${MIRA_DASHBOARD_DEPLOY_BUN_EXECUTABLE:-${HOME}/.bun/bin/bun}" +if [[ ! -f "$bootstrap_bun" || ! -x "$bootstrap_bun" || -L "$bootstrap_bun" ]]; then + echo "Dashboard bootstrap Bun is unavailable: $bootstrap_bun" >&2 + exit 1 +fi + +cd "$repository_root" +"$bootstrap_bun" install --frozen-lockfile +exec /usr/bin/env \ + MIRA_DASHBOARD_PROJECT_ROOT="$dashboard_project_root" \ + NODE_ENV=production \ + "$bootstrap_bun" scripts/productionBootstrap.ts diff --git a/scripts/productionBootstrap.ts b/scripts/productionBootstrap.ts new file mode 100644 index 000000000..ae99d52b1 --- /dev/null +++ b/scripts/productionBootstrap.ts @@ -0,0 +1,385 @@ +import fsp from "node:fs/promises"; +import path from "node:path"; + +import { + type DashboardProjectPaths, + resolveDashboardProjectPaths, +} from "../backend/src/lib/dashboardPaths.ts"; +import { runProcess } from "../backend/src/lib/processes.ts"; +import { parseSystemdProperties } from "../backend/src/lib/systemdProperties.ts"; +import { MANAGED_DASHBOARD_UNIT_NAMES } from "../backend/src/managedDashboardUnitPolicy.ts"; +import { stageDashboardRelease } from "../backend/src/releaseDeployment.ts"; +import { runReleaseLifecycleCommand } from "../backend/src/releaseLifecycle.ts"; +import { readDashboardReleaseState } from "../backend/src/releaseManager.ts"; + +const FULL_COMMIT_PATTERN = /^[\da-f]{40}$/u; +const SYSTEMCTL_EXECUTABLE = "/usr/bin/systemctl"; +const COMMAND_OUTPUT_LIMIT = 1024 * 1024; + +interface ProductionBootstrapCommandResult { + stderr: string; + stdout: string; +} + +export interface ProductionBootstrapCommandOptions { + cwd?: string; + timeoutMs: number; +} + +export type ProductionBootstrapCommandRunner = ( + command: string, + arguments_: readonly string[], + options: ProductionBootstrapCommandOptions +) => Promise; + +interface ProductionBootstrapReleaseSlots { + current?: string; + previous?: string; +} + +interface StagedProductionRelease { + commitSha: string; + path: string; +} + +export interface ProductionBootstrapOptions { + activateRelease?: (commitSha: string) => Promise; + commandRunner?: ProductionBootstrapCommandRunner; + environment?: NodeJS.ProcessEnv; + initializeDatabase?: () => Promise; + onProgress?: (message: string) => void; + paths?: DashboardProjectPaths; + readReleaseSlots?: () => Promise; + serviceStabilizationMs?: number; + stageRelease?: (commitSha: string) => Promise; +} + +export interface ProductionBootstrapResult { + commitSha: string; + databasePath: string; + releasePath: string; + services: Array<{ + activeState: string; + enabled: true; + name: string; + subState: string; + }>; +} + +export async function runProductionBootstrapCommand( + command: string, + arguments_: readonly string[], + options: ProductionBootstrapCommandOptions +): Promise { + const result = await runProcess(command, arguments_, { + cwd: options.cwd, + maxBuffer: COMMAND_OUTPUT_LIMIT, + timeoutMs: options.timeoutMs, + }); + if (result.code !== 0) { + const invocation = [command, ...arguments_].join(" "); + throw new Error( + `${invocation} failed with exit code ${ + result.code + }: ${result.stderr.trim() || result.stdout.trim()}` + ); + } + return { stderr: result.stderr, stdout: result.stdout }; +} + +async function ensureRealDirectory(directoryPath: string, mode: number): Promise { + let existingParent = directoryPath; + while (true) { + try { + const parentStat = await fsp.lstat(existingParent); + if (!parentStat.isDirectory() || parentStat.isSymbolicLink()) { + throw new TypeError( + `Dashboard bootstrap parent must be a real directory: ${existingParent}` + ); + } + if ((await fsp.realpath(existingParent)) !== path.resolve(existingParent)) { + throw new TypeError( + `Dashboard bootstrap parent must not traverse symlinks: ${existingParent}` + ); + } + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + const nextParent = path.dirname(existingParent); + if (nextParent === existingParent) { + throw new TypeError( + `Dashboard bootstrap path has no real parent: ${directoryPath}`, + { cause: error } + ); + } + existingParent = nextParent; + } + } + await fsp.mkdir(directoryPath, { mode, recursive: true }); + const stat = await fsp.lstat(directoryPath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new TypeError( + `Dashboard bootstrap path must be a real directory: ${directoryPath}` + ); + } + if ((await fsp.realpath(directoryPath)) !== path.resolve(directoryPath)) { + throw new TypeError( + `Dashboard bootstrap path must not traverse symlinks: ${directoryPath}` + ); + } + await fsp.chmod(directoryPath, mode); +} + +async function assertRealDirectory(directoryPath: string, label: string): Promise { + const stat = await fsp.lstat(directoryPath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new TypeError(`${label} must be a real directory`); + } + if ((await fsp.realpath(directoryPath)) !== path.resolve(directoryPath)) { + throw new TypeError(`${label} must not traverse symlinks`); + } +} + +export async function initializeProductionBootstrapDatabase(): Promise { + const { database } = await import("../backend/src/database.ts"); + try { + const quickCheck = database.query("PRAGMA quick_check").get() as { + quick_check?: unknown; + } | null; + if (quickCheck?.quick_check !== "ok") { + throw new Error("Fresh Dashboard database failed SQLite quick_check"); + } + } finally { + database.close(); + } +} + +function assertBootstrapEnvironment(environment: NodeJS.ProcessEnv): void { + if (environment.NODE_ENV !== "production") { + throw new Error("Dashboard production bootstrap requires NODE_ENV=production"); + } + if (typeof process.getuid === "function" && process.getuid() === 0) { + throw new Error( + "Dashboard production bootstrap must run as the managed system user, not root" + ); + } +} + +function assertServiceStabilizationMs(value: number): number { + if (!Number.isFinite(value) || value < 0) { + throw new RangeError( + "Dashboard bootstrap service stabilization delay must be non-negative" + ); + } + return value; +} + +async function resolveCheckoutCommit( + sourceRoot: string, + commandRunner: ProductionBootstrapCommandRunner +): Promise { + const repositoryRoot = await commandRunner("git", ["rev-parse", "--show-toplevel"], { + cwd: sourceRoot, + timeoutMs: 30_000, + }); + if (path.resolve(repositoryRoot.stdout.trim()) !== sourceRoot) { + throw new Error("Dashboard bootstrap checkout is not the repository root"); + } + const status = await commandRunner( + "git", + ["status", "--porcelain=v1", "--untracked-files=normal"], + { cwd: sourceRoot, timeoutMs: 30_000 } + ); + if (status.stdout.trim()) { + throw new Error("Dashboard bootstrap requires a clean production checkout"); + } + const identity = await commandRunner( + "git", + ["rev-parse", "--verify", "HEAD^{commit}"], + { cwd: sourceRoot, timeoutMs: 30_000 } + ); + const commitSha = identity.stdout.trim(); + if (!FULL_COMMIT_PATTERN.test(commitSha)) { + throw new Error("Dashboard bootstrap could not resolve a full lowercase Git SHA"); + } + return commitSha; +} + +async function verifyEnabledServices( + commandRunner: ProductionBootstrapCommandRunner +): Promise { + const services: ProductionBootstrapResult["services"] = []; + for (const name of MANAGED_DASHBOARD_UNIT_NAMES) { + const enabled = await commandRunner( + SYSTEMCTL_EXECUTABLE, + ["--user", "is-enabled", name], + { timeoutMs: 30_000 } + ); + if (enabled.stdout.trim() !== "enabled") { + throw new Error(`${name} was not persistently enabled`); + } + const state = await commandRunner( + SYSTEMCTL_EXECUTABLE, + [ + "--user", + "show", + name, + "--property=ActiveState", + "--property=Result", + "--property=SubState", + "--no-pager", + ], + { timeoutMs: 30_000 } + ); + const properties = parseSystemdProperties(state.stdout); + const activeState = properties.get("ActiveState") ?? ""; + const result = properties.get("Result") ?? ""; + const subState = properties.get("SubState") ?? ""; + if ( + activeState !== "active" || + subState !== "running" || + (result !== "" && result !== "success") + ) { + throw new Error( + `${name} did not remain active after bootstrap (${activeState}/${subState}/${result})` + ); + } + services.push({ + activeState, + enabled: true, + name, + subState, + }); + } + return services; +} + +/** + * Initializes and activates the first managed Dashboard release on a blank + * production host. Re-running the same checkout is safe; using this command to + * replace an existing different release is rejected. + * @param options Test and one-shot dependency overrides. + * @returns Activated release and service state. + */ +export async function bootstrapProductionDashboard( + options: ProductionBootstrapOptions = {} +): Promise { + const environment = options.environment ?? process.env; + assertBootstrapEnvironment(environment); + const paths = options.paths ?? resolveDashboardProjectPaths(environment); + const commandRunner = options.commandRunner ?? runProductionBootstrapCommand; + const onProgress = options.onProgress; + const stabilizationMs = assertServiceStabilizationMs( + options.serviceStabilizationMs ?? 2000 + ); + + await assertRealDirectory(paths.projectRoot, "Dashboard project root"); + await assertRealDirectory(paths.productionCheckoutRoot, "Production checkout"); + onProgress?.("Preparing managed production directories"); + for (const [directoryPath, mode] of [ + [paths.productionRoot, 0o755], + [paths.developmentRoot, 0o755], + [paths.developmentWorktreeRoot, 0o755], + [paths.productionReleasesRoot, 0o755], + [path.dirname(paths.productionBunRuntimeRoot), 0o755], + [paths.productionBunRuntimeRoot, 0o755], + [paths.productionStateRoot, 0o700], + [paths.productionOpenClawHome, 0o700], + ] as const) { + await ensureRealDirectory(directoryPath, mode); + } + + onProgress?.("Verifying clean production checkout"); + const commitSha = await resolveCheckoutCommit( + paths.productionCheckoutRoot, + commandRunner + ); + const readReleaseSlots = + options.readReleaseSlots ?? + (async () => { + const state = await readDashboardReleaseState(paths.productionReleasesRoot); + return { + current: state.current?.commitSha, + previous: state.previous?.commitSha, + }; + }); + const slots = await readReleaseSlots(); + if (slots.previous && !slots.current) { + throw new Error( + "Dashboard bootstrap found an invalid previous release without current" + ); + } + if (slots.current && slots.current !== commitSha) { + throw new Error( + `Dashboard bootstrap refuses to replace existing release ${slots.current}; use the normal deployment path` + ); + } + + onProgress?.("Initializing and verifying production SQLite"); + await (options.initializeDatabase ?? initializeProductionBootstrapDatabase)(); + + onProgress?.("Staging the initial managed release"); + const stageRelease = + options.stageRelease ?? + ((candidateCommit: string) => + stageDashboardRelease(candidateCommit, { + onProgress, + releasesRoot: paths.productionReleasesRoot, + sourceRoot: paths.productionCheckoutRoot, + worktreeRoot: paths.developmentWorktreeRoot, + })); + const release = await stageRelease(commitSha); + if (release.commitSha !== commitSha) { + throw new Error("Dashboard bootstrap staged an unexpected release"); + } + + onProgress?.("Activating release and reconciling managed systemd units"); + const activateRelease = + options.activateRelease ?? + (async (candidateCommit: string) => { + await runReleaseLifecycleCommand( + ["activate", candidateCommit], + paths.productionReleasesRoot + ); + }); + await activateRelease(commitSha); + + onProgress?.("Enabling and starting Dashboard services"); + await commandRunner( + SYSTEMCTL_EXECUTABLE, + ["--user", "enable", "--now", ...MANAGED_DASHBOARD_UNIT_NAMES], + { timeoutMs: 90_000 } + ); + if (stabilizationMs > 0) { + await Bun.sleep(stabilizationMs); + } + const services = await verifyEnabledServices(commandRunner); + onProgress?.("Dashboard production bootstrap completed"); + return { + commitSha, + databasePath: paths.productionDatabasePath, + releasePath: release.path, + services, + }; +} + +if (import.meta.main) { + try { + if (Bun.argv.length > 2) { + throw new TypeError("Usage: productionBootstrap.ts"); + } + const result = await bootstrapProductionDashboard({ + onProgress: (message) => { + process.stdout.write(`[bootstrap] ${message}\n`); + }, + }); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + process.stderr.write( + `${error instanceof Error ? error.message : "Dashboard production bootstrap failed"}\n` + ); + process.exitCode = 1; + } +} diff --git a/scripts/runManagedDashboardRelease.sh b/scripts/runManagedDashboardRelease.sh index 7ffd77f53..c54c04e3d 100755 --- a/scripts/runManagedDashboardRelease.sh +++ b/scripts/runManagedDashboardRelease.sh @@ -55,7 +55,7 @@ bun_version="$( /usr/bin/jq --exit-status --raw-output \ '.bunVersion | select(type == "string" and length > 0 and length <= 64) - | select(test("^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*))(\\.((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*)))*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$"))' \ + | select(test("^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*))(\\.((0|[1-9][0-9]*)|([0-9]*[A-Za-z-][0-9A-Za-z-]*)))*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)$"))' \ "$manifest_path" )" || { echo "Managed Dashboard release manifest has no valid Bun runtime" >&2 From ece7fb8d917fb5606708670b8952e34af76aa37b Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Thu, 30 Jul 2026 04:06:06 +0200 Subject: [PATCH 3/3] fix: close deployment review edge cases --- backend/src/services/cacheRefreshMetrics.ts | 76 +++++++++++++++++--- backend/src/services/pullRequests.ts | 2 +- backend/src/services/scheduledJobs.ts | 4 -- backend/test/cacheRefreshMetrics.test.ts | 63 +++++++++++++++-- backend/test/jobExecutionQueue.test.ts | 44 ++++++++++++ backend/test/managedBunRuntime.test.ts | 29 ++++++++ backend/test/productionBootstrap.test.ts | 77 +++++++++++++++++++-- backend/test/releaseManager.test.ts | 8 ++- backend/test/serviceBehavior.test.ts | 3 + docs/operations/scheduler-cache-backups.md | 14 ++-- docs/setup/new-vps.md | 6 +- scripts/bootstrapProduction.sh | 15 +++- scripts/productionBootstrap.ts | 56 +++++++++++---- scripts/runManagedDashboardRelease.sh | 7 +- 14 files changed, 353 insertions(+), 51 deletions(-) diff --git a/backend/src/services/cacheRefreshMetrics.ts b/backend/src/services/cacheRefreshMetrics.ts index eaff37b4f..899d351e2 100644 --- a/backend/src/services/cacheRefreshMetrics.ts +++ b/backend/src/services/cacheRefreshMetrics.ts @@ -8,8 +8,11 @@ import { cacheRefreshMetricsSchema } from "../../../contracts/metrics.ts"; const CACHE_REFRESH_METRICS_SNAPSHOT_VERSION = 1; const MAX_CACHE_REFRESH_METRICS_SNAPSHOT_BYTES = 16 * 1024; +const MAX_CACHE_REFRESH_METRICS_SNAPSHOT_CANDIDATES = 64; const RUNTIME_DIRECTORY_NAME = "mira-dashboard"; const SNAPSHOT_FILE_NAME = "cache-refresh-metrics.json"; +const SNAPSHOT_INSTANCE_ID_PATTERN = + /^[\da-f]{8}-[\da-f]{4}-7[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; interface CacheRefreshMetricsSnapshot { instanceId: string; @@ -90,6 +93,31 @@ function metricsSnapshot(): CacheRefreshMetrics { }; } +function instanceSnapshotPath(snapshotPath: string, instanceId: string): string { + const parsed = path.parse(snapshotPath); + return path.join(parsed.dir, `${parsed.name}.${instanceId}${parsed.ext}`); +} + +function instanceIdFromSnapshotName( + snapshotPath: string, + candidateName: string +): string | undefined { + const parsed = path.parse(snapshotPath); + const prefix = `${parsed.name}.`; + if ( + !candidateName.startsWith(prefix) || + !candidateName.endsWith(parsed.ext) || + candidateName === path.basename(snapshotPath) + ) { + return undefined; + } + const instanceId = candidateName.slice( + prefix.length, + parsed.ext === "" ? undefined : -parsed.ext.length + ); + return SNAPSHOT_INSTANCE_ID_PATTERN.test(instanceId) ? instanceId : undefined; +} + function ensurePrivateRuntimeDirectory(directoryPath: string): void { fs.mkdirSync(directoryPath, { mode: 0o700, recursive: true }); const stat = fs.lstatSync(directoryPath); @@ -206,6 +234,35 @@ function readSnapshot(snapshotPath: string): CacheRefreshMetricsSnapshot | undef } } +function readLatestSnapshot( + snapshotPath: string +): CacheRefreshMetricsSnapshot | undefined { + const directoryPath = path.dirname(snapshotPath); + let candidates: Array<{ instanceId: string; path: string }>; + try { + candidates = fs + .readdirSync(directoryPath, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .flatMap((entry) => { + const instanceId = instanceIdFromSnapshotName(snapshotPath, entry.name); + return instanceId + ? [{ instanceId, path: path.join(directoryPath, entry.name) }] + : []; + }) + .toSorted((left, right) => right.instanceId.localeCompare(left.instanceId)) + .slice(0, MAX_CACHE_REFRESH_METRICS_SNAPSHOT_CANDIDATES); + } catch { + return undefined; + } + for (const candidate of candidates) { + const snapshot = readSnapshot(candidate.path); + if (snapshot?.instanceId === candidate.instanceId) { + return snapshot; + } + } + return undefined; +} + /** * Starts a fresh in-memory metrics session and publishes its zero snapshot for * production IPC. Repeated registration inside the same worker is idempotent. @@ -215,27 +272,30 @@ export function startCacheRefreshMetricsSession( ): void { if (activeSession) return; cacheRefreshMetricsState = emptyCacheRefreshMetrics(); + const instanceId = Bun.randomUUIDv7(); + const snapshotPath = + options.snapshotPath ?? + resolveCacheRefreshMetricsSnapshotPath(options.environment ?? process.env); activeSession = { directoryValidated: false, - instanceId: Bun.randomUUIDv7(), + instanceId, snapshotPath: - options.snapshotPath ?? - resolveCacheRefreshMetricsSnapshotPath(options.environment ?? process.env), + snapshotPath === undefined + ? undefined + : instanceSnapshotPath(snapshotPath, instanceId), startedAt: new Date().toISOString(), }; publishSnapshot(); } /** - * Removes only this worker instance's volatile snapshot. A replacement worker - * that has already published a newer instance is left intact. + * Removes this worker instance's uniquely named volatile snapshot. Replacement + * workers publish to different paths, so cleanup cannot unlink their state. */ export function stopCacheRefreshMetricsSession(): void { const session = activeSession; activeSession = undefined; if (!session?.snapshotPath) return; - const current = readSnapshot(session.snapshotPath); - if (current?.instanceId !== session.instanceId) return; try { fs.unlinkSync(session.snapshotPath); } catch (error) { @@ -260,7 +320,7 @@ export function getCacheRefreshMetrics( options.snapshotPath ?? resolveCacheRefreshMetricsSnapshotPath(options.environment ?? process.env); return ( - (snapshotPath && readSnapshot(snapshotPath)?.metrics) || + (snapshotPath && readLatestSnapshot(snapshotPath)?.metrics) || emptyCacheRefreshMetrics() ); } diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 61f4af32e..af4cb604d 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -2123,7 +2123,7 @@ function releaseCutoverShellFunctions(): string[] { ' [ -f "$runtime_path" ] && [ -x "$runtime_path" ] && [ ! -L "$runtime_path" ] || return 1', ' [ "$(/usr/bin/realpath --canonicalize-existing "$runtime_path")" = "$runtime_path" ] || return 1', ' [ "$(/usr/bin/stat --format=\'%h\' -- "$runtime_path")" = 1 ] || return 1', - ' runtime_revision="$("$runtime_path" --revision)" || return 1', + ' runtime_revision="$(/usr/bin/timeout --signal=KILL 5s "$runtime_path" --revision 2>/dev/null)" || return 1', ' [ "$runtime_revision" = "$bun_version" ] || return 1', ' printf "%s" "$runtime_path"', "}", diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index de9a98d7e..5196a56fe 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -30,7 +30,6 @@ import { unregisterJobWorker, updateJobExecutionOutput, } from "./jobExecutionQueue.ts"; -import { getJobWorkerClaimsState } from "./jobWorkerControl.ts"; import { waitForJobExecution } from "./queuedJobExecution.ts"; const logger = createStructuredLogger("scheduled-jobs"); @@ -1510,9 +1509,6 @@ function executorTick(): void { if (hasPendingDeploymentCutover()) { return; } - if (getJobWorkerClaimsState().paused) { - return; - } const execution = claimNextJobExecution( scheduledJobRuntimeState.workerId, executorCapacity diff --git a/backend/test/cacheRefreshMetrics.test.ts b/backend/test/cacheRefreshMetrics.test.ts index 2bf2158e0..6d86e6960 100644 --- a/backend/test/cacheRefreshMetrics.test.ts +++ b/backend/test/cacheRefreshMetrics.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, rmSync, statSync, @@ -32,6 +33,11 @@ function temporaryRoot(): string { return root; } +function instanceSnapshotPath(snapshotPath: string, instanceId: string): string { + const parsed = path.parse(snapshotPath); + return path.join(parsed.dir, `${parsed.name}.${instanceId}${parsed.ext}`); +} + async function readText(stream: ReadableStream): Promise { return new Response(stream).text(); } @@ -72,8 +78,16 @@ describe("cache refresh runtime metrics", () => { requests: 1, totalDurationMs: 12.35, }); + const publishedSnapshotName = readdirSync(path.dirname(snapshotPath!)).find( + (name) => name.startsWith("cache-refresh-metrics.") && name.endsWith(".json") + ); + expect(publishedSnapshotName).toBeDefined(); + const publishedSnapshotPath = path.join( + path.dirname(snapshotPath!), + publishedSnapshotName! + ); expect(statSync(path.dirname(snapshotPath!)).mode & 0o777).toBe(0o700); - expect(statSync(snapshotPath!).mode & 0o777).toBe(0o600); + expect(statSync(publishedSnapshotPath).mode & 0o777).toBe(0o600); const moduleUrl = pathToFileURL( path.resolve(import.meta.dirname, "../src/services/cacheRefreshMetrics.ts") @@ -101,8 +115,36 @@ describe("cache refresh runtime metrics", () => { expect({ exitCode, stderr }).toEqual({ exitCode: 0, stderr: "" }); expect(JSON.parse(stdout)).toEqual(getCacheRefreshMetrics()); + const replacementInstanceId = "ffffffff-ffff-7fff-bfff-ffffffffffff"; + const replacementSnapshotPath = instanceSnapshotPath( + snapshotPath!, + replacementInstanceId + ); + writeFileSync( + replacementSnapshotPath, + `${JSON.stringify({ + instanceId: replacementInstanceId, + metrics: { + active: 0, + averageDurationMs: 9, + coalesced: 0, + failures: 0, + lastDurationMs: 9, + maxDurationMs: 9, + refreshes: 9, + requests: 9, + totalDurationMs: 81, + }, + pid: process.pid, + startedAt: "2026-07-30T08:00:00.000Z", + version: 1, + })}\n`, + { encoding: "utf8", flag: "wx", mode: 0o600 } + ); stopCacheRefreshMetricsSession(); - expect(existsSync(snapshotPath!)).toBe(false); + expect(existsSync(publishedSnapshotPath)).toBe(false); + expect(existsSync(replacementSnapshotPath)).toBe(true); + expect(getCacheRefreshMetrics({ environment }).requests).toBe(9); }); it("fails soft on missing or malformed runtime snapshots", () => { @@ -131,12 +173,17 @@ describe("cache refresh runtime metrics", () => { }); mkdirSync(path.dirname(snapshotPath), { mode: 0o700, recursive: true }); - writeFileSync(snapshotPath, '{"version":1,"metrics":"invalid"}\n', { + const malformedInstanceId = Bun.randomUUIDv7(); + const malformedSnapshotPath = instanceSnapshotPath( + snapshotPath, + malformedInstanceId + ); + writeFileSync(malformedSnapshotPath, '{"version":1,"metrics":"invalid"}\n', { encoding: "utf8", flag: "wx", mode: 0o600, }); - expect(readFileSync(snapshotPath, "utf8")).toContain('"invalid"'); + expect(readFileSync(malformedSnapshotPath, "utf8")).toContain('"invalid"'); expect(getCacheRefreshMetrics({ environment })).toEqual({ active: 0, averageDurationMs: 0, @@ -149,11 +196,13 @@ describe("cache refresh runtime metrics", () => { totalDurationMs: 0, }); - rmSync(snapshotPath); + rmSync(malformedSnapshotPath); + const staleInstanceId = Bun.randomUUIDv7(); + const staleSnapshotPath = instanceSnapshotPath(snapshotPath, staleInstanceId); writeFileSync( - snapshotPath, + staleSnapshotPath, `${JSON.stringify({ - instanceId: Bun.randomUUIDv7(), + instanceId: staleInstanceId, metrics: { active: 0, averageDurationMs: 4, diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 4ee543e18..5e70f5c31 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -634,6 +634,50 @@ printf 'LoadState=loaded\nActiveState=active\n' finishJobExecution(queued.id, workerId, "success", undefined, {}); }); + it("recovers leases that expire while new claims are paused", async () => { + const timestamp = new Date().toISOString(); + const running = insertJobExecution({ + actionKey: `test.paused-expired-${Bun.randomUUIDv7()}`, + cancellable: false, + displayName: "Paused expired execution", + leaseOwner: "missing-worker", + queuedAt: timestamp, + resourceClass: "exclusive", + status: "running", + timeoutMs: 60_000, + triggerType: "system", + }); + const queued = enqueueJobExecution({ + actionKey: `test.paused-queued-${Bun.randomUUIDv7()}`, + displayName: "Paused queued execution", + resourceClass: "exclusive", + timeoutMs: 60_000, + }); + testExecutionIds.add(running.id); + testExecutionIds.add(queued.id); + setJobWorkerClaimsPaused(true, timestamp); + + startScheduledJobExecutor(); + database + .prepare( + `UPDATE job_executions + SET lease_expires_at = ? + WHERE id = ?` + ) + .run("1970-01-01T00:00:00.000Z", running.id); + + expect( + await waitForJobExecution(running.id, { + pollIntervalMs: 25, + timeoutMs: 3000, + }) + ).toMatchObject({ + message: "Job failed after its worker lease expired", + status: "failed", + }); + expect(getJobExecution(queued.id)).toMatchObject({ status: "queued" }); + }); + it("reconciles orphaned deployment cutovers while new claims are paused", () => { const deploymentId = createVerifyingDeployment( "2026-07-26T03:00:00.000Z", diff --git a/backend/test/managedBunRuntime.test.ts b/backend/test/managedBunRuntime.test.ts index 043e78236..ba8d12231 100644 --- a/backend/test/managedBunRuntime.test.ts +++ b/backend/test/managedBunRuntime.test.ts @@ -288,6 +288,35 @@ describe("managed Bun runtimes", () => { "release manifest has no valid Bun runtime" ); + writeFileSync( + path.join(releaseRoot, "release-manifest.json"), + `${JSON.stringify({ bunVersion: "2.0.0+feedface" })}\n` + ); + writeFileSync( + runtime, + `#!/bin/sh +if [ "\${1:-}" = "--revision" ]; then + exit 1 +fi +printf '%s\\n' "\${1:-}" +` + ); + chmodSync(runtime, 0o700); + const failedProbe = Bun.spawnSync({ + cmd: [launcher, "dist/workerStart.js"], + cwd: releaseBackend, + env: { + MIRA_DASHBOARD_PROJECT_ROOT: projectRoot, + PATH: "/usr/bin:/bin", + }, + stderr: "pipe", + stdout: "pipe", + }); + expect(failedProbe.exitCode).toBe(78); + expect(new TextDecoder().decode(failedProbe.stderr)).toContain( + "runtime revision probe failed" + ); + const rejected = Bun.spawnSync({ cmd: [launcher, "arbitrary.js"], env: { diff --git a/backend/test/productionBootstrap.test.ts b/backend/test/productionBootstrap.test.ts index 29b42b398..8272af6fb 100644 --- a/backend/test/productionBootstrap.test.ts +++ b/backend/test/productionBootstrap.test.ts @@ -78,6 +78,12 @@ describe("production bootstrap", () => { expect((commandError as Error).message).toContain( "/usr/bin/false failed with exit code 1" ); + expect( + await runProductionBootstrapCommand("/usr/bin/false", [], { + allowNonZeroExit: true, + timeoutMs: 5000, + }) + ).toEqual({ stderr: "", stdout: "" }); await initializeProductionBootstrapDatabase(); }); @@ -140,13 +146,14 @@ describe("production bootstrap", () => { "Initializing and verifying production SQLite", "Staging the initial managed release", "Activating release and reconciling managed systemd units", - "Enabling and starting Dashboard services", + "Enabling and restarting Dashboard services", "Dashboard production bootstrap completed", ]); expect(calls).toContain( - `/usr/bin/systemctl --user enable --now ${MANAGED_DASHBOARD_UNIT_NAMES.join( - " " - )}` + `/usr/bin/systemctl --user enable ${MANAGED_DASHBOARD_UNIT_NAMES.join(" ")}` + ); + expect(calls).toContain( + `/usr/bin/systemctl --user restart ${MANAGED_DASHBOARD_UNIT_NAMES.join(" ")}` ); for (const unit of MANAGED_DASHBOARD_UNIT_NAMES) { expect(calls).toContain(`/usr/bin/systemctl --user is-enabled ${unit}`); @@ -227,6 +234,68 @@ describe("production bootstrap", () => { expect(stageRelease).not.toHaveBeenCalled(); }); + it("restarts repaired units when rerunning the current release", async () => { + const root = temporaryProjectRoot(); + const paths = dashboardProjectPaths(root); + const calls: string[] = []; + + await bootstrapProductionDashboard({ + activateRelease: () => Promise.resolve(), + commandRunner: commandRunner(paths.productionCheckoutRoot, calls), + environment: { NODE_ENV: "production" }, + initializeDatabase: () => Promise.resolve(), + paths, + readReleaseSlots: () => Promise.resolve({ current: COMMIT_SHA }), + serviceStabilizationMs: 0, + stageRelease: (commitSha) => Promise.resolve({ commitSha, path: "release" }), + }); + + const enableIndex = calls.indexOf( + `/usr/bin/systemctl --user enable ${MANAGED_DASHBOARD_UNIT_NAMES.join(" ")}` + ); + const restartIndex = calls.indexOf( + `/usr/bin/systemctl --user restart ${MANAGED_DASHBOARD_UNIT_NAMES.join(" ")}` + ); + expect(enableIndex).toBeGreaterThan(-1); + expect(restartIndex).toBeGreaterThan(enableIndex); + }); + + it("polls transient service startup state until both units are healthy", async () => { + const root = temporaryProjectRoot(); + const paths = dashboardProjectPaths(root); + const baseRunner = commandRunner(paths.productionCheckoutRoot, []); + let stateChecks = 0; + const settlingRunner: ProductionBootstrapCommandRunner = ( + command, + arguments_, + options + ) => { + if (command === "/usr/bin/systemctl" && arguments_[1] === "show") { + stateChecks += 1; + if (stateChecks === 1) { + return Promise.resolve({ + stderr: "", + stdout: "ActiveState=activating\nResult=success\nSubState=start\n", + }); + } + } + return baseRunner(command, arguments_, options); + }; + + await bootstrapProductionDashboard({ + activateRelease: () => Promise.resolve(), + commandRunner: settlingRunner, + environment: { NODE_ENV: "production" }, + initializeDatabase: () => Promise.resolve(), + paths, + readReleaseSlots: () => Promise.resolve({}), + serviceStabilizationMs: 10, + stageRelease: (commitSha) => Promise.resolve({ commitSha, path: "release" }), + }); + + expect(stateChecks).toBe(MANAGED_DASHBOARD_UNIT_NAMES.length + 1); + }); + it("fails closed for invalid slots, staged identity, and service state", async () => { const invalidDelayError = await captureRejection(() => bootstrapProductionDashboard({ diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index aeabdf5cc..26b7ec43f 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -714,12 +714,16 @@ describe("Dashboard immutable release manager", () => { expect(() => assertDashboardReleaseRuntimeAvailable(incompatibleRelease)).toThrow( "requires unavailable managed Bun runtime 0.0.0+missing" ); - expect( + const activationError = await captureRejection(() => activateDashboardRelease(FIRST_COMMIT, runtimeRoot, { ...SCHEMA_6_OPTIONS, hasRuntime: () => false, }) - ).rejects.toThrow("requires unavailable managed Bun runtime 0.0.0+missing"); + ); + expect(activationError).toBeInstanceOf(Error); + expect((activationError as Error).message).toContain( + "requires unavailable managed Bun runtime 0.0.0+missing" + ); const cachedMajorRuntimeRoot = temporaryReleasesRoot(); await createManagedRelease( diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 5d266a139..2bf90b464 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -2175,6 +2175,9 @@ printf 'scheduled\n' expect(guardian).toContain( `${releasesRoot}/releases/${currentCommit}/backend/dist/releaseLifecycle.js` ); + expect(guardian).toContain( + '/usr/bin/timeout --signal=KILL 5s "$runtime_path" --revision' + ); expect(guardian).toContain(`rollback '${currentCommit}' '${previousCommit}'`); expect(guardian).toContain(`rollback '${previousCommit}' '${currentCommit}'`); expect(guardian).toContain( diff --git a/docs/operations/scheduler-cache-backups.md b/docs/operations/scheduler-cache-backups.md index 89894b911..110a06699 100644 --- a/docs/operations/scheduler-cache-backups.md +++ b/docs/operations/scheduler-cache-backups.md @@ -136,12 +136,14 @@ The **Cache refresh** observability card is different from cached provider data. Its request/coalescing/failure/duration counters are runtime telemetry since the current worker process started, not historical totals and not SQLite rows. Production worker and web are separate processes, so the worker -atomically mirrors its in-memory counters to an owner-only snapshot below the -user's reboot-volatile `XDG_RUNTIME_DIR`; the web process samples that snapshot. -The managed processes are systemd user services, so their user manager supplies -`XDG_RUNTIME_DIR`; new-host bootstrap enables linger so `/run/user/` is -also created for boot-time starts without an interactive login. Worker restart -resets every counter. +atomically mirrors its in-memory counters to an instance-unique, owner-only +snapshot below the user's reboot-volatile `XDG_RUNTIME_DIR`; the web process +samples the newest live worker instance. An old worker removes only its own +snapshot, so overlapping restart cleanup cannot delete its replacement's +metrics. The managed processes are systemd user services, so their user manager +supplies `XDG_RUNTIME_DIR`; new-host bootstrap enables linger so +`/run/user/` is also created for boot-time starts without an interactive +login. Worker restart resets every counter. ### Status And Heartbeat Projections diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index ffab3f022..8f0822433 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -92,8 +92,8 @@ The command performs the complete first managed activation: exact revision-qualified Bun executable; 7. activates the release, atomically installs and verifies both tracked systemd unit files, and reloads the user manager; -8. enables and starts both services, waits briefly, and verifies that both - remain enabled and running. +8. enables and restarts both services, then polls within a bounded startup window + until both become enabled and running. Run the command as the managed user, not with `sudo`; only its one `loginctl enable-linger` child needs root. A normal sudo prompt may appear on a @@ -141,7 +141,7 @@ tests. ## Managed Systemd User Services -`deploy:bootstrap` installs, enables, starts, and verifies the tracked web and +`deploy:bootstrap` installs, enables, restarts, and verifies the tracked web and worker units. No separate unit-file installation or `daemon-reload` is needed. The web role owns HTTP, WebSocket, and the Gateway bridge. The worker role owns diff --git a/scripts/bootstrapProduction.sh b/scripts/bootstrapProduction.sh index 297c9901f..17b9af6f4 100755 --- a/scripts/bootstrapProduction.sh +++ b/scripts/bootstrapProduction.sh @@ -6,6 +6,19 @@ repository_root="$( pwd -P )" dashboard_project_root="${MIRA_DASHBOARD_PROJECT_ROOT:-/home/ubuntu/projects/mira-dashboard}" +dashboard_project_root="$( + printf '%s' "$dashboard_project_root" | + /usr/bin/sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +)" +if [[ -z "$dashboard_project_root" ]]; then + dashboard_project_root="/home/ubuntu/projects/mira-dashboard" +fi +if ! dashboard_project_root="$( + /usr/bin/realpath --canonicalize-existing -- "$dashboard_project_root" +)"; then + echo "Dashboard project root must be an existing real path" >&2 + exit 1 +fi expected_checkout="$dashboard_project_root/production/checkout" if [[ "$(id -u)" == "0" ]]; then @@ -23,7 +36,7 @@ if [[ "$linger_state" != "yes" ]]; then echo "Enabling persistent systemd user services for $managed_user" /usr/bin/sudo /usr/bin/loginctl enable-linger "$managed_user" fi -linger_state="$(/usr/bin/loginctl show-user "$managed_user" --property=Linger --value)" +linger_state="$(/usr/bin/loginctl show-user "$managed_user" --property=Linger --value 2>/dev/null || true)" if [[ "$linger_state" != "yes" ]]; then echo "systemd linger was not enabled for $managed_user" >&2 exit 1 diff --git a/scripts/productionBootstrap.ts b/scripts/productionBootstrap.ts index ae99d52b1..3d7247ea5 100644 --- a/scripts/productionBootstrap.ts +++ b/scripts/productionBootstrap.ts @@ -15,6 +15,8 @@ import { readDashboardReleaseState } from "../backend/src/releaseManager.ts"; const FULL_COMMIT_PATTERN = /^[\da-f]{40}$/u; const SYSTEMCTL_EXECUTABLE = "/usr/bin/systemctl"; const COMMAND_OUTPUT_LIMIT = 1024 * 1024; +const DEFAULT_SERVICE_STABILIZATION_MS = 30_000; +const SERVICE_POLL_INTERVAL_MS = 250; interface ProductionBootstrapCommandResult { stderr: string; @@ -22,6 +24,7 @@ interface ProductionBootstrapCommandResult { } export interface ProductionBootstrapCommandOptions { + allowNonZeroExit?: boolean; cwd?: string; timeoutMs: number; } @@ -76,7 +79,7 @@ export async function runProductionBootstrapCommand( maxBuffer: COMMAND_OUTPUT_LIMIT, timeoutMs: options.timeoutMs, }); - if (result.code !== 0) { + if (result.code !== 0 && options.allowNonZeroExit !== true) { const invocation = [command, ...arguments_].join(" "); throw new Error( `${invocation} failed with exit code ${ @@ -145,10 +148,15 @@ async function assertRealDirectory(directoryPath: string, label: string): Promis export async function initializeProductionBootstrapDatabase(): Promise { const { database } = await import("../backend/src/database.ts"); try { - const quickCheck = database.query("PRAGMA quick_check").get() as { - quick_check?: unknown; - } | null; - if (quickCheck?.quick_check !== "ok") { + const quickCheck = database.query("PRAGMA quick_check").all() as Array< + Record + >; + if ( + quickCheck.length !== 1 || + Object.values(quickCheck[0] ?? {}).every( + (value) => typeof value !== "string" || value.toLowerCase() !== "ok" + ) + ) { throw new Error("Fresh Dashboard database failed SQLite quick_check"); } } finally { @@ -170,7 +178,7 @@ function assertBootstrapEnvironment(environment: NodeJS.ProcessEnv): void { function assertServiceStabilizationMs(value: number): number { if (!Number.isFinite(value) || value < 0) { throw new RangeError( - "Dashboard bootstrap service stabilization delay must be non-negative" + "Dashboard bootstrap service stabilization window must be non-negative" ); } return value; @@ -215,7 +223,7 @@ async function verifyEnabledServices( const enabled = await commandRunner( SYSTEMCTL_EXECUTABLE, ["--user", "is-enabled", name], - { timeoutMs: 30_000 } + { allowNonZeroExit: true, timeoutMs: 30_000 } ); if (enabled.stdout.trim() !== "enabled") { throw new Error(`${name} was not persistently enabled`); @@ -256,6 +264,24 @@ async function verifyEnabledServices( return services; } +async function waitForEnabledServices( + commandRunner: ProductionBootstrapCommandRunner, + stabilizationMs: number +): Promise { + const deadline = Date.now() + stabilizationMs; + while (true) { + try { + return await verifyEnabledServices(commandRunner); + } catch (error) { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw error; + } + await Bun.sleep(Math.min(SERVICE_POLL_INTERVAL_MS, remainingMs)); + } + } +} + /** * Initializes and activates the first managed Dashboard release on a blank * production host. Re-running the same checkout is safe; using this command to @@ -272,7 +298,7 @@ export async function bootstrapProductionDashboard( const commandRunner = options.commandRunner ?? runProductionBootstrapCommand; const onProgress = options.onProgress; const stabilizationMs = assertServiceStabilizationMs( - options.serviceStabilizationMs ?? 2000 + options.serviceStabilizationMs ?? DEFAULT_SERVICE_STABILIZATION_MS ); await assertRealDirectory(paths.projectRoot, "Dashboard project root"); @@ -346,16 +372,18 @@ export async function bootstrapProductionDashboard( }); await activateRelease(commitSha); - onProgress?.("Enabling and starting Dashboard services"); + onProgress?.("Enabling and restarting Dashboard services"); await commandRunner( SYSTEMCTL_EXECUTABLE, - ["--user", "enable", "--now", ...MANAGED_DASHBOARD_UNIT_NAMES], + ["--user", "enable", ...MANAGED_DASHBOARD_UNIT_NAMES], { timeoutMs: 90_000 } ); - if (stabilizationMs > 0) { - await Bun.sleep(stabilizationMs); - } - const services = await verifyEnabledServices(commandRunner); + await commandRunner( + SYSTEMCTL_EXECUTABLE, + ["--user", "restart", ...MANAGED_DASHBOARD_UNIT_NAMES], + { timeoutMs: 90_000 } + ); + const services = await waitForEnabledServices(commandRunner, stabilizationMs); onProgress?.("Dashboard production bootstrap completed"); return { commitSha, diff --git a/scripts/runManagedDashboardRelease.sh b/scripts/runManagedDashboardRelease.sh index c54c04e3d..e10b38006 100755 --- a/scripts/runManagedDashboardRelease.sh +++ b/scripts/runManagedDashboardRelease.sh @@ -75,7 +75,12 @@ if [[ "$(/usr/bin/stat --format='%h' -- "$runtime_path")" != "1" ]]; then echo "Managed Dashboard Bun runtime must not have external hard links" >&2 exit 78 fi -runtime_revision="$("$runtime_path" --revision)" +if ! runtime_revision="$( + /usr/bin/timeout --signal=KILL 5s "$runtime_path" --revision 2>/dev/null +)"; then + echo "Managed Dashboard Bun runtime revision probe failed" >&2 + exit 78 +fi if [[ "$runtime_revision" != "$bun_version" ]]; then echo "Managed Dashboard Bun runtime version does not match the release" >&2 exit 78