From b81e27c689614d77060478f5cfde05da934b6191 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 05:18:13 +0200 Subject: [PATCH 01/11] Complete atomic release deployment cutover --- backend/src/releaseDeployment.ts | 551 +++++++++++++++ backend/src/releaseLifecycle.ts | 7 +- backend/src/releaseManager.ts | 132 +++- backend/src/releaseManifest.ts | 68 +- backend/src/requestPolicy.ts | 1 - backend/src/routes.ts | 32 - backend/src/services/logRotation.ts | 29 +- backend/src/services/pullRequests.ts | 219 ++++-- backend/test/bunNativeServerBehavior.test.ts | 11 - backend/test/healthReadiness.test.ts | 4 +- backend/test/httpApiBehavior.test.ts | 16 - backend/test/releaseDeployment.test.ts | 535 ++++++++++++++ backend/test/releaseManager.test.ts | 59 +- backend/test/releaseManifest.test.ts | 40 -- backend/test/serviceBehavior.test.ts | 176 ++++- backend/test/testDatabaseGuard.test.ts | 11 + docs/api/overview.md | 3 - docs/architecture/database.md | 107 ++- docs/architecture/gateway-and-chat.md | 3 +- docs/index.md | 8 +- docs/operations/runbooks.md | 76 +- docs/operations/scheduler-cache-backups.md | 15 +- docs/security/auth-and-trust-boundaries.md | 2 +- docs/setup/new-vps.md | 86 ++- docs/setup/production-deploy.md | 707 ++++++++----------- docs/setup/secrets-and-env.md | 50 +- systemd/mira-dashboard-worker.service | 7 +- systemd/mira-dashboard.service | 7 +- 28 files changed, 2146 insertions(+), 816 deletions(-) create mode 100644 backend/src/releaseDeployment.ts create mode 100644 backend/test/releaseDeployment.test.ts diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts new file mode 100644 index 000000000..63483a1bc --- /dev/null +++ b/backend/src/releaseDeployment.ts @@ -0,0 +1,551 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; + +import { runProcess } from "./lib/processes.ts"; +import { + type DashboardReleaseRetentionResult, + ensureDashboardReleaseLayout, + loadManagedRelease, + type ManagedDashboardRelease, + managedReleasePath, + pruneDashboardReleases, + resolveDashboardReleasesRoot, +} from "./releaseManager.ts"; +import { + loadReleaseManifest, + RELEASE_MANIFEST_FILE_NAME, + verifyReleaseArtifacts, + verifyReleaseBuildIdentities, +} from "./releaseManifest.ts"; + +const RELEASE_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; +const DEFAULT_DASHBOARD_SOURCE_ROOT = "/home/ubuntu/projects/mira-dashboard"; +const DEFAULT_DASHBOARD_WORKTREE_ROOT = "/home/ubuntu/projects/mira-dashboard-worktrees"; +const DEFAULT_DASHBOARD_STATE_ROOT = "/home/ubuntu/projects/mira-dashboard-state"; +const DEFAULT_DASHBOARD_DATABASE_PATH = `${DEFAULT_DASHBOARD_STATE_ROOT}/mira-dashboard.db`; +const DEFAULT_DASHBOARD_OPENCLAW_HOME = `${DEFAULT_DASHBOARD_STATE_ROOT}/openclaw-client`; +const DEFAULT_DASHBOARD_LOG_ROTATION_LOCK_FILE = `${DEFAULT_DASHBOARD_STATE_ROOT}/log-rotation.lock`; +const MAX_PROCESS_OUTPUT_BYTES = 20 * 1024 * 1024; +export const MANAGED_DASHBOARD_UNITS = { + "mira-dashboard-worker.service": "dist/workerStart.js", + "mira-dashboard.service": "dist/serverStart.js", +} as const; + +export interface DashboardReleaseCommandResult { + stderr: string; + stdout: string; +} + +export type DashboardReleaseCommandRunner = ( + command: string, + arguments_: readonly string[], + options: { + cwd: string; + environment: NodeJS.ProcessEnv; + signal?: AbortSignal; + timeoutMs: number; + } +) => Promise; + +export interface StageDashboardReleaseOptions { + commandRunner?: DashboardReleaseCommandRunner; + databasePath?: string; + onProgress?: (message: string) => void; + openClawHome?: string; + releasesRoot?: string; + signal?: AbortSignal; + sourceRoot?: string; + worktreeRoot?: string; +} + +export interface ManagedDashboardUnitContract { + databasePath: string; + logRotationLockFile: string; + openClawHome: string; + releaseRoot: string; + releasesRoot: string; +} + +type ManagedDashboardUnitName = keyof typeof MANAGED_DASHBOARD_UNITS; + +function assertFullCommitSha(commitSha: string): string { + if (!RELEASE_COMMIT_SHA_PATTERN.test(commitSha)) { + throw new TypeError("Release staging requires a full lowercase Git SHA"); + } + return commitSha; +} + +function resolveAbsoluteNonRootPath(value: string, label: string): string { + const trimmed = value.trim(); + if (!trimmed || trimmed.includes("\0") || !path.isAbsolute(trimmed)) { + throw new TypeError(`${label} must be an absolute non-root path`); + } + const resolved = path.resolve(trimmed); + if (resolved === path.parse(resolved).root) { + throw new TypeError(`${label} must be an absolute non-root path`); + } + return resolved; +} + +function hasExactEnvironmentAssignment( + serializedEnvironment: string, + assignment: string +): boolean { + const escaped = assignment.replaceAll(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`); + return new RegExp(String.raw`(?:^|[\s"])${escaped}(?=$|[\s"])`, "u").test( + serializedEnvironment + ); +} + +function hasExactSerializedToken(serializedValue: string, token: string): boolean { + const escaped = token.replaceAll(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`); + return new RegExp(String.raw`(?:^|[\s";])${escaped}(?=$|[\s";])`, "u").test( + serializedValue + ); +} + +async function pathExists(candidatePath: string): Promise { + try { + await fsp.lstat(candidatePath); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +} + +async function cleanupReleaseWorktree( + commandRunner: DashboardReleaseCommandRunner, + sourceRoot: string, + worktreePath: string, + environment: NodeJS.ProcessEnv, + isWorktreeCreated: boolean +): Promise { + if (!isWorktreeCreated && !(await pathExists(worktreePath))) { + return; + } + let removeError: unknown; + try { + await commandRunner("git", ["worktree", "remove", "--force", worktreePath], { + cwd: sourceRoot, + environment, + timeoutMs: 120_000, + }); + return; + } catch (error) { + removeError = error; + } + try { + await fsp.rm(worktreePath, { force: true, recursive: true }); + await commandRunner("git", ["worktree", "prune"], { + cwd: sourceRoot, + environment, + timeoutMs: 30_000, + }); + } catch (fallbackError) { + throw new AggregateError( + [removeError, fallbackError], + "Failed to remove release build worktree", + { cause: fallbackError } + ); + } +} + +async function assertRealDirectory(directoryPath: string, label: string): Promise { + const stat = await fsp.lstat(directoryPath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new TypeError(`${label} must be a real directory`); + } + if ((await fsp.realpath(directoryPath)) !== directoryPath) { + throw new TypeError(`${label} must not traverse symlinks`); + } +} + +async function syncFile(filePath: string): Promise { + const file = await fsp.open(filePath, fs.constants.O_RDONLY); + try { + await file.sync(); + } finally { + await file.close(); + } +} + +async function syncDirectory(directoryPath: string): Promise { + const directory = await fsp.open( + directoryPath, + fs.constants.O_RDONLY | fs.constants.O_DIRECTORY + ); + try { + await directory.sync(); + } finally { + await directory.close(); + } +} + +async function defaultCommandRunner( + command: string, + arguments_: readonly string[], + options: { + cwd: string; + environment: NodeJS.ProcessEnv; + signal?: AbortSignal; + timeoutMs: number; + } +): Promise { + const result = await runProcess(command, arguments_, { + cwd: options.cwd, + env: options.environment, + maxBuffer: MAX_PROCESS_OUTPUT_BYTES, + signal: options.signal, + timeoutMs: options.timeoutMs, + }); + if (result.code !== 0) { + throw new Error( + `${command} ${arguments_.join(" ")} failed with exit code ${ + result.code + }: ${result.stderr.trim() || result.stdout.trim()}` + ); + } + return { stderr: result.stderr, stdout: result.stdout }; +} + +async function copyVerifiedRelease( + buildRoot: string, + commitSha: string, + releasesRoot: string +): Promise { + const layout = await ensureDashboardReleaseLayout(releasesRoot); + const finalPath = managedReleasePath(releasesRoot, commitSha); + try { + return await loadManagedRelease(releasesRoot, commitSha); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + + const manifest = await loadReleaseManifest(buildRoot); + await verifyReleaseArtifacts(buildRoot, manifest); + await verifyReleaseBuildIdentities(buildRoot, manifest); + if (manifest.commitSha !== commitSha) { + throw new Error( + `Built release identity ${manifest.commitSha} does not match ${commitSha}` + ); + } + + const stagingPath = path.join( + layout.releasesPath, + `.staging-${commitSha}-${randomUUID()}` + ); + await fsp.mkdir(stagingPath, { mode: 0o755 }); + try { + const files = [ + ...manifest.artifacts.map((artifact) => artifact.path), + RELEASE_MANIFEST_FILE_NAME, + ]; + const createdDirectories = new Set([stagingPath]); + for (const relativePath of files) { + const sourcePath = path.join(buildRoot, relativePath); + const destinationPath = path.join(stagingPath, relativePath); + const destinationDirectory = path.dirname(destinationPath); + await fsp.mkdir(destinationDirectory, { mode: 0o755, recursive: true }); + for ( + let directory = destinationDirectory; + directory.startsWith(`${stagingPath}${path.sep}`); + directory = path.dirname(directory) + ) { + createdDirectories.add(directory); + } + await fsp.copyFile(sourcePath, destinationPath, fs.constants.COPYFILE_EXCL); + await syncFile(destinationPath); + } + + const stagedManifest = await loadReleaseManifest(stagingPath); + await verifyReleaseArtifacts(stagingPath, stagedManifest); + await verifyReleaseBuildIdentities(stagingPath, stagedManifest); + if (stagedManifest.commitSha !== commitSha) { + throw new Error( + `Staged release identity ${stagedManifest.commitSha} does not match ${commitSha}` + ); + } + const deepestFirst = [...createdDirectories].toSorted( + (left, right) => right.length - left.length + ); + for (const directory of deepestFirst) { + await syncDirectory(directory); + } + try { + await fsp.rename(stagingPath, finalPath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST" && code !== "ENOTEMPTY") { + throw error; + } + // A concurrent publisher may have won the same immutable SHA. + // Accept it only after full manifest, artifact, and identity verification. + const concurrentlyPublished = await loadManagedRelease( + releasesRoot, + commitSha + ); + await fsp.rm(stagingPath, { recursive: true }); + await syncDirectory(layout.releasesPath); + return concurrentlyPublished; + } + await syncDirectory(layout.releasesPath); + } catch (error) { + await fsp.rm(stagingPath, { force: true, recursive: true }); + throw error; + } + return loadManagedRelease(releasesRoot, commitSha); +} + +export function managedDashboardUnitContract( + releasesRoot = resolveDashboardReleasesRoot(), + databasePath = process.env.MIRA_DASHBOARD_DB_PATH ?? DEFAULT_DASHBOARD_DATABASE_PATH, + openClawHome = process.env.MIRA_DASHBOARD_OPENCLAW_HOME ?? + DEFAULT_DASHBOARD_OPENCLAW_HOME +): ManagedDashboardUnitContract { + const root = resolveAbsoluteNonRootPath(releasesRoot, "Dashboard releases root"); + return { + databasePath: resolveAbsoluteNonRootPath(databasePath, "Dashboard database path"), + logRotationLockFile: resolveAbsoluteNonRootPath( + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE ?? + DEFAULT_DASHBOARD_LOG_ROTATION_LOCK_FILE, + "Dashboard log rotation lock file" + ), + openClawHome: resolveAbsoluteNonRootPath(openClawHome, "Dashboard OpenClaw home"), + releaseRoot: path.join(root, "current"), + releasesRoot: root, + }; +} + +export function assertManagedDashboardUnitProperties( + unit: ManagedDashboardUnitName, + properties: string, + contract = managedDashboardUnitContract() +): void { + const expectedEnvironment = [ + `MIRA_DASHBOARD_DB_PATH=${contract.databasePath}`, + `MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile}`, + `MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome}`, + `MIRA_DASHBOARD_RELEASE_ROOT=${contract.releaseRoot}`, + `MIRA_DASHBOARD_RELEASES_ROOT=${contract.releasesRoot}`, + ]; + const expectedWorkingDirectory = `${contract.releaseRoot}/backend`; + const actual = new Map( + properties + .split("\n") + .filter(Boolean) + .map((line) => { + const separator = line.indexOf("="); + return separator === -1 + ? [line, ""] + : [line.slice(0, separator), line.slice(separator + 1)]; + }) + ); + if (actual.get("WorkingDirectory") !== expectedWorkingDirectory) { + throw new Error( + `${unit} must run from managed current/backend before Dashboard deployment` + ); + } + const execStart = actual.get("ExecStart") ?? ""; + if (!hasExactSerializedToken(execStart, MANAGED_DASHBOARD_UNITS[unit])) { + throw new Error(`${unit} has an unexpected managed release entrypoint`); + } + const environment = actual.get("Environment") ?? ""; + const missingEnvironment = expectedEnvironment.filter( + (entry) => !hasExactEnvironmentAssignment(environment, entry) + ); + if (missingEnvironment.length > 0) { + throw new Error( + `${unit} is missing stable managed release environment: ${missingEnvironment + .map((entry) => entry.slice(0, entry.indexOf("="))) + .join(", ")}` + ); + } +} + +export async function stageDashboardRelease( + commitSha: string, + options: StageDashboardReleaseOptions = {} +): Promise { + const expectedCommit = assertFullCommitSha(commitSha); + const releasesRoot = resolveAbsoluteNonRootPath( + options.releasesRoot ?? resolveDashboardReleasesRoot(), + "Dashboard releases root" + ); + try { + return await loadManagedRelease(releasesRoot, expectedCommit); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + + const sourceRoot = resolveAbsoluteNonRootPath( + options.sourceRoot ?? DEFAULT_DASHBOARD_SOURCE_ROOT, + "Dashboard source root" + ); + const worktreeRoot = resolveAbsoluteNonRootPath( + options.worktreeRoot ?? DEFAULT_DASHBOARD_WORKTREE_ROOT, + "Dashboard worktree root" + ); + await assertRealDirectory(sourceRoot, "Dashboard source root"); + await assertRealDirectory(worktreeRoot, "Dashboard worktree root"); + const commandRunner = options.commandRunner ?? defaultCommandRunner; + const worktreePath = path.join( + worktreeRoot, + `release-${expectedCommit.slice(0, 12)}-${randomUUID()}` + ); + const contract = managedDashboardUnitContract( + releasesRoot, + options.databasePath ?? + process.env.MIRA_DASHBOARD_DB_PATH ?? + DEFAULT_DASHBOARD_DATABASE_PATH, + options.openClawHome + ); + const environment: NodeJS.ProcessEnv = { + ...process.env, + MIRA_DASHBOARD_DB_PATH: contract.databasePath, + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE: contract.logRotationLockFile, + MIRA_DASHBOARD_OPENCLAW_HOME: contract.openClawHome, + MIRA_DASHBOARD_RELEASES_ROOT: contract.releasesRoot, + NODE_ENV: "production", + }; + let isWorktreeCreated = false; + let stagedRelease: ManagedDashboardRelease | undefined; + let stagingError: unknown; + try { + options.onProgress?.("Creating isolated release worktree"); + await commandRunner( + "git", + ["worktree", "add", "--detach", worktreePath, expectedCommit], + { + cwd: sourceRoot, + environment, + signal: options.signal, + timeoutMs: 120_000, + } + ); + isWorktreeCreated = true; + const identity = await commandRunner("git", ["rev-parse", "HEAD"], { + cwd: worktreePath, + environment, + signal: options.signal, + timeoutMs: 30_000, + }); + if (identity.stdout.trim() !== expectedCommit) { + throw new Error("Release worktree resolved an unexpected commit"); + } + + options.onProgress?.("Installing frontend release dependencies"); + await commandRunner("bun", ["install", "--frozen-lockfile"], { + cwd: worktreePath, + environment, + signal: options.signal, + timeoutMs: 180_000, + }); + options.onProgress?.("Installing backend release dependencies"); + await commandRunner("bun", ["install", "--frozen-lockfile"], { + cwd: path.join(worktreePath, "backend"), + environment, + signal: options.signal, + timeoutMs: 120_000, + }); + options.onProgress?.("Building and preflighting release"); + await commandRunner("bun", ["run", "deploy:prepare"], { + cwd: worktreePath, + environment, + signal: options.signal, + timeoutMs: 12 * 60 * 1000, + }); + options.onProgress?.("Publishing verified immutable release"); + stagedRelease = await copyVerifiedRelease( + worktreePath, + expectedCommit, + contract.releasesRoot + ); + } catch (error) { + stagingError = error; + } + let cleanupError: unknown; + try { + await cleanupReleaseWorktree( + commandRunner, + sourceRoot, + worktreePath, + environment, + isWorktreeCreated + ); + } catch (error) { + cleanupError = error; + } + if (stagingError !== undefined) { + if (cleanupError !== undefined) { + throw new AggregateError( + [stagingError, cleanupError], + "Release staging and worktree cleanup both failed", + { cause: stagingError } + ); + } + throw stagingError; + } + if (cleanupError !== undefined) { + throw cleanupError; + } + if (!stagedRelease) { + throw new Error("Release staging completed without a published release"); + } + return stagedRelease; +} + +export async function prunePublishedDashboardReleases( + retainCount = 3, + releasesRoot = resolveDashboardReleasesRoot() +): Promise { + return pruneDashboardReleases(retainCount, releasesRoot); +} + +export async function runReleaseDeploymentCommand( + arguments_: string[], + releasesRoot = resolveDashboardReleasesRoot() +) { + const [command, value, ...extra] = arguments_; + if (extra.length > 0) { + throw new TypeError("Release deployment command received unexpected arguments"); + } + if (command === "stage") { + if (!value) { + throw new TypeError("Release deployment stage requires a commit SHA"); + } + const release = await stageDashboardRelease(value, { releasesRoot }); + return { + commitSha: release.commitSha, + commitTitle: release.manifest.commitTitle, + path: release.path, + }; + } + if (command === "prune") { + const retainCount = value === undefined ? 3 : Number(value); + return prunePublishedDashboardReleases(retainCount, releasesRoot); + } + throw new TypeError( + "Usage: releaseDeployment.ts " + ); +} + +if (import.meta.main) { + try { + const result = await runReleaseDeploymentCommand(Bun.argv.slice(2)); + console.log(JSON.stringify(result)); + } catch (error) { + console.error( + error instanceof Error ? error.message : "Release deployment failed" + ); + process.exitCode = 1; + } +} diff --git a/backend/src/releaseLifecycle.ts b/backend/src/releaseLifecycle.ts index 84036f47c..ff7de4d5c 100644 --- a/backend/src/releaseLifecycle.ts +++ b/backend/src/releaseLifecycle.ts @@ -4,6 +4,7 @@ import type { } from "./releaseManager.ts"; import { activateDashboardRelease, + pruneDashboardReleases, readDashboardReleaseState, resolveDashboardReleasesRoot, rollbackDashboardRelease, @@ -70,9 +71,13 @@ export async function runReleaseLifecycleCommand( state = await readDashboardReleaseState(releasesRoot); break; } + case "prune": { + const retainCount = commitSha === undefined ? 3 : Number(commitSha); + return pruneDashboardReleases(retainCount, releasesRoot); + } default: { throw new TypeError( - "Usage: releaseLifecycle.js " + "Usage: releaseLifecycle.js " ); } } diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index 401c3dfa8..3ab686b87 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -29,6 +29,8 @@ const RELEASE_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; const RELEASE_TRANSITION_FORMAT_VERSION = 1; const RELEASE_TRANSITION_JOURNAL_FILE_NAME = ".release-transition.json"; export const RELEASE_TRANSITION_LOCK_FILE_NAME = ".release-transition.lock"; +const RETIRED_RELEASE_DIRECTORY_PATTERN = + /^\.retired-[\da-f]{40}-[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; const MAX_RELEASE_TRANSITION_FILE_BYTES = 4096; export const RELEASE_TRANSITION_LOCK_PROGRAM = "/usr/bin/flock"; @@ -53,6 +55,11 @@ export interface DashboardReleaseState { root: string; } +export interface DashboardReleaseRetentionResult { + removed: string[]; + retained: string[]; +} + export interface DashboardReleaseManagerOptions { readLiveSchemaState?: ( maximumCompatibleVersion: number @@ -397,6 +404,17 @@ function releaseDirectoryIdentity(stat: fs.BigIntStats): string { return [stat.dev, stat.ino, stat.ctimeNs, stat.birthtimeNs].join(":"); } +function isSameReleaseDirectoryInode( + left: fs.BigIntStats, + right: fs.BigIntStats +): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.birthtimeNs === right.birthtimeNs + ); +} + function releaseManifestIdentity(manifest: DashboardReleaseManifest): string { const canonicalManifest = { artifacts: manifest.artifacts @@ -630,12 +648,6 @@ function assertReleaseMigrationHistoryCompatible( action: "Activation" | "Rollback" ): void { const expectedMigrations = release.schema.migrations; - if (!expectedMigrations) { - // Only temporary format-v1 manifests omit migration identities. They stay - // readable for the first managed cutover rollback window, but cannot bind - // activation or rollback to the exact applied migration history. - return; - } for (const actual of liveState.migrations.slice(0, release.schema.target)) { const expected = expectedMigrations[actual.version - 1]; if ( @@ -1195,3 +1207,111 @@ export async function rollbackDashboardRelease( }); }); } + +export async function pruneDashboardReleases( + retainCount = 3, + releasesRoot = resolveDashboardReleasesRoot() +): Promise { + if (!Number.isSafeInteger(retainCount) || retainCount < 2 || retainCount > 20) { + throw new TypeError("Managed release retention must be between 2 and 20"); + } + + const layout = await ensureDashboardReleaseLayout(releasesRoot); + return withReleaseTransitionLock(layout, "exclusive", async () => { + await recoverInterruptedReleaseTransition(layout); + const state = await readDashboardReleaseStateFromLayout(layout); + const protectedCommits = new Set( + [state.current?.commitSha, state.previous?.commitSha].filter( + (commitSha): commitSha is string => commitSha !== undefined + ) + ); + const entries = await fsp.readdir(layout.releasesPath, { + withFileTypes: true, + }); + let hasFilesystemChanges = false; + const releases: ManagedDashboardRelease[] = []; + for (const entry of entries) { + if (RETIRED_RELEASE_DIRECTORY_PATTERN.test(entry.name)) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new TypeError( + `Retired release entry must be a real directory: ${entry.name}` + ); + } + await fsp.rm(path.join(layout.releasesPath, entry.name), { + recursive: true, + }); + hasFilesystemChanges = true; + continue; + } + if (!RELEASE_COMMIT_SHA_PATTERN.test(entry.name)) { + continue; + } + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new TypeError( + `Managed release entry must be a real directory: ${entry.name}` + ); + } + releases.push(await loadManagedReleaseFromLayout(layout, entry.name)); + } + + const newestFirst = releases.toSorted((left, right) => { + const builtAtComparison = right.manifest.builtAt.localeCompare( + left.manifest.builtAt + ); + return builtAtComparison || right.commitSha.localeCompare(left.commitSha); + }); + const retained = new Set(protectedCommits); + for (const release of newestFirst) { + if (retained.size >= retainCount) { + break; + } + retained.add(release.commitSha); + } + + const removed: string[] = []; + for (const release of newestFirst.toReversed()) { + if (retained.has(release.commitSha)) { + continue; + } + const currentStat = await fsp.lstat(release.path, { bigint: true }); + if ( + !currentStat.isDirectory() || + currentStat.isSymbolicLink() || + releaseDirectoryIdentity(currentStat) !== release.directoryIdentity + ) { + throw new Error( + `Managed release changed before retention cleanup: ${release.commitSha}` + ); + } + const retiredPath = path.join( + layout.releasesPath, + `.retired-${release.commitSha}-${randomUUID()}` + ); + await fsp.rename(release.path, retiredPath); + await syncDirectory(layout.releasesPath); + const retiredStat = await fsp.lstat(retiredPath, { bigint: true }); + if ( + !retiredStat.isDirectory() || + retiredStat.isSymbolicLink() || + !isSameReleaseDirectoryInode(currentStat, retiredStat) + ) { + throw new Error( + `Managed release changed during retention cleanup: ${release.commitSha}` + ); + } + await fsp.rm(retiredPath, { recursive: true }); + hasFilesystemChanges = true; + removed.push(release.commitSha); + } + if (hasFilesystemChanges) { + await syncDirectory(layout.releasesPath); + } + + return { + removed, + retained: newestFirst + .filter((release) => retained.has(release.commitSha)) + .map((release) => release.commitSha), + }; + }); +} diff --git a/backend/src/releaseManifest.ts b/backend/src/releaseManifest.ts index 0f7569a79..0dd4afafe 100644 --- a/backend/src/releaseManifest.ts +++ b/backend/src/releaseManifest.ts @@ -26,7 +26,7 @@ const RELEASE_STATIC_ARTIFACTS = [ "bun.lock", "package.json", ] as const; -const FORMAT_2_REQUIRED_RELEASE_ARTIFACTS = [ +const REQUIRED_RELEASE_ARTIFACTS = [ ...RELEASE_STATIC_ARTIFACTS, "backend/dist/build-identity.json", "backend/dist/databasePreflight.js", @@ -37,11 +37,6 @@ const FORMAT_2_REQUIRED_RELEASE_ARTIFACTS = [ "dist/build-identity.json", "dist/index.html", ] as const; -// Format 1 remains readable only for the first managed cutover and its rollback -// window. Remove this compatibility list once current/previous cannot reference v1. -const FORMAT_1_REQUIRED_RELEASE_ARTIFACTS = FORMAT_2_REQUIRED_RELEASE_ARTIFACTS.filter( - (artifactPath) => artifactPath !== "backend/dist/releaseLifecycle.js" -); const MAX_BUILD_IDENTITY_BYTES = 4096; const RUNTIME_RELEASE_VERIFICATION_CACHE_MS = 15_000; const SHA_256_PATTERN = /^[\da-f]{64}$/u; @@ -65,11 +60,11 @@ export interface DashboardReleaseManifest { backendCommit: string; frontendCommit: string; }; - formatVersion: 1 | 2; + formatVersion: 2; schema: { maximumCompatible: number; - migrations?: DatabaseMigrationIdentity[]; - migrationInventorySha256?: string; + migrations: DatabaseMigrationIdentity[]; + migrationInventorySha256: string; migrationRegistrySha256: string; minimumCompatible: number; target: number; @@ -512,16 +507,14 @@ function parseMigrationIdentity(value: unknown): DatabaseMigrationIdentity { }; } -function parseSchema( - value: unknown, - formatVersion: DashboardReleaseManifest["formatVersion"] -): DashboardReleaseManifest["schema"] { +function parseSchema(value: unknown): DashboardReleaseManifest["schema"] { const expectedKeys = [ "maximumCompatible", + "migrations", + "migrationInventorySha256", "migrationRegistrySha256", "minimumCompatible", "target", - ...(formatVersion === 2 ? ["migrations", "migrationInventorySha256"] : []), ]; if (!isPlainRecord(value) || !hasExactKeys(value, expectedKeys)) { throw new TypeError("Release manifest schema declaration is invalid"); @@ -539,30 +532,25 @@ function parseSchema( ) { throw new TypeError("Release manifest schema range is invalid"); } - const migrations = - formatVersion === 2 && Array.isArray(value.migrations) - ? value.migrations.map((migration) => parseMigrationIdentity(migration)) - : undefined; + const migrations = Array.isArray(value.migrations) + ? value.migrations.map((migration) => parseMigrationIdentity(migration)) + : undefined; // This digest proves only that a foreign manifest is internally consistent. // Runtime and release-manager validation bind it to local code and live history. if ( - formatVersion === 2 && - (!migrations || - migrations.length !== (target as number) || - migrations.some((migration, index) => migration.version !== index + 1) || - typeof value.migrationInventorySha256 !== "string" || - !SHA_256_PATTERN.test(value.migrationInventorySha256) || - value.migrationInventorySha256 !== - databaseMigrationInventorySha256(migrations)) + !migrations || + migrations.length !== (target as number) || + migrations.some((migration, index) => migration.version !== index + 1) || + typeof value.migrationInventorySha256 !== "string" || + !SHA_256_PATTERN.test(value.migrationInventorySha256) || + value.migrationInventorySha256 !== databaseMigrationInventorySha256(migrations) ) { throw new TypeError("Release manifest migration inventory is invalid"); } return { maximumCompatible: maximumCompatible as number, - ...(migrations && { migrations }), - ...(formatVersion === 2 && { - migrationInventorySha256: value.migrationInventorySha256 as string, - }), + migrations, + migrationInventorySha256: value.migrationInventorySha256 as string, migrationRegistrySha256: value.migrationRegistrySha256, minimumCompatible: minimumCompatible as number, target: target as number, @@ -583,8 +571,7 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { "formatVersion", "schema", ]) || - (value.formatVersion !== 1 && - value.formatVersion !== RELEASE_MANIFEST_FORMAT_VERSION) || + value.formatVersion !== RELEASE_MANIFEST_FORMAT_VERSION || typeof value.commitSha !== "string" || typeof value.commitShort !== "string" || typeof value.commitTitle !== "string" || @@ -620,10 +607,9 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { artifactPaths.some( (artifactPath_, index) => artifactPath_ !== sortedArtifactPaths[index] ) || - (value.formatVersion === 1 - ? FORMAT_1_REQUIRED_RELEASE_ARTIFACTS - : FORMAT_2_REQUIRED_RELEASE_ARTIFACTS - ).some((requiredPath) => !artifactPaths.includes(requiredPath)) + REQUIRED_RELEASE_ARTIFACTS.some( + (requiredPath) => !artifactPaths.includes(requiredPath) + ) ) { throw new TypeError("Release manifest artifact inventory is invalid"); } @@ -640,7 +626,7 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { frontendCommit: value.components.frontendCommit as string, }, formatVersion: value.formatVersion, - schema: parseSchema(value.schema, value.formatVersion), + schema: parseSchema(value.schema), }; } @@ -774,12 +760,8 @@ export async function loadRuntimeReleaseIdentity( DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum && manifest.schema.maximumCompatible === DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum && - // Format v1 has no migration inventory, so readiness cannot bind it to - // local migration identities. Remove this branch after the first - // managed-cutover rollback window no longer contains a v1 release. - (manifest.formatVersion === 1 || - manifest.schema.migrationInventorySha256 === - databaseMigrationInventorySha256()) && + manifest.schema.migrationInventorySha256 === + databaseMigrationInventorySha256() && manifest.schema.migrationRegistrySha256 === databaseMigrationRegistrySha256(); return { artifactCount: manifest.artifacts.length, diff --git a/backend/src/requestPolicy.ts b/backend/src/requestPolicy.ts index 2ceb93734..317edab29 100644 --- a/backend/src/requestPolicy.ts +++ b/backend/src/requestPolicy.ts @@ -83,7 +83,6 @@ const BUCKET_CLEANUP_INTERVAL_MS = 60_000; const BUCKET_STALE_MS = Math.max(apiRule.windowMs, authRule.windowMs) * 2; const SAFE_REQUEST_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); const PUBLIC_API_METHODS = new Map>([ - ["/api/health", new Set(["GET", "HEAD"])], ["/api/health/live", new Set(["GET", "HEAD"])], ["/api/health/ready", new Set(["GET", "HEAD"])], ["/api/auth/bootstrap", new Set(["GET", "HEAD"])], diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 78584326c..93005148f 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -40,50 +40,18 @@ async function ready() { return json(snapshot, { status: snapshot.status === "isReady" ? 200 : 503 }); } -async function legacyReady() { - const snapshot = await readinessSnapshot(); - const isReady = snapshot.status === "isReady"; - return json( - { - status: isReady ? "isOk" : "notReady", - workerOnline: snapshot.checks.worker.ready, - }, - { status: isReady ? 200 : 503 } - ); -} - async function diagnostics() { return json(await diagnosticsSnapshot()); } -function retiredHealth() { - return json( - { - error: "Gone", - replacements: ["/api/health/live", "/api/health/ready"], - }, - { status: 410 } - ); -} - function sessions() { return json(gateway.getSessions()); } const routeTable = { - "/health": { - GET: retiredHealth, - HEAD: retiredHealth, - }, "/api/health/diagnostics": { GET: diagnostics, }, - // Transitional compatibility for the in-flight pre-readiness deploy - // executor. Remove after the atomic release executor has completed cutover. - "/api/health": { - GET: legacyReady, - HEAD: legacyReady, - }, "/api/health/live": { GET: live, HEAD: live, diff --git a/backend/src/services/logRotation.ts b/backend/src/services/logRotation.ts index 0a365dee9..a91af96dc 100644 --- a/backend/src/services/logRotation.ts +++ b/backend/src/services/logRotation.ts @@ -68,7 +68,22 @@ const ELEVATED_LOG_ROTATION_MAX_BUFFER = 16 * 1024 * 1024; const LOG_ROTATION_JOB_ID = "ops.log-rotation"; const LOG_ROTATION_FAILURE_OUTPUT_MAX_CHARS = 100_000; const BUN_EXECUTABLE = process.env.BUN_BINARY || "bun"; -const logRotationLockFile = DEFAULT_LOCK_FILE; +function resolveLogRotationLockFile(): string { + const configured = + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE?.trim() || DEFAULT_LOCK_FILE; + if ( + configured.includes("\0") || + !path.isAbsolute(configured) || + path.resolve(configured) === path.parse(path.resolve(configured)).root + ) { + throw new TypeError( + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE must be an absolute non-root path" + ); + } + return path.resolve(configured); +} + +const logRotationLockFile = resolveLogRotationLockFile(); type ExecFileRunner = ( file: string, @@ -2031,9 +2046,17 @@ function buildElevatedLogRotationCliArguments( } function elevatedLogRotationEnvironment(): NodeJS.ProcessEnv { - const allowed = ["PATH", "HOME", "LANG", "NODE_ENV", "TZ", "MIRA_DASHBOARD_DB_PATH"]; + const allowed = [ + "PATH", + "HOME", + "LANG", + "NODE_ENV", + "TZ", + "MIRA_DASHBOARD_DB_PATH", + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", + ]; const environment: NodeJS.ProcessEnv = {}; - // Keep sudo -E narrow: only runtime lookup, home/locale, mode, and database path. + // Keep sudo -E narrow: only runtime lookup, home/locale, mode, and stable state paths. for (const key of allowed) { if (process.env[key] !== undefined) { environment[key] = process.env[key]; diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 17c91dc66..987241ec4 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -1,4 +1,3 @@ -import { rm } from "node:fs/promises"; import path from "node:path"; import { database, getMiraDatabasePath, sqlNullable } from "../database.ts"; @@ -10,6 +9,18 @@ import { spawnProcess, } from "../lib/processes.ts"; import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; +import { + assertManagedDashboardUnitProperties, + MANAGED_DASHBOARD_UNITS, + managedDashboardUnitContract, + stageDashboardRelease, +} from "../releaseDeployment.ts"; +import { + activateDashboardRelease, + readDashboardReleaseState, + resolveDashboardReleasesRoot, + rollbackDashboardRelease, +} from "../releaseManager.ts"; import { enqueueJobExecution, type JobExecution, @@ -1450,42 +1461,102 @@ try { ].join(" "); } -/** Performs schedule restart health check. */ -async function scheduleRestartHealthCheck( +async function assertManagedDashboardServiceContract( + signal?: AbortSignal +): Promise { + const contract = managedDashboardUnitContract(); + for (const unit of Object.keys(MANAGED_DASHBOARD_UNITS) as Array< + keyof typeof MANAGED_DASHBOARD_UNITS + >) { + const { stdout } = await runCommand( + "systemctl", + [ + "--user", + "show", + unit, + "--property=Environment", + "--property=ExecStart", + "--property=WorkingDirectory", + ], + { signal, timeoutMs: 30_000 } + ); + assertManagedDashboardUnitProperties(unit, stdout, contract); + } +} + +/** Schedules detached service restart, commit-bound readiness, and rollback. */ +async function scheduleReleaseCutover( job: DeploymentJob, + rollbackCommit: string, signal?: AbortSignal ): Promise { + if (!job.commit || !/^[\da-f]{8}$/u.test(job.commit)) { + throw new TypeError("Release cutover requires an eight-character commit"); + } + if (!/^[\da-f]{8}$/u.test(rollbackCommit)) { + throw new TypeError( + "Release cutover requires an eight-character rollback commit" + ); + } + const releasesRoot = resolveDashboardReleasesRoot(); + const releaseRoot = path.join(releasesRoot, "current"); + const lifecycleCommand = path.join( + releaseRoot, + "backend", + "dist", + "releaseLifecycle.js" + ); const okJob: DeploymentJob = { ...job, status: "isOk", updatedAt: dateToISOString(new Date()), - note: "Restarted web and worker services. Health checks passed", + note: "Atomic release activated. Web, worker, and commit readiness passed", + }; + const okWithRetentionWarningJob: DeploymentJob = { + ...okJob, + note: "Atomic release activated and ready; release retention cleanup failed", + }; + const rolledBackJob: DeploymentJob = { + ...job, + status: "failed", + updatedAt: dateToISOString(new Date()), + note: `Release readiness failed; automatic rollback restored ${rollbackCommit}`, }; - const failedJob: DeploymentJob = { + const rollbackFailedJob: DeploymentJob = { ...job, status: "failed", updatedAt: dateToISOString(new Date()), - note: "Restart was triggered, but health check failed", + note: `Release readiness failed and automatic rollback to ${rollbackCommit} failed`, }; const script = [ "sleep 2", - "restart_status=0", - `systemctl --user restart ${DASHBOARD_SERVICES.join(" ")} || restart_status=$?`, - "health_status=1", - 'if [ "$restart_status" -eq 0 ]; then', - " for attempt in {1..20}; do", - " if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 http://127.0.0.1:3100/api/health/ready >/dev/null; then", - " health_status=0", - " break", + "ready_for_commit() {", + ' expected_commit="$1"', + " for attempt in {1..30}; do", + " response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 http://127.0.0.1:3100/api/health/ready 2>/dev/null || true)", + ' if printf "%s" "$response" | /usr/bin/jq --exit-status --arg expected "$expected_commit" \'.status == "isReady" and .checks.release.ready == true and .checks.release.backendCommit == $expected and .checks.release.frontendCommit == $expected and .checks.worker.ready == true\' >/dev/null 2>&1; then', + " return 0", " fi", " sleep 1", " done", - "fi", - `if [ "$restart_status" -eq 0 ] && [ "$health_status" -eq 0 ]; then`, - ` ${deploymentJobUpdateCommand(okJob)}`, + " return 1", + "}", + "restart_services() {", + ` /usr/bin/systemctl --user restart ${DASHBOARD_SERVICES.join(" ")}`, + "}", + `if restart_services && ready_for_commit ${shellQuote(job.commit)}; then`, + ` if MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)} MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())} NODE_ENV=production ${shellQuote(resolveBunExecutable())} ${shellQuote(lifecycleCommand)} prune 3; then`, + ` ${deploymentJobUpdateCommand(okJob)}`, + " else", + ` ${deploymentJobUpdateCommand(okWithRetentionWarningJob)}`, + " fi", "else", - ` ${deploymentJobUpdateCommand(failedJob)}`, + ` if MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)} MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())} NODE_ENV=production ${shellQuote(resolveBunExecutable())} ${shellQuote(lifecycleCommand)} rollback && restart_services && ready_for_commit ${shellQuote(rollbackCommit)}; then`, + ` ${deploymentJobUpdateCommand(rolledBackJob)}`, + " else", + ` ${deploymentJobUpdateCommand(rollbackFailedJob)}`, + " fi", "fi", ].join("\n"); @@ -1494,7 +1565,7 @@ async function scheduleRestartHealthCheck( [ "--user", `--unit=mira-dashboard-deploy-${job.id}`, - "--description=Mira Dashboard deploy restart + health check", + "--description=Mira Dashboard atomic release cutover", "/bin/bash", "-lc", script, @@ -1510,61 +1581,87 @@ async function runDeploymentJob( ): Promise { let currentJob = job; const dashboardRoot = getDashboardRoot(); + const releasesRoot = resolveDashboardReleasesRoot(); try { currentJob = refreshDeploymentHeartbeat(currentJob); await syncMain(signal); currentJob = refreshDeploymentHeartbeat(currentJob); - + await assertManagedDashboardServiceContract(signal); currentJob = refreshDeploymentHeartbeat(currentJob); - await rm(path.join(dashboardRoot, "node_modules"), { - force: true, - recursive: true, - }); - await runCommand("bun", ["install", "--frozen-lockfile"], { - signal, - timeoutMs: 180_000, - }); + const currentState = await readDashboardReleaseState(releasesRoot); + if (!currentState.current) { + throw new Error( + "Managed deployment requires a current release from the one-time production cutover" + ); + } currentJob = refreshDeploymentHeartbeat(currentJob); - await rm(path.join(dashboardRoot, "backend", "node_modules"), { - force: true, - recursive: true, - }); - await runCommand("bun", ["install", "--frozen-lockfile"], { - cwd: path.join(dashboardRoot, "backend"), + const { stdout: commitSha } = await runCommand("git", ["rev-parse", "HEAD"], { signal, - timeoutMs: 120_000, + timeoutMs: 30_000, }); - currentJob = refreshDeploymentHeartbeat(currentJob); - await runCommand("bun", ["run", "deploy:prepare"], { + const expectedCommit = commitSha.trim(); + const candidate = await stageDashboardRelease(expectedCommit, { + commandRunner: async (command, arguments_, options) => + runCommand(command, [...arguments_], { + cwd: options.cwd, + environment: options.environment, + signal: options.signal, + timeoutMs: options.timeoutMs, + }), + databasePath: getMiraDatabasePath(), + onProgress: () => { + currentJob = refreshDeploymentHeartbeat(currentJob); + }, + releasesRoot, signal, - timeoutMs: 12 * 60 * 1000, + sourceRoot: dashboardRoot, + worktreeRoot: getDashboardWorktreeRoot(), }); currentJob = refreshDeploymentHeartbeat(currentJob); - const { stdout: commit } = await runCommand( - "git", - ["rev-parse", "--short", "HEAD"], - { - signal, - timeoutMs: 30_000, + const activated = await activateDashboardRelease(expectedCommit, releasesRoot); + const didActivateNewRelease = currentState.current.commitSha !== expectedCommit; + if ( + activated.current?.commitSha !== expectedCommit || + !activated.previous || + activated.previous.commitSha === expectedCommit + ) { + if (didActivateNewRelease) { + await rollbackDashboardRelease(releasesRoot); } - ); - const { stdout: commitTitle } = await runCommand( - "git", - ["log", "-1", "--pretty=%s"], - { signal, timeoutMs: 30_000 } - ); - currentJob = refreshDeploymentHeartbeat(currentJob); + throw new Error( + "Managed release activation did not preserve a distinct rollback release" + ); + } - const restartScheduled: DeploymentJob = { - ...currentJob, - status: "restart-scheduled", - updatedAt: dateToISOString(new Date()), - commit: commit.trim(), - commitTitle: commitTitle.trim(), - note: "Build passed. Restart + health check scheduled", - }; - writeDeploymentJob(restartScheduled); - await scheduleRestartHealthCheck(restartScheduled, signal); + try { + const restartScheduled: DeploymentJob = { + ...currentJob, + status: "restart-scheduled", + updatedAt: dateToISOString(new Date()), + commit: candidate.manifest.commitShort, + commitTitle: candidate.manifest.commitTitle, + note: "Immutable release published. Atomic restart and rollback check scheduled", + }; + writeDeploymentJob(restartScheduled); + await scheduleReleaseCutover( + restartScheduled, + activated.previous.manifest.commitShort, + signal + ); + } catch (error) { + if (didActivateNewRelease) { + try { + await rollbackDashboardRelease(releasesRoot); + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + "Deployment scheduling failed and release-link rollback also failed", + { cause: rollbackError } + ); + } + } + throw error; + } return true; } catch (error) { const failed: DeploymentJob = { diff --git a/backend/test/bunNativeServerBehavior.test.ts b/backend/test/bunNativeServerBehavior.test.ts index c9594976b..67544748e 100644 --- a/backend/test/bunNativeServerBehavior.test.ts +++ b/backend/test/bunNativeServerBehavior.test.ts @@ -513,17 +513,6 @@ describe("Bun-native dashboard backend", () => { }); it("serves the app shell and hashed static assets", async () => { - const retiredHealth = await fetch(`${state.baseUrl}/health`); - expect(retiredHealth.status).toBe(410); - await expect(retiredHealth.json()).resolves.toEqual({ - error: "Gone", - replacements: ["/api/health/live", "/api/health/ready"], - }); - const retiredHealthHead = await fetch(`${state.baseUrl}/health`, { - method: "HEAD", - }); - expect(retiredHealthHead.status).toBe(410); - const appRoute = await fetch(`${state.baseUrl}/tasks`); expect(appRoute.status).toBe(200); expect(appRoute.headers.get("content-type")).toContain("text/html"); diff --git a/backend/test/healthReadiness.test.ts b/backend/test/healthReadiness.test.ts index d8c1d0631..2e037b57f 100644 --- a/backend/test/healthReadiness.test.ts +++ b/backend/test/healthReadiness.test.ts @@ -19,7 +19,7 @@ function readySignals(): ReadinessSignals { backendCommit: "aaaaaaaa", commitSha: "a".repeat(40), frontendCommit: "aaaaaaaa", - manifestFormatVersion: 1, + manifestFormatVersion: 2, ready: true, source: "manifest", }, @@ -43,7 +43,7 @@ describe("Dashboard readiness contract", () => { expect(ready.checks.release).toEqual({ backendCommit: "aaaaaaaa", frontendCommit: "aaaaaaaa", - manifestFormatVersion: 1, + manifestFormatVersion: 2, ready: true, source: "manifest", }); diff --git a/backend/test/httpApiBehavior.test.ts b/backend/test/httpApiBehavior.test.ts index 1c0744dab..030202614 100644 --- a/backend/test/httpApiBehavior.test.ts +++ b/backend/test/httpApiBehavior.test.ts @@ -330,22 +330,6 @@ describe("Mira Dashboard backend integration", () => { }); expect(readyHead).toEqual({ body: undefined, status: 503 }); - const legacyReady = await api<{ - status: string; - workerOnline: boolean; - }>("/api/health"); - expect(legacyReady).toEqual({ - body: { - status: "notReady", - workerOnline: false, - }, - status: 503, - }); - const legacyReadyHead = await api("/api/health", { - method: "HEAD", - }); - expect(legacyReadyHead).toEqual({ body: undefined, status: 503 }); - const diagnostics = await api<{ error: string }>("/api/health/diagnostics"); expect(diagnostics.status).toBe(401); expect(diagnostics.body).toEqual({ error: "Unauthorized" }); diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts new file mode 100644 index 000000000..c221180b8 --- /dev/null +++ b/backend/test/releaseDeployment.test.ts @@ -0,0 +1,535 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "bun:test"; + +import { + assertManagedDashboardUnitProperties, + type DashboardReleaseCommandRunner, + managedDashboardUnitContract, + runReleaseDeploymentCommand, + stageDashboardRelease, +} from "../src/releaseDeployment.ts"; +import { managedReleasePath } from "../src/releaseManager.ts"; +import { writeReleaseManifest } from "../src/releaseManifest.ts"; + +const COMMIT_SHA = "a".repeat(40); +const OTHER_COMMIT_SHA = "b".repeat(40); +const temporaryRoots: string[] = []; + +function temporaryRoot(label: string): string { + const root = mkdtempSync(path.join(tmpdir(), `${label}-`)); + temporaryRoots.push(root); + return root; +} + +async function createBuiltRelease( + releaseRoot: string, + commitSha = COMMIT_SHA +): Promise { + mkdirSync(path.join(releaseRoot, "backend", "config"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "backend", "dist"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "dist", "assets"), { recursive: true }); + writeFileSync(path.join(releaseRoot, "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "bun.lock"), "root-lock\n"); + writeFileSync(path.join(releaseRoot, "backend", "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "backend", "bun.lock"), "backend-lock\n"); + writeFileSync( + path.join(releaseRoot, "backend", "config", "log-rotation.json"), + '{"jobs":[]}\n' + ); + writeFileSync(path.join(releaseRoot, "dist", "index.html"), "
ok
\n"); + writeFileSync( + path.join(releaseRoot, "dist", "assets", "app.js"), + "export const ok = true;\n" + ); + writeFileSync( + path.join(releaseRoot, "dist", "build-identity.json"), + `${JSON.stringify({ + bunVersion: Bun.version, + commitSha, + component: "frontend", + formatVersion: 1, + })}\n` + ); + writeFileSync( + path.join(releaseRoot, "backend", "dist", "build-identity.json"), + `${JSON.stringify({ + bunVersion: Bun.version, + commitSha, + component: "backend", + formatVersion: 1, + })}\n` + ); + for (const entrypoint of [ + "databasePreflight", + "releaseLifecycle", + "resetDashboardPassword", + "serverStart", + "workerStart", + ]) { + writeFileSync( + path.join(releaseRoot, "backend", "dist", `${entrypoint}.js`), + `export const commit = "${commitSha}";\n` + ); + } + writeFileSync(path.join(releaseRoot, "not-a-release-artifact.txt"), "ignore me\n"); + await writeReleaseManifest({ + builtAt: new Date("2026-07-26T01:30:00.000Z"), + commitSha, + commitTitle: `Release ${commitSha.slice(0, 8)}`, + releaseRoot, + }); +} + +function stagingOptions() { + const base = temporaryRoot("mira-release-deployment-test"); + const sourceRoot = path.join(base, "source"); + const worktreeRoot = path.join(base, "worktrees"); + const releasesRoot = path.join(base, "managed"); + const databasePath = path.join(base, "state", "mira-dashboard.db"); + const openClawHome = path.join(base, "state", "openclaw-client"); + mkdirSync(sourceRoot); + mkdirSync(worktreeRoot); + mkdirSync(path.dirname(databasePath)); + return { + databasePath, + openClawHome, + releasesRoot, + sourceRoot, + worktreeRoot, + }; +} + +afterEach(() => { + for (const root of temporaryRoots) { + rmSync(root, { force: true, recursive: true }); + } + temporaryRoots.length = 0; +}); + +describe("immutable release deployment", () => { + it("keeps tracked production state outside source and release directories", () => { + for (const unitName of [ + "mira-dashboard.service", + "mira-dashboard-worker.service", + ]) { + const unit = readFileSync( + path.resolve(import.meta.dirname, "../../systemd", unitName), + "utf8" + ); + expect(unit).toContain( + "WorkingDirectory=/home/ubuntu/projects/mira-dashboard-releases/current/backend" + ); + expect(unit).toContain( + "MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db" + ); + expect(unit).toContain( + "MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client" + ); + expect(unit).toContain( + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock" + ); + expect(unit).not.toContain( + "/home/ubuntu/projects/mira-dashboard/backend/data" + ); + } + }); + + it("builds in an isolated worktree and atomically publishes only artifacts", async () => { + const options = stagingOptions(); + const calls: Array<{ + arguments_: readonly string[]; + command: string; + cwd: string; + }> = []; + const progress: string[] = []; + const runner: DashboardReleaseCommandRunner = async ( + command, + arguments_, + commandOptions + ) => { + calls.push({ arguments_, command, cwd: commandOptions.cwd }); + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + const worktreePath = String(arguments_[3]); + mkdirSync(worktreePath); + await createBuiltRelease(worktreePath); + } else if (arguments_[1] === "remove") { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" + ? `${COMMIT_SHA}\n` + : "", + }; + }; + + const release = await stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + onProgress: (message) => { + progress.push(message); + }, + }); + + expect(release.commitSha).toBe(COMMIT_SHA); + expect(release.path).toBe(managedReleasePath(options.releasesRoot, COMMIT_SHA)); + expect(existsSync(path.join(release.path, "dist", "index.html"))).toBe(true); + expect( + existsSync(path.join(release.path, "backend", "config", "log-rotation.json")) + ).toBe(true); + expect(existsSync(path.join(release.path, "not-a-release-artifact.txt"))).toBe( + false + ); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + expect(calls.map(({ command, arguments_ }) => [command, ...arguments_])).toEqual([ + ["git", "worktree", "add", "--detach", expect.any(String), COMMIT_SHA], + ["git", "rev-parse", "HEAD"], + ["bun", "install", "--frozen-lockfile"], + ["bun", "install", "--frozen-lockfile"], + ["bun", "run", "deploy:prepare"], + ["git", "worktree", "remove", "--force", expect.any(String)], + ]); + expect(progress).toEqual([ + "Creating isolated release worktree", + "Installing frontend release dependencies", + "Installing backend release dependencies", + "Building and preflighting release", + "Publishing verified immutable release", + ]); + }); + + it("reuses an already verified immutable release without running commands", async () => { + const options = stagingOptions(); + const buildRoot = path.join(options.worktreeRoot, "prepared"); + mkdirSync(buildRoot); + await createBuiltRelease(buildRoot); + const initialRunner: DashboardReleaseCommandRunner = async ( + command, + arguments_ + ) => { + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + const worktreePath = String(arguments_[3]); + mkdirSync(worktreePath); + await createBuiltRelease(worktreePath); + } else { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" ? COMMIT_SHA : "", + }; + }; + await stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: initialRunner, + }); + + const reused = await stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: async () => { + throw new Error("command runner should not be called"); + }, + }); + + expect(reused.commitSha).toBe(COMMIT_SHA); + }); + + it("accepts a concurrently published copy of the same verified release", async () => { + const options = stagingOptions(); + let buildsReady = 0; + const { promise: buildsReleased, resolve: releaseBuilds } = + Promise.withResolvers(); + const runner: DashboardReleaseCommandRunner = async (command, arguments_) => { + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + const worktreePath = String(arguments_[3]); + mkdirSync(worktreePath); + await createBuiltRelease(worktreePath); + } else { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + } + if ( + command === "bun" && + arguments_[0] === "run" && + arguments_[1] === "deploy:prepare" + ) { + buildsReady += 1; + if (buildsReady === 2) { + releaseBuilds(); + } + await buildsReleased; + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" ? COMMIT_SHA : "", + }; + }; + + const [left, right] = await Promise.all([ + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }), + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }), + ]); + + expect(left.commitSha).toBe(COMMIT_SHA); + expect(right.commitSha).toBe(COMMIT_SHA); + expect(left.path).toBe(right.path); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + expect( + readdirSync(path.join(options.releasesRoot, "releases")).filter((entry) => + entry.startsWith(".staging-") + ) + ).toEqual([]); + }); + + it("removes the temporary worktree when commit verification fails", async () => { + const options = stagingOptions(); + const calls: string[] = []; + const runner: DashboardReleaseCommandRunner = async (command, arguments_) => { + calls.push(`${command} ${arguments_.slice(0, 2).join(" ")}`); + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + mkdirSync(String(arguments_[3])); + } else { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" + ? OTHER_COMMIT_SHA + : "", + }; + }; + + await expect( + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }) + ).rejects.toThrow("unexpected commit"); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + expect(calls.at(-1)).toBe("git worktree remove"); + expect(existsSync(managedReleasePath(options.releasesRoot, COMMIT_SHA))).toBe( + false + ); + }); + + it("cleans a partially-created worktree when git worktree add fails", async () => { + const options = stagingOptions(); + const calls: string[] = []; + const runner: DashboardReleaseCommandRunner = async (command, arguments_) => { + calls.push(`${command} ${arguments_.slice(0, 2).join(" ")}`); + if ( + command === "git" && + arguments_[0] === "worktree" && + arguments_[1] === "add" + ) { + mkdirSync(String(arguments_[3])); + throw new Error("worktree add failed"); + } + if ( + command === "git" && + arguments_[0] === "worktree" && + arguments_[1] === "remove" + ) { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + return { stderr: "", stdout: "" }; + }; + + await expect( + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }) + ).rejects.toThrow("worktree add failed"); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + expect(calls).toEqual(["git worktree add", "git worktree remove"]); + }); + + it("falls back to filesystem cleanup and prunes stale worktree metadata", async () => { + const options = stagingOptions(); + const calls: string[] = []; + const runner: DashboardReleaseCommandRunner = async (command, arguments_) => { + calls.push(`${command} ${arguments_.slice(0, 2).join(" ")}`); + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + const worktreePath = String(arguments_[3]); + mkdirSync(worktreePath); + await createBuiltRelease(worktreePath); + } else if (arguments_[1] === "remove") { + throw new Error("registered worktree removal failed"); + } + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" ? COMMIT_SHA : "", + }; + }; + + await expect( + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }) + ).resolves.toMatchObject({ commitSha: COMMIT_SHA }); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + expect(calls.at(-2)).toBe("git worktree remove"); + expect(calls.at(-1)).toBe("git worktree prune"); + }); + + it("rejects mismatched build identity and cleans the worktree", async () => { + const options = stagingOptions(); + const runner: DashboardReleaseCommandRunner = async (command, arguments_) => { + if (command === "git" && arguments_[0] === "worktree") { + if (arguments_[1] === "add") { + const worktreePath = String(arguments_[3]); + mkdirSync(worktreePath); + await createBuiltRelease(worktreePath, OTHER_COMMIT_SHA); + } else { + rmSync(String(arguments_[3]), { force: true, recursive: true }); + } + } + return { + stderr: "", + stdout: + command === "git" && arguments_[0] === "rev-parse" ? COMMIT_SHA : "", + }; + }; + + await expect( + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: runner, + }) + ).rejects.toThrow(`does not match ${COMMIT_SHA}`); + expect(readdirSync(options.worktreeRoot)).toEqual([]); + }); + + it("validates paths, commits, and CLI commands", async () => { + const options = stagingOptions(); + expect(() => + managedDashboardUnitContract( + options.releasesRoot, + options.databasePath, + options.openClawHome + ) + ).not.toThrow(); + expect(() => + managedDashboardUnitContract( + options.releasesRoot, + "relative.db", + options.openClawHome + ) + ).toThrow("database path must be an absolute non-root path"); + const contract = managedDashboardUnitContract( + options.releasesRoot, + options.databasePath, + options.openClawHome + ); + const properties = [ + `WorkingDirectory=${contract.releaseRoot}/backend`, + `Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${contract.databasePath} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${contract.releaseRoot} MIRA_DASHBOARD_RELEASES_ROOT=${contract.releasesRoot}`, + "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run -- bun dist/serverStart.js ; }", + ].join("\n"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties, + contract + ) + ).not.toThrow(); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard-worker.service", + properties, + contract + ) + ).toThrow("unexpected managed release entrypoint"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + "bun dist/serverStart.js", + "bun not-dist/serverStart.js" + ), + contract + ) + ).toThrow("unexpected managed release entrypoint"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + `WorkingDirectory=${contract.releaseRoot}/backend`, + () => `WorkingDirectory=${options.sourceRoot}/backend` + ), + contract + ) + ).toThrow("must run from managed current/backend"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + ` MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome}`, + "" + ), + contract + ) + ).toThrow("MIRA_DASHBOARD_OPENCLAW_HOME"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + ` MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome}`, + () => ` NOT_MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome}` + ), + contract + ) + ).toThrow("MIRA_DASHBOARD_OPENCLAW_HOME"); + await expect( + stageDashboardRelease("short", { + ...options, + commandRunner: async () => ({ stderr: "", stdout: "" }), + }) + ).rejects.toThrow("full lowercase Git SHA"); + await expect( + runReleaseDeploymentCommand(["unknown"], options.releasesRoot) + ).rejects.toThrow("Usage"); + await expect( + runReleaseDeploymentCommand(["prune", "1"], options.releasesRoot) + ).rejects.toThrow("retention must be between 2 and 20"); + await expect( + runReleaseDeploymentCommand(["prune", "3", "extra"], options.releasesRoot) + ).rejects.toThrow("unexpected arguments"); + expect( + await runReleaseDeploymentCommand(["prune"], options.releasesRoot) + ).toEqual({ removed: [], retained: [] }); + }); +}); diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index acbc20fb1..49f658f1d 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -30,6 +30,7 @@ import { isReleaseTransitionLockAvailable, loadManagedRelease, managedReleasePath, + pruneDashboardReleases, readDashboardReleaseState, RELEASE_TRANSITION_LOCK_FILE_NAME, RELEASE_TRANSITION_LOCK_PROGRAM, @@ -48,6 +49,7 @@ const temporaryRoots: string[] = []; const FIRST_COMMIT = "a".repeat(40); const SECOND_COMMIT = "b".repeat(40); const THIRD_COMMIT = "c".repeat(40); +const FOURTH_COMMIT = "d".repeat(40); const TEST_FUTURE_MIGRATIONS: DatabaseMigrationIdentity[] = [ { checksum: "7".repeat(64), @@ -382,7 +384,17 @@ describe("Dashboard immutable release manager", () => { }, root, }); - expect(status.current).not.toHaveProperty("manifest"); + expect(status).not.toHaveProperty("current.manifest"); + await expect(runReleaseLifecycleCommand(["prune"], root)).resolves.toEqual({ + removed: [], + retained: [SECOND_COMMIT, FIRST_COMMIT], + }); + await expect(runReleaseLifecycleCommand(["prune", "1"], root)).rejects.toThrow( + "retention must be between 2 and 20" + ); + await expect( + runReleaseLifecycleCommand(["prune", "3", "extra"], root) + ).rejects.toThrow("unexpected arguments"); await expect( runReleaseLifecycleCommand(["rollback", FIRST_COMMIT], root) ).rejects.toThrow("takes no commit SHA"); @@ -814,4 +826,49 @@ describe("Dashboard immutable release manager", () => { "requires two distinct releases" ); }); + + it("prunes old releases while preserving current and previous", async () => { + const root = temporaryReleasesRoot(); + await createManagedRelease(root, FIRST_COMMIT); + await createManagedRelease(root, SECOND_COMMIT); + await createManagedRelease(root, THIRD_COMMIT); + await createManagedRelease(root, FOURTH_COMMIT); + await activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS); + await activateDashboardRelease(THIRD_COMMIT, root, SCHEMA_6_OPTIONS); + const interruptedRetirementPath = path.join( + root, + "releases", + `.retired-${"e".repeat(40)}-00000000-0000-4000-8000-000000000000` + ); + mkdirSync(interruptedRetirementPath); + writeFileSync(path.join(interruptedRetirementPath, "stale"), "stale\n"); + + const result = await pruneDashboardReleases(3, root); + + expect(result).toEqual({ + removed: [FIRST_COMMIT], + retained: [FOURTH_COMMIT, THIRD_COMMIT, SECOND_COMMIT], + }); + expect(existsSync(managedReleasePath(root, FIRST_COMMIT))).toBe(false); + expect(existsSync(managedReleasePath(root, SECOND_COMMIT))).toBe(true); + expect(existsSync(managedReleasePath(root, THIRD_COMMIT))).toBe(true); + expect(existsSync(managedReleasePath(root, FOURTH_COMMIT))).toBe(true); + expect(existsSync(interruptedRetirementPath)).toBe(false); + const state = await readDashboardReleaseState(root); + expect(state.current?.commitSha).toBe(THIRD_COMMIT); + expect(state.previous?.commitSha).toBe(SECOND_COMMIT); + }); + + it("validates release retention bounds", async () => { + const root = temporaryReleasesRoot(); + await expect(pruneDashboardReleases(1, root)).rejects.toThrow( + "retention must be between 2 and 20" + ); + await expect(pruneDashboardReleases(21, root)).rejects.toThrow( + "retention must be between 2 and 20" + ); + await expect(pruneDashboardReleases(NaN, root)).rejects.toThrow( + "retention must be between 2 and 20" + ); + }); }); diff --git a/backend/test/releaseManifest.test.ts b/backend/test/releaseManifest.test.ts index 2a16f70b9..4181034d6 100644 --- a/backend/test/releaseManifest.test.ts +++ b/backend/test/releaseManifest.test.ts @@ -231,46 +231,6 @@ describe("Dashboard release manifest", () => { ).toEndWith("\n"); }); - it("keeps deployed version 1 manifests readable during the format transition", async () => { - const root = temporaryReleaseRoot(); - const manifest = await createReleaseManifest(manifestOptions(root)); - rmSync(path.join(root, "backend", "dist", "releaseLifecycle.js")); - const legacySchema = { - maximumCompatible: manifest.schema.maximumCompatible, - migrationRegistrySha256: manifest.schema.migrationRegistrySha256, - minimumCompatible: manifest.schema.minimumCompatible, - target: manifest.schema.target, - }; - - const legacyManifest = parseReleaseManifest({ - ...manifest, - artifacts: manifest.artifacts.filter( - (artifact) => artifact.path !== "backend/dist/releaseLifecycle.js" - ), - formatVersion: 1, - schema: legacySchema, - }); - writeFileSync( - path.join(root, RELEASE_MANIFEST_FILE_NAME), - `${JSON.stringify(legacyManifest, undefined, 2)}\n` - ); - - expect(await loadReleaseManifest(root)).toMatchObject({ - formatVersion: 1, - schema: legacySchema, - }); - await expect( - verifyReleaseArtifacts(root, legacyManifest) - ).resolves.toBeUndefined(); - await expect( - loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) - ).resolves.toMatchObject({ - manifestFormatVersion: 1, - ready: true, - source: "manifest", - }); - }); - it("refuses to write a manifest larger than the loader accepts", async () => { const root = temporaryReleaseRoot(); const longName = "x".repeat(120); diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 07fd09a32..e1dc92aba 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -6,6 +6,7 @@ import { mkdtempSync, readdirSync, readFileSync, + readlinkSync, rmSync, symlinkSync, utimesSync, @@ -17,8 +18,13 @@ import path from "node:path"; import { afterEach, describe, expect, it, jest } from "bun:test"; import type { DashboardSocket } from "../src/dashboardSocket.ts"; -import { database, sqlNullable } from "../src/database.ts"; +import { database, getMiraDatabasePath, sqlNullable } from "../src/database.ts"; import * as processModule from "../src/lib/processes.ts"; +import { + ensureDashboardReleaseLayout, + managedReleasePath, +} from "../src/releaseManager.ts"; +import { writeReleaseManifest } from "../src/releaseManifest.ts"; const cleanupCallbacks: Array<() => Promise | void> = []; @@ -41,6 +47,66 @@ function createTemporaryRoot(prefix: string): string { return root; } +async function createDeploymentReleaseFixture( + releaseRoot: string, + commitSha: string, + commitTitle: string +): Promise { + mkdirSync(path.join(releaseRoot, "backend", "config"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "backend", "dist"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "dist", "assets"), { recursive: true }); + writeFileSync(path.join(releaseRoot, "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "bun.lock"), "root-lock\n"); + writeFileSync(path.join(releaseRoot, "backend", "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "backend", "bun.lock"), "backend-lock\n"); + writeFileSync( + path.join(releaseRoot, "backend", "config", "log-rotation.json"), + '{"jobs":[]}\n' + ); + writeFileSync(path.join(releaseRoot, "dist", "index.html"), "
ready
\n"); + writeFileSync( + path.join(releaseRoot, "dist", "assets", "app.js"), + `export const commit = "${commitSha}";\n` + ); + writeFileSync( + path.join(releaseRoot, "not-a-release-artifact.txt"), + "must not publish\n" + ); + for (const component of ["frontend", "backend"] as const) { + const componentRoot = + component === "frontend" + ? path.join(releaseRoot, "dist") + : path.join(releaseRoot, "backend", "dist"); + writeFileSync( + path.join(componentRoot, "build-identity.json"), + `${JSON.stringify({ + bunVersion: Bun.version, + commitSha, + component, + formatVersion: 1, + })}\n` + ); + } + for (const entrypoint of [ + "databasePreflight", + "releaseLifecycle", + "resetDashboardPassword", + "serverStart", + "workerStart", + ]) { + writeFileSync( + path.join(releaseRoot, "backend", "dist", `${entrypoint}.js`), + `export const commit = "${commitSha}";\n` + ); + } + await writeReleaseManifest({ + builtAt: new Date("2026-07-26T02:00:00.000Z"), + commitSha, + commitTitle, + releaseRoot, + }); +} + function readableUtf8Stream(value: string): ReadableStream { return new ReadableStream({ start(controller) { @@ -1810,17 +1876,43 @@ describe("backend service behavior", () => { } }); - it("runs deploy latest build flow against an isolated checkout", async () => { + it("publishes and activates an immutable release before detached cutover", async () => { rememberEnvironment("PATH"); rememberEnvironment("MIRA_DASHBOARD_ROOT"); rememberEnvironment("MIRA_DASHBOARD_WORKTREE_ROOT"); + rememberEnvironment("MIRA_DASHBOARD_RELEASES_ROOT"); + rememberEnvironment("MIRA_DASHBOARD_OPENCLAW_HOME"); + rememberEnvironment("MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE"); const fakeRoot = createTemporaryRoot("mira-pr-deploy-root-"); const fakeBin = createTemporaryRoot("mira-pr-deploy-bin-"); + const worktreeRoot = path.join(fakeRoot, "worktrees"); + const releasesRoot = path.join(fakeRoot, "managed-releases"); + const candidateTemplate = path.join(fakeRoot, "candidate-template"); + const openClawHome = path.join(fakeRoot, "state", "openclaw-client"); + const logRotationLockFile = path.join(fakeRoot, "state", "log-rotation.lock"); + const oldCommit = "c".repeat(40); + const candidateCommit = "d".repeat(40); const gitLog = path.join(fakeRoot, "git.log"); const bunLog = path.join(fakeRoot, "bun.log"); + const systemctlLog = path.join(fakeRoot, "systemctl.log"); const systemdLog = path.join(fakeRoot, "systemd.log"); - mkdirSync(path.join(fakeRoot, "backend", "node_modules"), { recursive: true }); - mkdirSync(path.join(fakeRoot, "node_modules"), { recursive: true }); + mkdirSync(path.join(fakeRoot, "backend"), { recursive: true }); + mkdirSync(worktreeRoot); + mkdirSync(path.dirname(openClawHome), { recursive: true }); + mkdirSync(candidateTemplate); + await createDeploymentReleaseFixture( + candidateTemplate, + candidateCommit, + "Deployable dashboard commit" + ); + await ensureDashboardReleaseLayout(releasesRoot); + const oldReleasePath = managedReleasePath(releasesRoot, oldCommit); + await createDeploymentReleaseFixture( + oldReleasePath, + oldCommit, + "Previous dashboard commit" + ); + symlinkSync(`releases/${oldCommit}`, path.join(releasesRoot, "current"), "dir"); writeFileSync( path.join(fakeBin, "git"), String.raw`#!/usr/bin/env bash @@ -1831,15 +1923,20 @@ if [[ "$*" == "rev-parse --show-toplevel" ]]; then elif [[ "$*" == "rev-parse --abbrev-ref HEAD" ]]; then printf 'main\n' elif [[ "$*" == "rev-parse --short HEAD" ]]; then - printf 'def5678\n' + printf '%s\n' ${JSON.stringify(candidateCommit.slice(0, 8))} +elif [[ "$*" == "rev-parse HEAD" ]]; then + printf '%s\n' ${JSON.stringify(candidateCommit)} elif [[ "$*" == "rev-parse --abbrev-ref --symbolic-full-name ${"@{u}"}" ]]; then printf 'origin/main\n' elif [[ "$*" == "status --short" ]]; then printf '' elif [[ "$*" == "fetch --prune origin" || "$*" == "checkout main" || "$*" == "pull --ff-only origin main" ]]; then printf '' -elif [[ "$*" == "log -1 --pretty=%s" ]]; then - printf 'Deployable dashboard commit\n' +elif [[ "$1 $2 $3" == "worktree add --detach" ]]; then + mkdir -p "$4" + cp -a ${JSON.stringify(`${candidateTemplate}/.`)} "$4/" +elif [[ "$1 $2 $3" == "worktree remove --force" ]]; then + rm -rf "$4" else echo "unexpected git args: $*" >&2 exit 2 @@ -1857,6 +1954,26 @@ else echo "unexpected bun args: $*" >&2 exit 2 fi +` + ); + writeFileSync( + path.join(fakeBin, "systemctl"), + String.raw`#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> ${JSON.stringify(systemctlLog)} +if [[ "$*" != *"--user show"* ]]; then + echo "unexpected systemctl args: $*" >&2 + exit 2 +fi +if [[ "$*" == *"mira-dashboard-worker.service"* ]]; then + entrypoint="dist/workerStart.js" +else + entrypoint="dist/serverStart.js" +fi +printf '%s\n' \ + 'Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${getMiraDatabasePath()} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${releasesRoot}/current MIRA_DASHBOARD_RELEASES_ROOT=${releasesRoot}' \ + "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run -- bun $entrypoint ; }" \ + 'WorkingDirectory=${releasesRoot}/current/backend' ` ); writeFileSync( @@ -1869,10 +1986,14 @@ printf 'scheduled\n' ); chmodSync(path.join(fakeBin, "git"), 0o755); chmodSync(path.join(fakeBin, "bun"), 0o755); + chmodSync(path.join(fakeBin, "systemctl"), 0o755); chmodSync(path.join(fakeBin, "systemd-run"), 0o755); process.env.PATH = `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`; process.env.MIRA_DASHBOARD_ROOT = fakeRoot; - process.env.MIRA_DASHBOARD_WORKTREE_ROOT = path.join(fakeRoot, "worktrees"); + process.env.MIRA_DASHBOARD_WORKTREE_ROOT = worktreeRoot; + process.env.MIRA_DASHBOARD_RELEASES_ROOT = releasesRoot; + process.env.MIRA_DASHBOARD_OPENCLAW_HOME = openClawHome; + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE = logRotationLockFile; const { registerPullRequestExecutionActions, startDeployLatest } = await import("../src/services/pullRequests.ts"); @@ -1933,19 +2054,22 @@ printf 'scheduled\n' status: string; }; expect(row).toEqual({ - commit_sha: "def5678", + commit_sha: candidateCommit.slice(0, 8), commit_title: "Deployable dashboard commit", - note: "Build passed. Restart + health check scheduled", + note: "Immutable release published. Atomic restart and rollback check scheduled", status: "restart-scheduled", }); await expect(Bun.file(gitLog).text()).resolves.toContain( "pull --ff-only origin main" ); await expect(Bun.file(bunLog).text()).resolves.toContain( - `${fakeRoot}|install --frozen-lockfile` + `${worktreeRoot}/release-${candidateCommit.slice(0, 12)}-` ); await expect(Bun.file(bunLog).text()).resolves.toContain( - `${fakeRoot}|run deploy:prepare` + "|run deploy:prepare" + ); + await expect(Bun.file(systemctlLog).text()).resolves.toContain( + "show mira-dashboard.service" ); await expect(Bun.file(systemdLog).text()).resolves.toContain( `mira-dashboard-deploy-${job.id}` @@ -1953,13 +2077,31 @@ printf 'scheduled\n' const restartCommand = await Bun.file(systemdLog).text(); expect(restartCommand).toContain("/api/health/ready"); expect(restartCommand).toContain("--connect-timeout 2 --max-time 5"); - expect(restartCommand).toContain("for attempt in {1..20}"); - expect(restartCommand).not.toContain('"workerOnline":true'); + expect(restartCommand).toContain("for attempt in {1..30}"); + expect(restartCommand).toContain(".checks.release.backendCommit"); + expect(restartCommand).toContain("releaseLifecycle.js"); + expect(restartCommand).toContain("rollback"); + expect(restartCommand).toContain("prune 3"); expect(restartCommand).not.toContain("/api/job-executions"); - expect(existsSync(path.join(fakeRoot, "node_modules"))).toBe(false); - expect(existsSync(path.join(fakeRoot, "backend", "node_modules"))).toBe( - false + expect(readlinkSync(path.join(releasesRoot, "current"))).toBe( + `releases/${candidateCommit}` + ); + expect(readlinkSync(path.join(releasesRoot, "previous"))).toBe( + `releases/${oldCommit}` ); + const publishedReleasePath = managedReleasePath( + releasesRoot, + candidateCommit + ); + expect( + readFileSync( + path.join(publishedReleasePath, "release-manifest.json"), + "utf8" + ) + ).toContain(candidateCommit); + expect( + existsSync(path.join(publishedReleasePath, "not-a-release-artifact.txt")) + ).toBe(false); expect(getJobExecution(deploymentExecution.id)).toMatchObject({ cancellable: false, status: "success", diff --git a/backend/test/testDatabaseGuard.test.ts b/backend/test/testDatabaseGuard.test.ts index d251ab842..4a0379cca 100644 --- a/backend/test/testDatabaseGuard.test.ts +++ b/backend/test/testDatabaseGuard.test.ts @@ -144,6 +144,17 @@ describe("database test safety guard", () => { } ); + it("refuses the production state database path while running tests", async () => { + const { exitCode, stderr } = await importDatabaseInChild( + "/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db" + ); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain( + "Refusing to open non-temporary Dashboard test database" + ); + }); + nonTemporaryTest( "refuses preflight access to a non-temporary database while running tests", async () => { diff --git a/docs/api/overview.md b/docs/api/overview.md index 08a22c4c9..aebe63a84 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -62,9 +62,6 @@ Public routes: - `POST /api/auth/login/webauthn/verify` - `POST /api/auth/logout` -The retired top-level `GET|HEAD /health` path returns `410 Gone` instead of -falling through to the SPA shell. This makes stale monitors fail visibly. - The WebSocket endpoint `/ws` is also authenticated and origin-checked. Account-security endpoints under `/api/account/security/*` are protected diff --git a/docs/architecture/database.md b/docs/architecture/database.md index f6289b259..c8adde2eb 100644 --- a/docs/architecture/database.md +++ b/docs/architecture/database.md @@ -8,6 +8,15 @@ Default path: backend/data/mira-dashboard.db ``` +That default is for development. Production units set: + +```text +/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +``` + +The `-wal`/`-shm` sidecars and `backups/` directory stay below the same +persistent state root, outside the control checkout and immutable releases. + Override: ```bash @@ -62,40 +71,40 @@ edit a released migration. Add the next numbered file instead. ## Tables -| Table | Purpose | Lifecycle | -| ---------------------------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| `schema_migrations` | Applied migration versions and immutable checksums. | Immutable audit history; never age-pruned. | -| `audit_events` | Redacted request and privileged job lifecycle audit trail. | Triggers reject update, delete, and replacement. No automatic pruning. | -| `users` | Dashboard auth users and explicit MFA-enabled timestamp. | Authoritative records; removed only by explicit auth flows. | -| `auth_sessions` | Hashed-validator sessions with idle, MFA, elevation, and device state. | Removed after idle/absolute expiry or explicit revocation. | -| `auth_pending_logins` | Hashed-validator password-first MFA handoffs. | Five-minute expiry; consumed on success or bounded failures. | -| `auth_webauthn_challenges` | Session/pending-bound registration and assertion challenges. | Five-minute expiry; atomically consumed by verification. | -| `user_totp_factors` | Named TOTP factors with encrypted seeds and replay state. | Unconfirmed setup expires; confirmed factors require explicit removal. | -| `user_webauthn_credentials` | Named WebAuthn public keys, counters, transports, and device state. | Retained until explicit removal; multiple backup keys are supported. | -| `user_recovery_codes` | One-time recovery selectors and password-hashed validators. | Consumed once; replaced as one set on rotation. | -| `auth_rate_limit_buckets` | Hashed account/failure buckets and progressive cooldown state. | Cleared on success; stale state removed after 24 hours. | -| `app_config` | Small persistent config, currently including an encrypted `gateway_token` envelope. | Keyed upsert or explicit removal; naturally bounded. | -| `tasks` | Local task records. | Done tasks are removed after 365 idle days. | -| `task_events` | Audit/event records for task changes. | Follows old done tasks; otherwise at most 5,000 rows per task. | -| `task_updates` | Markdown progress updates on tasks. | Follows old done tasks; otherwise at most 5,000 rows per task. | -| `notifications` | Notification bell items, including report links and ops alerts. | Read: 14 days/300 rows; unread retained; report links follow reports. | -| `reports` | Daily briefs, daily summaries, heartbeats, and custom reports. | 365 days and at most 5,000 rows. | -| `cache_entries` | Cache refresh state and cached provider data. | Fixed producer keys updated in place. | -| `quota_alert_state` | Notification arming state for quota thresholds. | Finite provider/bucket keys updated in place. | -| `openclaw_alert_state` | Notification arming state for OpenClaw update alerts. | Singleton row. | -| `agent_task_history` | Agent current/completed task history. | Completed: 90 days/10,000 rows; active rows retained. | -| `deployment_jobs` | Dashboard deploy job state/output. | Non-active: 90 days/500 rows; active rows retained. | -| `deployment_lock` | Single active deployment lock. | Singleton removed when the owning deploy releases it. | -| `scheduled_jobs` | Dashboard-local scheduled job definitions. | Reconciled against registered actions; explicit operator state retained. | -| `scheduled_job_runs` | Scheduled job run history. | Completed: 90 days/20,000 rows; active rows retained. | -| `scheduled_job_execution_policies` | Resource class and timeout for each Dashboard job. | One row per job; cascades when the job is removed. | -| `openclaw_cron_job_metadata` | Disable intent and Dashboard metadata for OpenClaw cron jobs. | Keyed operator intent; removed explicitly when intent is cleared. | -| `job_executions` | Persistent execution queue with leases, heartbeats, and cancellation. | Terminal: 90 days/20,000 rows; queued/running rows retained. | -| `job_workers` | Worker capacity and liveness heartbeats. | Stale heartbeats removed after 24 hours. | -| `chat_runtime_snapshots` | Durable OpenClaw chat replay/session snapshots. | Per-scope live cap plus global 30-day/200-row maintenance safety net. | -| `chat_runtime_snapshot_events` | Ordered durable replay events for those snapshots. | Follows retained snapshots; orphan rows are removed. | -| `docker_managed_services` | Docker updater managed service inventory. | Reconciled with current Compose inventory. | -| `docker_update_events` | Docker updater event history. | 180 days and at most 5,000 rows. | +| Table | Purpose | Lifecycle | +| ---------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| `schema_migrations` | Applied migration versions and immutable checksums. | Immutable audit history; never age-pruned. | +| `audit_events` | Redacted request and privileged job lifecycle audit trail. | Triggers reject update, delete, and replacement. No automatic pruning. | +| `users` | Dashboard auth users and explicit MFA-enabled timestamp. | Authoritative records; removed only by explicit auth flows. | +| `auth_sessions` | Hashed-validator sessions with idle, MFA, elevation, and device state. | Removed after idle/absolute expiry or explicit revocation. | +| `auth_pending_logins` | Hashed-validator password-first MFA handoffs. | Five-minute expiry; consumed on success or bounded failures. | +| `auth_webauthn_challenges` | Session/pending-bound registration and assertion challenges. | Five-minute expiry; atomically consumed by verification. | +| `user_totp_factors` | Named TOTP factors with encrypted seeds and replay state. | Unconfirmed setup expires; confirmed factors require explicit removal. | +| `user_webauthn_credentials` | Named WebAuthn public keys, counters, transports, and device state. | Retained until explicit removal; multiple backup keys are supported. | +| `user_recovery_codes` | One-time recovery selectors and password-hashed validators. | Consumed once; replaced as one set on rotation. | +| `auth_rate_limit_buckets` | Hashed account/failure buckets and progressive cooldown state. | Cleared on success; stale state removed after 24 hours. | +| `app_config` | Small persistent config, currently including an encrypted `gateway_token` envelope. | Keyed upsert or explicit removal; naturally bounded. | +| `tasks` | Local task records. | Done tasks are removed after 365 idle days. | +| `task_events` | Audit/event records for task changes. | Follows old done tasks; otherwise at most 5,000 rows per task. | +| `task_updates` | Markdown progress updates on tasks. | Follows old done tasks; otherwise at most 5,000 rows per task. | +| `notifications` | Notification bell items, including report links and ops alerts. | Read: 14 days/300 rows; unread retained; report links follow reports. | +| `reports` | Daily briefs, daily summaries, heartbeats, and custom reports. | 365 days and at most 5,000 rows. | +| `cache_entries` | Cache refresh state and cached provider data. | Fixed producer keys updated in place. | +| `quota_alert_state` | Notification arming state for quota thresholds. | Finite provider/bucket keys updated in place. | +| `openclaw_alert_state` | Notification arming state for OpenClaw update alerts. | Singleton row. | +| `agent_task_history` | Agent current/completed task history. | Completed: 90 days/10,000 rows; active rows retained. | +| `deployment_jobs` | Dashboard deploy job state/output. | Non-active: 90 days/500 rows; active rows retained. | +| `deployment_lock` | Single active deployment lock. | Singleton removed when the owning deploy releases it. | +| `scheduled_jobs` | Dashboard-local scheduled job definitions. | Reconciled against registered actions; explicit operator state retained. | +| `scheduled_job_runs` | Scheduled job run history. | Completed: 90 days/20,000 rows; active rows retained. | +| `scheduled_job_execution_policies` | Resource class and timeout for each Dashboard job. | One row per job; cascades when the job is removed. | +| `openclaw_cron_job_metadata` | Disable intent and Dashboard metadata for OpenClaw cron jobs. | Keyed operator intent; removed explicitly when intent is cleared. | +| `job_executions` | Persistent execution queue with leases, heartbeats, and cancellation. | Terminal: 90 days/20,000 rows; queued/running rows retained. | +| `job_workers` | Worker capacity and liveness heartbeats. | Stale heartbeats removed after 24 hours. | +| `chat_runtime_snapshots` | Durable OpenClaw chat replay/session snapshots. | Per-scope live cap plus global 30-day/200-row maintenance safety net. | +| `chat_runtime_snapshot_events` | Ordered durable replay events for those snapshots. | Follows retained snapshots; orphan rows are removed. | +| `docker_managed_services` | Docker updater managed service inventory. | Reconciled with current Compose inventory. | +| `docker_update_events` | Docker updater event history. | 180 days and at most 5,000 rows. | TOTP seeds and the persisted OpenClaw Gateway token are verifiable secrets and therefore cannot be one-way hashed. Dashboard encrypts them with versioned @@ -142,7 +151,9 @@ schema enforces append-only history. Any future archive/retention design must arrive through a reviewed forward migration that preserves required audit history. -Snapshots live beside the database under `data/backups/` by default: +Snapshots live below `dirname(MIRA_DASHBOARD_DB_PATH)/backups/`. This is +`backend/data/backups/` in development and +`/home/ubuntu/projects/mira-dashboard-state/backups/` in production: | Kind | Maximum age | Maximum count | | --------------- | ----------- | ------------- | @@ -156,16 +167,11 @@ committed data may still be in `-wal`. ## Useful Inspection Commands -Run production inspection through Doppler so the command resolves the same -`MIRA_DASHBOARD_DB_PATH` value as the services: +Use the same explicit path as the managed production units: ```bash set -euo pipefail -cd /home/ubuntu/projects/mira-dashboard/backend -db_path="$( - /usr/local/bin/doppler run --config prd --project rajohan -- \ - sh -c 'realpath -m -- "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}"' -)" +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db sqlite3 -readonly "$db_path" ".tables" sqlite3 -readonly "$db_path" "PRAGMA integrity_check;" sqlite3 -readonly "$db_path" \ @@ -191,22 +197,11 @@ Use this only when Raymond explicitly wants to re-run setup. ```bash set -euo pipefail -backend_dir=/home/ubuntu/projects/mira-dashboard/backend -configured_db_path="$( - cd "$backend_dir" - /usr/local/bin/doppler run --config prd --project rajohan -- \ - sh -c 'printf "%s" "${MIRA_DASHBOARD_DB_PATH-}"' -)" -if [[ -z "$configured_db_path" ]]; then - db_path="$backend_dir/data/mira-dashboard.db" -elif [[ "$configured_db_path" = /* ]]; then - db_path="$configured_db_path" -else - db_path="$backend_dir/$configured_db_path" -fi +backend_dir=/home/ubuntu/projects/mira-dashboard-releases/current/backend +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db cd "$backend_dir" /usr/local/bin/doppler run --config prd --project rajohan -- \ - bun run db:preflight + env MIRA_DASHBOARD_DB_PATH="$db_path" bun run db:preflight sqlite3 -cmd ".timeout 5000" "$db_path" "DELETE FROM auth_sessions; DELETE FROM users; DELETE FROM app_config WHERE key='gateway_token';" sqlite3 -cmd ".timeout 5000" "$db_path" "PRAGMA integrity_check;" curl http://127.0.0.1:3100/api/auth/bootstrap diff --git a/docs/architecture/gateway-and-chat.md b/docs/architecture/gateway-and-chat.md index 6c42f48e1..4c9f09167 100644 --- a/docs/architecture/gateway-and-chat.md +++ b/docs/architecture/gateway-and-chat.md @@ -617,7 +617,6 @@ openclaw status Do not print Gateway token values while debugging. Inspect length/metadata only: ```bash -cd backend -sqlite3 "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}" \ +sqlite3 /home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ "SELECT key, length(value), updated_at FROM app_config WHERE key='gateway_token';" ``` diff --git a/docs/index.md b/docs/index.md index c05fbbec0..473cb28bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,9 +48,11 @@ It is a Bun-native application with: - background schedulers for cache refresh, backups, Docker update checks, log rotation, quota notifications, and OpenClaw update notifications. -The production service is `mira-dashboard.service`, running from -`/home/ubuntu/projects/mira-dashboard/backend` through Doppler -`rajohan/prd`. +Production uses `mira-dashboard.service` plus +`mira-dashboard-worker.service`. Both run through Doppler `rajohan/prd` from +the immutable release selected by +`/home/ubuntu/projects/mira-dashboard-releases/current`; the Git checkout is +only the deployment control plane. ## Documentation Rules diff --git a/docs/operations/runbooks.md b/docs/operations/runbooks.md index 07a0af9a5..71aefc0c9 100644 --- a/docs/operations/runbooks.md +++ b/docs/operations/runbooks.md @@ -60,22 +60,11 @@ Use only when Raymond wants to re-run bootstrap. ```bash set -euo pipefail -backend_dir=/home/ubuntu/projects/mira-dashboard/backend -configured_db_path="$( - cd "$backend_dir" - /usr/local/bin/doppler run --config prd --project rajohan -- \ - sh -c 'printf "%s" "${MIRA_DASHBOARD_DB_PATH-}"' -)" -if [[ -z "$configured_db_path" ]]; then - db_path="$backend_dir/data/mira-dashboard.db" -elif [[ "$configured_db_path" = /* ]]; then - db_path="$configured_db_path" -else - db_path="$backend_dir/$configured_db_path" -fi +backend_dir=/home/ubuntu/projects/mira-dashboard-releases/current/backend +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db cd "$backend_dir" /usr/local/bin/doppler run --config prd --project rajohan -- \ - bun run db:preflight + env MIRA_DASHBOARD_DB_PATH="$db_path" bun run db:preflight sqlite3 -cmd ".timeout 5000" "$db_path" "DELETE FROM auth_sessions; DELETE FROM users;" sqlite3 -cmd ".timeout 5000" "$db_path" "PRAGMA integrity_check;" curl http://127.0.0.1:3100/api/auth/bootstrap @@ -84,7 +73,8 @@ curl http://127.0.0.1:3100/api/auth/bootstrap To force Gateway token entry during bootstrap too: ```bash -sqlite3 data/mira-dashboard.db "DELETE FROM app_config WHERE key='gateway_token';" +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +sqlite3 "$db_path" "DELETE FROM app_config WHERE key='gateway_token';" ``` ## Reset A Forgotten Dashboard Password @@ -94,7 +84,7 @@ watched by the web service, or unauthenticated reset endpoint. Use the host-local interactive command from an SSH/console TTY: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend +cd /home/ubuntu/projects/mira-dashboard-releases/current/backend bun run auth:reset-password -- --username ``` @@ -119,8 +109,9 @@ recovery codes offline. ## Inspect Gateway Token Metadata Without Printing It ```bash -cd /home/ubuntu/projects/mira-dashboard/backend -sqlite3 data/mira-dashboard.db "SELECT key, length(value), updated_at FROM app_config WHERE key='gateway_token';" +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +sqlite3 "$db_path" \ + "SELECT key, length(value), updated_at FROM app_config WHERE key='gateway_token';" ``` Do not print token values. A current row must contain a versioned encrypted @@ -166,11 +157,14 @@ and [New VPS setup](../setup/new-vps.md#provision-local-openclaw-api-callers). Symptom: browser shows `Frontend Not Built` or `/` returns 503. -```bash -cd /home/ubuntu/projects/mira-dashboard -bun run build -systemctl --user restart mira-dashboard.service -``` +The managed deploy executor verifies `dist/index.html`, every declared frontend +artifact, and both component identities before activation. Do not build inside +the control checkout or active release. Inspect `/api/health/ready`, both unit +logs, and the managed `current`/`previous` state, then use the automatic/manual +release rollback procedure in +[Production deploy](../setup/production-deploy.md#rollback). An incomplete +release must be restaged from its exact Git commit rather than repaired in +place. ## SQLite Locked @@ -194,38 +188,24 @@ is compatible with its schema. Never overwrite the live database or remove `-wal`/`-shm` while either Dashboard process is running. 1. Confirm the execution queue is idle and record the absolute snapshot path. -2. In one shell invocation, resolve the configured database path, verify the - snapshot, stop worker then web, preserve the current SQLite files, install - and validate the standalone snapshot, and only then start web and worker: +2. In one shell invocation, use the managed production database path, verify + the snapshot, stop worker then web, preserve the current SQLite files, + install and validate the standalone snapshot, and only then start web and + worker: ```bash set -euo pipefail -backend_dir=/home/ubuntu/projects/mira-dashboard/backend backup_path=/absolute/path/to/selected/mira-dashboard-....db -configured_db_path="$( - cd "$backend_dir" - /usr/local/bin/doppler run --config prd --project rajohan -- \ - sh -c 'printf "%s" "${MIRA_DASHBOARD_DB_PATH-}"' -)" -if [[ -z "$configured_db_path" ]]; then - db_path="$backend_dir/data/mira-dashboard.db" -elif [[ "$configured_db_path" = /* ]]; then - db_path="$configured_db_path" -else - db_path="$backend_dir/$configured_db_path" -fi +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db test -f "$backup_path" test "$(sqlite3 -readonly "$backup_path" "PRAGMA quick_check;")" = "ok" has_migration_history="$( sqlite3 -readonly "$backup_path" \ "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name = 'schema_migrations';" )" -if [[ "$has_migration_history" = "1" ]]; then - sqlite3 -readonly "$backup_path" \ - "SELECT version, name FROM schema_migrations ORDER BY version;" -else - printf '%s\n' "Legacy snapshot without schema_migrations; pair it with pre-lifecycle code." -fi +test "$has_migration_history" = "1" +sqlite3 -readonly "$backup_path" \ + "SELECT version, name FROM schema_migrations ORDER BY version;" systemctl --user stop mira-dashboard-worker.service systemctl --user stop mira-dashboard.service db_dir="$(dirname "$db_path")" @@ -295,9 +275,11 @@ Inspect the `database.maintenance` job on Jobs and the Database page's attention list. A manual deploy preflight can create and restore-verify a fresh snapshot: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend +cd /home/ubuntu/projects/mira-dashboard-releases/current/backend /usr/local/bin/doppler run --config prd --project rajohan -- \ - bun run db:preflight + env \ + MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ + bun run db:preflight ``` “Reusable space” is SQLite freelist capacity that can be reused by future diff --git a/docs/operations/scheduler-cache-backups.md b/docs/operations/scheduler-cache-backups.md index 76bb8897f..1fb397b80 100644 --- a/docs/operations/scheduler-cache-backups.md +++ b/docs/operations/scheduler-cache-backups.md @@ -232,16 +232,14 @@ run automatic `VACUUM`. List scheduled job tables: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend -sqlite3 "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}" \ +sqlite3 /home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ "SELECT id, name, enabled, schedule_type, next_run_at, updated_at FROM scheduled_jobs ORDER BY id;" ``` Inspect recent runs: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend -sqlite3 "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}" \ +sqlite3 /home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ "SELECT job_id, status, started_at, finished_at FROM scheduled_job_runs ORDER BY id DESC LIMIT 20;" ``` @@ -249,11 +247,7 @@ Inspect SQLite lifecycle state: ```bash set -euo pipefail -cd /home/ubuntu/projects/mira-dashboard/backend -db_path="$( - /usr/local/bin/doppler run --config prd --project rajohan -- \ - sh -c 'realpath -m -- "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}"' -)" +db_path=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db sqlite3 -readonly "$db_path" \ "SELECT version, name, applied_at FROM schema_migrations ORDER BY version;" sqlite3 -readonly "$db_path" \ @@ -263,7 +257,6 @@ sqlite3 -readonly "$db_path" \ Inspect cache freshness: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend -sqlite3 "${MIRA_DASHBOARD_DB_PATH:-data/mira-dashboard.db}" \ +sqlite3 /home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ "SELECT key, status, updated_at FROM cache_entries ORDER BY updated_at DESC LIMIT 30;" ``` diff --git a/docs/security/auth-and-trust-boundaries.md b/docs/security/auth-and-trust-boundaries.md index 04e2ecacf..7c1963198 100644 --- a/docs/security/auth-and-trust-boundaries.md +++ b/docs/security/auth-and-trust-boundaries.md @@ -110,7 +110,7 @@ when enabled, rotates the current session, and revokes every other session. Forgotten-password recovery is intentionally host-local: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend +cd /home/ubuntu/projects/mira-dashboard-releases/current/backend bun run auth:reset-password -- --username ``` diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index c8d49c155..33bc03b91 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -43,43 +43,83 @@ cd backend bun install --frozen-lockfile ``` -## Build Frontend And Backend - -Build the frontend from the repository root: +Create the managed runtime roots: ```bash -cd /home/ubuntu/projects/mira-dashboard -bun run build +install -d -m 0755 \ + /home/ubuntu/projects/mira-dashboard-worktrees \ + /home/ubuntu/projects/mira-dashboard-releases +install -d -m 0700 /home/ubuntu/projects/mira-dashboard-state ``` -Build the backend from its own package directory: +## Publish The Initial Managed Release + +Build, preflight, checksum, and publish the checked-out commit from an isolated +worktree: ```bash -cd /home/ubuntu/projects/mira-dashboard/backend -bun run build +cd /home/ubuntu/projects/mira-dashboard +RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +DATABASE_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +OPENCLAW_CLIENT_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client +LOG_ROTATION_LOCK=/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock +CANDIDATE_SHA="$(git rev-parse HEAD)" + +# A new host has no live database to preflight yet. Initialize and migrate the +# empty state database once from this exact checked-out candidate. +( + cd backend + env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + NODE_ENV=production \ + bun -e ' + const { database } = await import("./src/database.ts"); + try { + const result = database.query("PRAGMA quick_check").get(); + if (!result || Object.values(result)[0] !== "ok") { + throw new Error("Fresh Dashboard database failed quick_check"); + } + } finally { + database.close(); + } + ' +) + +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_OPENCLAW_HOME="$OPENCLAW_CLIENT_HOME" \ + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE="$LOG_ROTATION_LOCK" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun backend/src/releaseDeployment.ts stage "$CANDIDATE_SHA" ``` -Return to the repository root and create the checksummed runtime manifest: +Activate it before installing/starting the managed systemd units: ```bash -cd /home/ubuntu/projects/mira-dashboard -bun run release:manifest +RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +DATABASE_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +CANDIDATE_SHA="$(git rev-parse HEAD)" +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun backend/src/releaseLifecycle.ts activate "$CANDIDATE_SHA" ``` -The frontend build writes to `dist/`, the backend build writes to -`backend/dist/`, and `release:manifest` binds both outputs to the checked-out -commit. A fresh host may not have a Dashboard database yet, so this first build -uses `release:manifest` directly instead of the normal database-aware -`deploy:prepare`; first startup creates and migrates the database. +The one-shot initialization creates the fresh database in WAL mode and applies +the immutable migration registry. The staging command then installs frozen +dependencies, runs the normal database-aware `deploy:prepare`, and atomically +publishes only manifest-declared artifacts. +The control checkout is not a production runtime directory. See +[Production deploy](production-deploy.md) for the release/state layout, +automatic rollback, retention, and recovery contract. ## Configure Secrets -Dashboard reads production secrets through Doppler: - -```bash -cd /home/ubuntu/projects/mira-dashboard/backend -doppler run --config prd --project rajohan -- bun dist/serverStart.js -``` +Dashboard reads production secrets through Doppler project/config +`rajohan/prd`. Do not start `serverStart.js` manually to test them; the managed +systemd units below own the only production web and worker processes. See [Secrets and environment](secrets-and-env.md) for the full list. The minimum production setup normally needs: @@ -218,7 +258,7 @@ Healthy response shape: "release": { "backendCommit": "12345678", "frontendCommit": "12345678", - "manifestFormatVersion": 1, + "manifestFormatVersion": 2, "ready": true, "source": "manifest" }, diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 69541a795..51567bb0a 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -1,238 +1,295 @@ # Production Deploy -Dashboard production runs from: +Production separates source control, immutable code, and persistent state: -```text -/home/ubuntu/projects/mira-dashboard -``` +| Purpose | Path | +| ------------------------------------------------- | ------------------------------------------------- | +| Control checkout and deployment scripts | `/home/ubuntu/projects/mira-dashboard` | +| Temporary detached build worktrees | `/home/ubuntu/projects/mira-dashboard-worktrees/` | +| Immutable releases and `current`/`previous` links | `/home/ubuntu/projects/mira-dashboard-releases` | +| Persistent production state | `/home/ubuntu/projects/mira-dashboard-state` | -The service runs from the backend directory: +Web and worker execute from: ```text -/home/ubuntu/projects/mira-dashboard/backend +/home/ubuntu/projects/mira-dashboard-releases/current/backend ``` -## Deployment Model +`current` and `previous` are atomic relative symlinks to full-SHA directories +below `mira-dashboard-releases/releases/`. Production never builds in, writes +to, or executes backend code from the control checkout. -This is a single-host service: +## Persistent State -- frontend assets are built into `dist/`; -- backend TypeScript is built into `backend/dist/`; -- `mira-dashboard.service` runs the HTTP/WebSocket process from - `bun dist/serverStart.js` through Doppler; -- `mira-dashboard-worker.service` runs the persistent scheduler/executor from - `bun dist/workerStart.js` through Doppler; -- SQLite state lives under `backend/data/` unless `MIRA_DASHBOARD_DB_PATH` is set. -- both units use `UMask=0077`; startup enforces `0700` on the SQLite directory - and `0600` on database/sidecar files. +Mutable state is deliberately outside both Git and every release: -Both tracked units preserve the production environment contract by launching -through Doppler project/config `rajohan/prd`. Auth and origin settings such as -`MIRA_DASHBOARD_AUTOMATION_CREDENTIALS`, -`MIRA_DASHBOARD_SECRET_ENCRYPTION_KEY`, -`MIRA_DASHBOARD_WEBAUTHN_RP_ID`, -`MIRA_DASHBOARD_WEBAUTHN_ORIGINS`, and -`MIRA_DASHBOARD_ALLOWED_ORIGINS` remain owned by Doppler. Do not duplicate their -values in unit files. +| State | Stable path | +| ---------------------------------- | -------------------------------------------------------------- | +| SQLite database | `/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db` | +| SQLite WAL and shared-memory files | next to the database | +| Restore-verified SQLite backups | `/home/ubuntu/projects/mira-dashboard-state/backups/` | +| Dashboard Gateway device identity | `/home/ubuntu/projects/mira-dashboard-state/openclaw-client/` | +| Log-rotation lock | `/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock` | -There is no container image for the Dashboard service today. +The backup directory is derived from `dirname(MIRA_DASHBOARD_DB_PATH)`, so +pre-deploy and pre-migration snapshots automatically stay under the state root. +Kopia mounts `/home/ubuntu/projects` as its projects source; the separate state +directory remains in that backup scope. -## Prepare Deployment +`backend/config/log-rotation.json` is not mutable state. It is versioned +application configuration, is listed in every release manifest, and is copied +and checksum-verified with the other immutable release artifacts. -Install both dependency sets: +Both managed units set the stable paths explicitly: + +```text +MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client +MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock +MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current +MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +``` + +The OpenClaw home preserves the signed Gateway device identity across releases. +Secrets remain in Doppler `rajohan/prd`; tracked unit files contain no secret +values. + +## Normal Deployment + +The Dashboard worker owns the deployment: + +1. Require a clean control checkout and fast-forward `main`. +2. Create a detached build worktree below + `/home/ubuntu/projects/mira-dashboard-worktrees/`. +3. Install frozen frontend and backend dependencies. +4. Run `deploy:prepare` against the stable production database. +5. Verify the release manifest, component identities, schema contract, and + every checksummed artifact. +6. Copy only declared artifacts to a hidden directory and atomically publish + it as `releases/`. +7. Atomically switch `current`; retain the old release as `previous`. +8. Restart web and worker. +9. Require `/api/health/ready` to report the exact expected frontend/backend + commit and a fresh worker heartbeat from that commit. +10. On failure, switch back to `previous`, restart both units, verify the old + commit, and mark the deployment failed. +11. On success, retain `current`, `previous`, and one additional newest + verified release. + +The executor fails closed unless both units already run from managed +`current/backend` with the exact stable state paths. A deployment never modifies +the running release. + +## One-Time Managed Cutover + +Run this once after the atomic-executor change has been merged and built by the +old in-place deployment. The Jobs queue must be idle. PR #333 is the known-good +format-2 bootstrap release. + +### 1. Stage both releases while the old services remain online + +The existing database is still below the control checkout during this step. +Staging is read-safe and creates a restore-verified pre-deploy backup. +Run all four cutover sections in the same interactive shell so the validated +path and commit variables remain available. ```bash cd /home/ubuntu/projects/mira-dashboard -git pull --ff-only -bun install --frozen-lockfile -cd backend -bun install --frozen-lockfile -cd .. -/usr/local/bin/doppler run --config prd --project rajohan -- \ - bun run deploy:prepare +git switch main +git pull --ff-only origin main + +RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +OLD_STATE_ROOT=/home/ubuntu/projects/mira-dashboard/backend/data +STATE_ROOT=/home/ubuntu/projects/mira-dashboard-state +OLD_DATABASE_PATH="$OLD_STATE_ROOT/mira-dashboard.db" +OLD_OPENCLAW_CLIENT_HOME="$OLD_STATE_ROOT/openclaw-client" +OLD_LOG_ROTATION_LOCK="$OLD_STATE_ROOT/log-rotation.lock" +DATABASE_PATH="$STATE_ROOT/mira-dashboard.db" +OPENCLAW_CLIENT_HOME="$STATE_ROOT/openclaw-client" +LOG_ROTATION_LOCK="$STATE_ROOT/log-rotation.lock" +BOOTSTRAP_SHA=4aca68e0cffed68c42a630c4221e11e725ab294b +CANDIDATE_SHA="$(git rev-parse HEAD)" + +env \ + MIRA_DASHBOARD_DB_PATH="$OLD_DATABASE_PATH" \ + MIRA_DASHBOARD_OPENCLAW_HOME="$OLD_OPENCLAW_CLIENT_HOME" \ + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE="$OLD_LOG_ROTATION_LOCK" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun backend/src/releaseDeployment.ts stage "$BOOTSTRAP_SHA" +env \ + MIRA_DASHBOARD_DB_PATH="$OLD_DATABASE_PATH" \ + MIRA_DASHBOARD_OPENCLAW_HOME="$OLD_OPENCLAW_CLIENT_HOME" \ + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE="$OLD_LOG_ROTATION_LOCK" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun backend/src/releaseDeployment.ts stage "$CANDIDATE_SHA" ``` -`deploy:prepare` builds the frontend and backend, runs `db:preflight`, and -writes the checksummed release manifest before service restart. Keep ordinary -`build` commands side-effect free; use this combined command for every -supported manual or Dashboard-driven deploy so the database and release gates -cannot be skipped accidentally. +### 2. Stop both units and atomically move persistent state + +Keep copies of the installed pre-cutover units for the recovery procedure. +Both source and destination are below `/home/ubuntu/projects`, so `mv` is a +same-filesystem directory rename rather than a live database copy. + +```bash +CUTOVER_UNIT_BACKUP="$(mktemp -d)" +cp --preserve=mode,timestamps \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service \ + "$CUTOVER_UNIT_BACKUP/mira-dashboard.service" +cp --preserve=mode,timestamps \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service \ + "$CUTOVER_UNIT_BACKUP/mira-dashboard-worker.service" + +systemctl --user stop mira-dashboard-worker.service mira-dashboard.service +test -d "$OLD_STATE_ROOT" +test ! -e "$STATE_ROOT" +mv --no-target-directory "$OLD_STATE_ROOT" "$STATE_ROOT" +test -f "$DATABASE_PATH" +``` -## Install Or Refresh Units +Do not leave a compatibility symlink at `backend/data`. Production units use +the new absolute paths and development retains its normal local `backend/data` +default. -After building, install the tracked resource-limited units: +### 3. Activate the bootstrap and install managed units ```bash -cd /home/ubuntu/projects/mira-dashboard +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_OPENCLAW_HOME="$OPENCLAW_CLIENT_HOME" \ + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE="$LOG_ROTATION_LOCK" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun backend/src/releaseLifecycle.ts activate "$BOOTSTRAP_SHA" + install -m 0644 systemd/mira-dashboard.service \ /home/ubuntu/.config/systemd/user/mira-dashboard.service install -m 0644 systemd/mira-dashboard-worker.service \ /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service systemctl --user daemon-reload +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service ``` -For the first split-process rollout, restart the web unit with its explicit -`web` role before starting the worker. This avoids overlapping the legacy -combined scheduler with the dedicated worker: +Use a commit-bound readiness function; exhausting the loop is a failure: ```bash -systemctl --user restart mira-dashboard.service -systemctl --user enable --now mira-dashboard-worker.service +ready_for_commit() { + local full_sha="$1" + local expected="${full_sha:0:8}" + local response + for attempt in {1..30}; do + response="$(curl --fail --silent --show-error \ + http://127.0.0.1:3100/api/health/ready || true)" + if jq --exit-status --arg expected "$expected" \ + '.status == "isReady" + and .checks.release.backendCommit == $expected + and .checks.release.frontendCommit == $expected + and .checks.worker.ready == true' <<<"$response" >/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} + +ready_for_commit "$BOOTSTRAP_SHA" ``` -## Restart +If bootstrap readiness fails, stop both units, restore the saved unit files, +move the state directory back, reload systemd, and restart the old deployment: -Always tell Raymond before restarting OpenClaw Gateway. Dashboard restart is -safe after a merged/deployed Dashboard change. A web-only restart does not -interrupt queued/running actions. Before restarting the worker, verify the Jobs -queue is idle or explicitly accept that its active action will be cancelled: +```bash +systemctl --user stop mira-dashboard-worker.service mira-dashboard.service +install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard.service" \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service +install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard-worker.service" \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service +mv --no-target-directory "$STATE_ROOT" "$OLD_STATE_ROOT" +systemctl --user daemon-reload +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +``` + +Investigate before retrying. Do not continue to candidate activation. + +### 4. Activate and verify the candidate ```bash -systemctl --user restart mira-dashboard.service -systemctl --user restart mira-dashboard-worker.service -systemctl --user status mira-dashboard.service --no-pager -systemctl --user status mira-dashboard-worker.service --no-pager +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/releases/$BOOTSTRAP_SHA/backend/dist/releaseLifecycle.js" \ + activate "$CANDIDATE_SHA" +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service ``` -Logs: +Require `ready_for_commit "$CANDIDATE_SHA"`. If it fails: ```bash -journalctl --user -u mira-dashboard.service -n 120 --no-pager -journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/releases/$CANDIDATE_SHA/backend/dist/releaseLifecycle.js" \ + rollback +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +ready_for_commit "$BOOTSTRAP_SHA" ``` -## Smoke Test +Do not complete the cutover unless the restored bootstrap is ready. After a +successful candidate check: ```bash -wait_for_dashboard_ready() { - for attempt in {1..20}; do - if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ - http://127.0.0.1:3100/api/health/ready >/dev/null; then - return 0 - fi - sleep 1 - done - return 1 -} -wait_for_dashboard_ready -curl http://127.0.0.1:3100/api/auth/bootstrap +env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" prune 3 +gio trash -- "$CUTOVER_UNIT_BACKUP" ``` -Every other API route requires a valid Dashboard session or an explicitly -allowed minimum-scope bearer credential. Direct loopback is not an -authentication mechanism. A tokenless local check should therefore fail: +## Restart And Smoke Test + +Normal deploys schedule their own restart. For manual recovery, first confirm +no action is running, then: ```bash -test "$(curl --silent --output /dev/null --write-out '%{http_code}' \ - http://127.0.0.1:3100/api/cache/heartbeat)" = "401" +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +systemctl --user status mira-dashboard.service --no-pager +systemctl --user status mira-dashboard-worker.service --no-pager +journalctl --user -u mira-dashboard.service -n 120 --no-pager +journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager +curl --fail --silent --show-error \ + http://127.0.0.1:3100/api/health/ready | jq ``` -### Scoped Automation Rollout - -The release removes direct-loopback bypass code. Provision and migrate local -callers before restarting into this version: - -1. In an untracked privileged shell, generate a separate validator per - automation identity. Store only its SHA-256 hash plus minimum scopes in - `MIRA_DASHBOARD_AUTOMATION_CREDENTIALS`. Never generate it through Dashboard - Terminal or another tracked exec path. -2. Keep the full validator only in the caller's secret store. Do not put it in - a prompt, command argument, transcript, unit file, or the same configuration - surface as its hash. - On this host, use the four `0600` files under - `/home/ubuntu/.config/mira-dashboard/automation/` through - `/home/ubuntu/projects/mira-dashboard/scripts/miraDashboardApi.ts`. -3. Migrate and smoke-test every caller against the currently running - scoped-credential-compatible release: - - heartbeat: `cache:read`, `reports:write`; - - task tracking: `agents:write`, `tasks:read`, and `tasks:write`; - - daily summary: `cache:read`, `reports:write`; - - daily brief: `cache:read`, `reports:write`, `tasks:read`. -4. Confirm allowed and intentionally denied calls have the expected automation - actor and scope in `/api/audit-events`. -5. Deploy this release, restart the web unit, verify every scoped caller again, - and confirm tokenless loopback returns `401`. - -The OpenClaw heartbeat must retain its dedicated -`cache:read`/`reports:write` credential. -Task/report credentials must not be reused for heartbeat. - -For an authenticated browser session, also verify: - -- a pre-v6 session is rejected and a fresh login succeeds; -- first bootstrap still accepts username, password, and Gateway token, then - stores the Gateway token only as an encrypted envelope and directs the - operator to **Settings → Dashboard** for MFA enrollment; -- two named security keys can be registered and one can authenticate while the - other remains offline; -- TOTP and one-time recovery each complete a test verification; -- privileged actions require fresh second-factor verification; -- structured OpenClaw config is masked and raw reveal requires recent MFA; -- header/WebSocket status is connected; -- Jobs shows the execution queue and the worker becomes idle after startup seeds; -- Dashboard page cards load; -- Reports page loads recent reports; -- Notifications bell loads without global chat/tool errors. +Direct loopback is transport, not authentication. Tokenless protected API calls +must still return `401`. ## Rollback -Rollback is git-based: +Normal activation automatically rolls back on restart or commit-bound readiness +failure. Manual rollback is a failure-only operation: ```bash -cd /home/ubuntu/projects/mira-dashboard -git log --oneline -n 10 -git switch main -git reset --hard -if ! bun -e 'const packageJson = await Bun.file("package.json").json(); process.exit(typeof packageJson.scripts?.["deploy:prepare"] === "string" ? 0 : 1)'; then - echo "Rollback target predates the supported deploy contract" >&2 - exit 1 -fi -bun install --frozen-lockfile -(cd backend && bun install --frozen-lockfile) -if test -f scripts/writeReleaseManifest.ts; then - release_health_path=/api/health/ready -else - # One-time bootstrap rollback to the pre-manifest release. - release_health_path=/api/health -fi -/usr/local/bin/doppler run --config prd --project rajohan -- \ - bun run deploy:prepare -install -m 0644 systemd/mira-dashboard.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard.service -install -m 0644 systemd/mira-dashboard-worker.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service -systemctl --user daemon-reload -systemctl --user restart mira-dashboard.service -systemctl --user restart mira-dashboard-worker.service -wait_for_dashboard_ready() { - for attempt in {1..20}; do - if test "$release_health_path" = "/api/health"; then - if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ - "http://127.0.0.1:3100${release_health_path}" | - grep -Fq '"workerOnline":true'; then - return 0 - fi - elif curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ - "http://127.0.0.1:3100${release_health_path}" >/dev/null; then - return 0 - fi - sleep 1 - done - return 1 -} -wait_for_dashboard_ready +RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +DATABASE_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +env MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" status +env MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" rollback +systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +curl --fail --silent --show-error \ + http://127.0.0.1:3100/api/health/ready | jq ``` -The conditional health target exists only for the first rollback across the -manifest-contract cutover. Manifest-aware releases always regenerate their -ignored manifest through `deploy:prepare` and verify `/api/health/ready`. - -Rollback targets older than the split-worker/database-preflight contract are -deliberately unsupported. This private single-operator service keeps a supported -known-good release instead of retaining an untested legacy activation path. - -Do not use `git reset --hard` casually in normal work. It is a rollback -procedure for production incidents after an explicit decision. +Git reset and rebuilding in the control checkout are not production rollback +mechanisms. ## SQLite Deploy Lifecycle @@ -240,270 +297,90 @@ SQLite schema changes are numbered, immutable migrations recorded in `schema_migrations`. Build/deploy preflight: 1. requires the live database to be in WAL mode; -2. rejects unknown migration versions, gaps, names, or checksum drift; +2. rejects unknown versions, gaps, names, or checksum drift; 3. creates a WAL-consistent `pre-deploy` backup with `VACUUM INTO`; -4. copies that snapshot to an isolated restore location, requires - `PRAGMA quick_check = ok` plus valid migration history, and applies every - pending migration to the disposable copy; +4. restores that snapshot in isolation, requires `PRAGMA quick_check = ok`, + validates history, and applies pending migrations to the disposable copy; 5. applies bounded backup retention. On restart, web and worker independently validate history. `BEGIN IMMEDIATE` -serializes pending migrations and the second process revalidates after waiting -for the first. The process holding that writer lock creates a separate -restore-verified `pre-migration` backup through a read-only connection before -running migration SQL, so no other writer can commit between the rollback -snapshot and the migration. - -The first deployment that introduces this lifecycle cannot make the -already-running old worker call the new preflight command. Its new startup path -therefore creates the verified `pre-migration` backup before adopting the -legacy schema. Subsequent Dashboard deploys run both protections. +serializes pending migrations. The process holding the writer lock creates a +separate restore-verified `pre-migration` backup before running migration SQL. Do not copy only the main `.db` file while Dashboard is running. WAL mode may hold committed writes in the `-wal` sidecar until a checkpoint. Code rollback and data rollback are separate decisions. A code rollback may use the migrated database only when the older code is schema-compatible. Otherwise -stop both Dashboard units and restore the selected matching snapshot using the +stop both units and restore the matching snapshot using the [SQLite restore runbook](../operations/runbooks.md#restore-dashboard-sqlite). -### Schema Compatibility And Release Rollback Contract - -Classify every future migration before release: - -- **expand/backward-compatible:** the previous retained release can safely read - and write the migrated schema. An immutable release manager may switch code - back without restoring data; -- **contract/incompatible:** older code cannot safely use the resulting schema - or data semantics. Automatic code-only rollback must be blocked. - -Prefer expand/migrate/contract across separate releases. Add new structures -first, deploy code that tolerates both representations, backfill with a bounded -and resumable job, then remove the old representation only after the previous -release has left the rollback window. The contract migration gets a new -forward-only version; released migration files are never edited. - -If an incompatible change cannot be phased, treat activation as a coordinated -code-and-data cutover. This procedure becomes executable only after the final -deploy integration has switched both systemd units and the executor to the -managed `current` link; incompatible cutovers are unsupported while production -still uses the in-place checkout: - -1. run the candidate's production preflight, then record the release SHA, - supported schema range, and preflight result in the deployment record; -2. stop both Dashboard units for the cutover and verify the execution queue is - idle; -3. rerun the candidate's database preflight against the stable production - database, require restore verification, and record the newly created - `pre-deploy` snapshot; do not reuse the snapshot from step 1 because writes - may have committed before the units stopped; -4. activate the immutable release with the explicit - `--coordinated-schema-cutover` flag; -5. start both units through the managed `current` link, migrate forward on - startup, and run readiness against the new release and schema; -6. on failure, stop both units, restore the snapshot recorded in step 3, - switch the `current` release link back, and only then restart. - -The migration runner intentionally has no destructive down-migration path. -Unknown newer migration versions make older code fail closed. A future release -manager must therefore read the release/schema compatibility declaration before -offering or automatically performing rollback; it must never start an -incompatible older release against a newer live database. +### Schema Compatibility -## Release Manifest Contract +Classify every migration before release: -`bun run deploy:prepare` builds frontend and backend, completes the verified -SQLite preflight, and writes an ignored `release-manifest.json` in the release -root. The manifest is the release identity source when `NODE_ENV=production`; -Git is only a development/test fallback. - -Manifest format version 2 records: - -- the full and eight-character Git commit plus commit title and build time; -- the Bun version used for the build; -- matching frontend/backend commit identities emitted inside both build trees; -- the target, minimum-compatible, and maximum-compatible SQLite schema; -- a checksum of the immutable migration registry; -- the ordered migration identities and their inventory digest; -- the SHA-256 and byte length of every frontend/backend build artifact plus - both package manifests, Bun lockfiles, and the default runtime log-rotation - configuration. - -Format version 1 remains readable only for the first managed cutover and its -rollback window because the currently deployed release predates the migration -inventory and lifecycle artifact. Remove v1 support once neither `current` nor -`previous` can reference that release; all newly built releases are format v2. - -The backend bundle also embeds its full build commit. Runtime readiness requires -that embedded commit, both build-identity files, and the release manifest to -agree. Running `release:manifest` against ignored output left behind by another -checkout therefore fails instead of relabeling stale code. +- **expand/backward-compatible:** `previous` can safely use the migrated schema; +- **contract/incompatible:** older code cannot safely use the new schema or + data semantics, so automatic code-only rollback is blocked. -Manifest creation and verification reject absolute/traversal paths, symlinks, -hard-linked files, special files, unsorted/duplicate inventories, checksum -drift, and undeclared runtime artifacts. The schema compatibility range is an -explicit code constant. Adding a future migration without reviewing that range -fails the release contract. - -The release lifecycle layer validates immutable directories named by full Git -SHA under `/home/ubuntu/projects/mira-dashboard-releases/releases/`. This is the -production default; a deliberately configured `MIRA_DASHBOARD_RELEASES_ROOT` -overrides it. Run lifecycle commands from a known format-v2 release with the -same Doppler production environment as the services so schema checks always -inspect the live Dashboard database. Do not invoke the lifecycle CLI through -`current`: the first managed rollback may point `current` at the retained -format-v1 release, which does not contain that artifact. Record and retain the -first format-v2 release SHA as the management release until the v1 rollback -window closes. - -Set `DATABASE_PATH` to the exact stable absolute -`MIRA_DASHBOARD_DB_PATH` used by both production units, including when that -value normally comes from Doppler. Passing it after Doppler injection prevents -an immutable release from redirecting SQLite state or silently inspecting a -different configured database: +Prefer expand/migrate/contract across separate releases. If an incompatible +change cannot be phased, use a coordinated code-and-data cutover: -```bash -RELEASES_ROOT="/home/ubuntu/projects/mira-dashboard-releases" -DATABASE_PATH="/home/ubuntu/projects/mira-dashboard/backend/data/mira-dashboard.db" -LIFECYCLE_RELEASE_SHA="REPLACE_WITH_RETAINED_FORMAT_2_SHA" -CANDIDATE_RELEASE_SHA="REPLACE_WITH_CANDIDATE_FULL_SHA" -LIFECYCLE_CLI="$RELEASES_ROOT/releases/$LIFECYCLE_RELEASE_SHA/backend/dist/releaseLifecycle.js" -test -f "$LIFECYCLE_CLI" -test -f "$DATABASE_PATH" +1. require an idle execution queue; +2. stop both units; +3. rerun candidate preflight and record its fresh verified snapshot; +4. activate with `--coordinated-schema-cutover`; +5. start both units and require commit/schema readiness; +6. on failure, stop both units, restore the recorded snapshot, switch code back, + and only then restart. -# Inspect the current slots before activation. -doppler run --config prd --project rajohan -- \ - env MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ - NODE_ENV=production \ - MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ - bun "$LIFECYCLE_CLI" status - -# Activate only after preflight succeeds. -doppler run --config prd --project rajohan -- \ - env MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ - NODE_ENV=production \ - MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ - bun "$LIFECYCLE_CLI" activate "$CANDIDATE_RELEASE_SHA" -``` - -If production uses a non-default release root, replace `RELEASES_ROOT` with -that explicit absolute path. Likewise, replace `DATABASE_PATH` with the exact -path configured for the services. Passing both through `env` after Doppler -injection ensures the lifecycle process, shell-resolved CLI, and live database -always refer to the same production state. - -Rollback is a separate, failure-only operation. Do not run it as part of the -normal activation sequence: - -```bash -doppler run --config prd --project rajohan -- \ - env MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ - NODE_ENV=production \ - MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ - bun "$LIFECYCLE_CLI" rollback -``` - -`current` and `previous` are relative links inside the release root. Link -replacement uses same-directory temporary symlinks, atomic rename, and a -directory fsync, then re-verifies the linked release before committing the -transition. Activation verifies every artifact, both component build -identities, the exact manifest/directory SHA, the host Bun version, the actual -live SQLite schema, and the previous release's rollback window. Rollback also -checks the live schema rather than assuming it was downgraded by a code-only -rollback. +The migration runner has no destructive down-migration path. Unknown newer +migrations make older code fail closed. -The lifecycle CLI changes release links only; an already-running process keeps -executing the physical release it started from. After the final systemd -cutover, every activation and rollback must therefore restart both units and -verify release readiness before reporting success: +## Release Manifest Contract -```bash -systemctl --user restart mira-dashboard-worker.service -systemctl --user restart mira-dashboard.service -curl --fail --silent --show-error http://127.0.0.1:3100/api/health/ready -``` +`bun run deploy:prepare` builds frontend and backend, performs verified SQLite +preflight, and writes `release-manifest.json`. Format version 2 is the only +supported release format. It records: -The final deploy executor owns this sequence and automatically runs the same -restart/readiness checks after switching back on failure. The commands above -are the required manual fallback, not an optional post-deploy check. +- full/short Git identity, title, build time, and Bun version; +- matching frontend/backend build identities; +- target and compatible SQLite schema range; +- immutable migration identities and registry/inventory digests; +- SHA-256 and byte length for every frontend/backend artifact, package + manifest, Bun lockfile, and `backend/config/log-rotation.json`. -Normal activation refuses a schema target outside the current release's rollback -window. After the final systemd/executor cutover, the exceptional snapshot-backed -procedure above runs the candidate command only after preflight succeeds, both -services are stopped, their `WorkingDirectory`/`ExecStart` resolve through the -managed `current` link, and the queue is idle: +Manifest creation and verification reject absolute/traversal paths, symlinks, +hard-linked files, special files, unsorted/duplicate inventories, checksum +drift, and undeclared runtime artifacts. Runtime readiness requires the embedded +backend commit, both build identities, and manifest to agree. -```bash -doppler run --config prd --project rajohan -- \ - env MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ - NODE_ENV=production \ - MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ - bun "$LIFECYCLE_CLI" activate "$CANDIDATE_RELEASE_SHA" \ - --coordinated-schema-cutover -``` +Release link transitions use a kernel-owned `flock`, atomic same-directory +symlink replacement, directory fsync, and a durable recovery journal. Every +status read, activation, rollback, and interrupted-transition recovery verifies +the managed release and live schema contract. -The flag is rejected for ordinary compatible releases. It permits the candidate -startup migration across the incompatible boundary, but it does not permit -automatic code-only rollback afterward; restore the recorded matching snapshot -before switching back. - -Every status read, activation, rollback, and interrupted-transition recovery is -serialized by a kernel-owned `flock` held on an open descriptor. The kernel -releases the lock if the lifecycle process exits, so stale PID metadata and PID -reuse cannot block recovery. A durable transition journal is written before -either link changes. Status takes a shared lock and remains observational; it -fails clearly when a journal requires recovery. The next exclusive activation -or rollback restores the recorded pre-transition slots. Successful transitions -verify both final slots and remove only the exact journal inode they inspected, -so an interruption cannot discard the known-good rollback target. - -The Dashboard executor still uses the in-place transition flow until the final -deploy integration performs the controlled systemd cutover to these links. -That cutover must set both units' stable state paths explicitly before changing -their working directories: - -```ini -[Service] -Environment=MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard/backend/data/mira-dashboard.db -Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard/backend/data/openclaw-client -``` +## Health Signals -The OpenClaw home value preserves the existing signed Gateway device identity -at -`backend/data/openclaw-client/.openclaw/identity/device.json`. Leaving it unset -would derive a different path below each SHA-specific working directory. +- `GET /api/health/live` proves the web process can answer. +- `GET /api/health/ready` requires a valid release identity, compatible + accessible SQLite schema, built frontend, and a fresh worker heartbeat from + the exact release commit. It returns HTTP 503 with `status: "notReady"` when + an activation check fails. +- `GET /api/health/diagnostics` adds the authenticated readiness breakdown and + session count. -## Health Signals +There is no legacy `/health` or `/api/health` route. Production activation and +automatic rollback use `/api/health/ready`. -Deployment health is split by purpose: - -- `GET /api/health/live` proves that the web process can answer requests. -- `GET /api/health/ready` requires a valid release identity, - current/accessible SQLite schema, built frontend, and a fresh worker heartbeat - from the exact manifest commit. - Concurrent probes share one artifact scan, and a completed result is reused - for at most 15 seconds before the checksummed inventory is verified again. - This readiness route returns HTTP 503 with `status: "notReady"` when an - internal activation check fails. -- `GET /api/health/diagnostics` returns the readiness breakdown plus session - count and requires an authenticated Dashboard session. -- `GET /api/health` is a temporary compatibility adapter for the pre-readiness - deploy executor. It returns 503 unless the full readiness contract passes and - retains `workerOnline` only until the atomic executor cutover is complete. - -Gateway connectivity is reported as an external dependency but deliberately -does not fail release readiness: rolling Dashboard code back cannot repair an -OpenClaw Gateway outage. Production activation and automatic rollback must use -`/api/health/ready`. +Gateway connectivity is reported as an external dependency but does not fail +release readiness: rolling Dashboard code back cannot repair a Gateway outage. Important failures: -- `dependencies.gatewayConnected: false` in authenticated diagnostics: check - OpenClaw Gateway service and Gateway token. -- `checks.worker.ready: false`: the worker heartbeat is stale or queue telemetry is - unavailable; check both Dashboard and worker service logs. -- HTTP `503 Frontend Not Built`: build root frontend with `bun run build`. -- `Unauthorized` on API routes: auth/session or cookie issue. -- `database is locked`: another process is holding SQLite; retry after - background jobs settle, then inspect both service logs. Dashboard uses a - five-second SQLite busy timeout and requires WAL mode. +- `dependencies.gatewayConnected: false`: check Gateway service and token. +- `checks.worker.ready: false`: inspect worker heartbeat and both unit logs. +- HTTP `503 Frontend Not Built`: the release is incomplete and must not activate. +- `Unauthorized`: inspect Dashboard session or scoped automation credentials. +- `database is locked`: wait for background work, then inspect both service + logs; production requires WAL mode and uses a five-second busy timeout. diff --git a/docs/setup/secrets-and-env.md b/docs/setup/secrets-and-env.md index 7175a131a..ef2087cf6 100644 --- a/docs/setup/secrets-and-env.md +++ b/docs/setup/secrets-and-env.md @@ -23,32 +23,44 @@ Environment token precedence is: ## Dashboard Storage And Paths -| Variable | Required | Default | Purpose | -| ------------------------------ | -------- | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -| `MIRA_DASHBOARD_DB_PATH` | Optional | `backend/data/mira-dashboard.db` from backend cwd | SQLite database path. | -| `MIRA_DASHBOARD_FRONTEND_PATH` | Optional | repo `dist/` | Static frontend build served by the backend. | -| `OPENCLAW_HOME` | Optional | `~/.openclaw` | Primary OpenClaw home for file/config/media/agent lookups when set. | -| `MIRA_DASHBOARD_OPENCLAW_HOME` | Optional | `~/.openclaw` | Dashboard-specific fallback OpenClaw home. Most file/config/media routes use this only when `OPENCLAW_HOME` is absent. | -| `WORKSPACE_ROOT` | Optional | OpenClaw workspace | Root exposed by `/api/files`. Must be absolute and normalized if set. | -| `MIRA_DASHBOARD_LOGS_ROOT` | Optional | system log root default | Root used by log stream services. | +| Variable | Required | Default | Purpose | +| --------------------------------------- | ------------------------------ | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `MIRA_DASHBOARD_DB_PATH` | Explicit in production units | `backend/data/mira-dashboard.db` from backend cwd | SQLite database path. Production uses `/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db`. | +| `MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE` | Explicit in production units | `backend/data/log-rotation.lock` from backend cwd | Stable cross-release lock for elevated log rotation. | +| `MIRA_DASHBOARD_FRONTEND_PATH` | Optional | repo `dist/` | Static frontend build served by the backend. | +| `OPENCLAW_HOME` | Optional | `~/.openclaw` | Primary OpenClaw home for file/config/media/agent lookups when set. | +| `MIRA_DASHBOARD_OPENCLAW_HOME` | Explicit in production units | `backend/data/openclaw-client` from backend cwd | Dashboard Gateway-client identity home. Production uses the persistent state root so its signed device identity survives releases. | +| `MIRA_DASHBOARD_RELEASE_ROOT` | Explicit in production units | inferred runtime root | Active immutable release root; production uses `/home/ubuntu/projects/mira-dashboard-releases/current`. | +| `MIRA_DASHBOARD_RELEASES_ROOT` | Explicit in production tooling | `/home/ubuntu/projects/mira-dashboard-releases` | Managed release layout containing `releases/`, `current`, `previous`, locks, and transition journal. | +| `WORKSPACE_ROOT` | Optional | OpenClaw workspace | Root exposed by `/api/files`. Must be absolute and normalized if set. | +| `MIRA_DASHBOARD_LOGS_ROOT` | Optional | system log root default | Root used by log stream services. | `MIRA_DASHBOARD_FRONTEND_PATH` is a development/test escape hatch. Production serves the active release's checksummed `dist/`; any configured value that does not resolve exactly to that directory is rejected. +Production mutable state lives in +`/home/ubuntu/projects/mira-dashboard-state`, outside both the control checkout +and immutable releases. SQLite backups are derived as +`dirname(MIRA_DASHBOARD_DB_PATH)/backups`, so the production backup directory is +`/home/ubuntu/projects/mira-dashboard-state/backups/`. Versioned +`backend/config/` files are release artifacts, not external state. +`OPENCLAW_HOME` remains the primary OpenClaw installation/configuration root; +`MIRA_DASHBOARD_OPENCLAW_HOME` is the separate Dashboard client identity root. + ## Network, Auth, And Browser Access -| Variable | Required | Default | Purpose | -| ------------------------------------------- | --------------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------ | -| `MIRA_DASHBOARD_ALLOWED_ORIGINS` | Production browser access | same-origin/localhost behavior | Comma-separated allowed origins for browser/WebSocket checks. | -| `MIRA_DASHBOARD_AUTOMATION_CREDENTIALS` | Local non-browser callers | none | Strict JSON list of hash-only, minimum-scope automation credentials. There is no loopback auth bypass. | -| `MIRA_DASHBOARD_SECRET_ENCRYPTION_KEY` | Always | none | Base64 that decodes to exactly 32 bytes. External AES-256-GCM key for persisted Gateway token and TOTP seeds; preserve it with backups. | -| `MIRA_DASHBOARD_WEBAUTHN_RP_ID` | Security-key enrollment/use | none | Stable DNS relying-party id, for example `dashboard.example.com`. Raw IP addresses are rejected. | -| `MIRA_DASHBOARD_WEBAUTHN_ORIGINS` | Security-key enrollment/use | none | Explicit comma-separated HTTPS origins belonging to the RP ID. `http://localhost` is allowed for dev only. | -| `MIRA_DASHBOARD_SESSION_IDLE_MINUTES` | Optional | `30` | Idle session lifetime, integer `5`–`1440`. Polling alone does not refresh it. | -| `MIRA_DASHBOARD_RECENT_AUTH_MINUTES` | Optional | `10` | Fresh password/MFA verification window, integer `1`–`60`. | -| `MIRA_DASHBOARD_TRUSTED_PROXY_IPS` | Optional | none | Trusted proxy IPs. Only use if the proxy strips or overwrites untrusted forwarding headers. | -| `OPENCLAW_GATEWAY_URL` | Optional | `ws://127.0.0.1:18789` | Gateway WebSocket URL. | +| Variable | Required | Default | Purpose | +| --------------------------------------- | --------------------------- | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `MIRA_DASHBOARD_ALLOWED_ORIGINS` | Production browser access | same-origin/localhost behavior | Comma-separated allowed origins for browser/WebSocket checks. | +| `MIRA_DASHBOARD_AUTOMATION_CREDENTIALS` | Local non-browser callers | none | Strict JSON list of hash-only, minimum-scope automation credentials. There is no loopback auth bypass. | +| `MIRA_DASHBOARD_SECRET_ENCRYPTION_KEY` | Always | none | Base64 that decodes to exactly 32 bytes. External AES-256-GCM key for persisted Gateway token and TOTP seeds; preserve it with backups. | +| `MIRA_DASHBOARD_WEBAUTHN_RP_ID` | Security-key enrollment/use | none | Stable DNS relying-party id, for example `dashboard.example.com`. Raw IP addresses are rejected. | +| `MIRA_DASHBOARD_WEBAUTHN_ORIGINS` | Security-key enrollment/use | none | Explicit comma-separated HTTPS origins belonging to the RP ID. `http://localhost` is allowed for dev only. | +| `MIRA_DASHBOARD_SESSION_IDLE_MINUTES` | Optional | `30` | Idle session lifetime, integer `5`–`1440`. Polling alone does not refresh it. | +| `MIRA_DASHBOARD_RECENT_AUTH_MINUTES` | Optional | `10` | Fresh password/MFA verification window, integer `1`–`60`. | +| `MIRA_DASHBOARD_TRUSTED_PROXY_IPS` | Optional | none | Trusted proxy IPs. Only use if the proxy strips or overwrites untrusted forwarding headers. | +| `OPENCLAW_GATEWAY_URL` | Optional | `ws://127.0.0.1:18789` | Gateway WebSocket URL. | See [Auth and trust boundaries](../security/auth-and-trust-boundaries.md) for route auth, scope names, token generation, two-step login, proxy trust, diff --git a/systemd/mira-dashboard-worker.service b/systemd/mira-dashboard-worker.service index 4a40f3b11..54b44efa8 100644 --- a/systemd/mira-dashboard-worker.service +++ b/systemd/mira-dashboard-worker.service @@ -6,11 +6,16 @@ Wants=network-online.target [Service] Type=simple UMask=0077 -WorkingDirectory=/home/ubuntu/projects/mira-dashboard/backend +WorkingDirectory=/home/ubuntu/projects/mira-dashboard-releases/current/backend Environment=NODE_ENV=production Environment=MIRA_DASHBOARD_EXECUTION_ROLE=worker Environment=MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 Environment=MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard-worker.service +Environment=MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock +Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client +Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current +Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/workerStart.js Restart=on-failure RestartSec=5 diff --git a/systemd/mira-dashboard.service b/systemd/mira-dashboard.service index bab90f861..b7dd20f88 100644 --- a/systemd/mira-dashboard.service +++ b/systemd/mira-dashboard.service @@ -6,11 +6,16 @@ Wants=network-online.target [Service] Type=simple UMask=0077 -WorkingDirectory=/home/ubuntu/projects/mira-dashboard/backend +WorkingDirectory=/home/ubuntu/projects/mira-dashboard-releases/current/backend Environment=NODE_ENV=production Environment=MIRA_DASHBOARD_EXECUTION_ROLE=web Environment=MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 Environment=MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard.service +Environment=MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock +Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client +Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current +Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/serverStart.js Restart=on-failure RestartSec=5 From bdebc02232965d2b609278ad1d8de014d2e8d829 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 06:16:27 +0200 Subject: [PATCH 02/11] fix(deploy): harden atomic cutover contracts --- backend/src/lib/safePath.ts | 13 +++ backend/src/lib/values.ts | 10 ++ backend/src/releaseDeployment.ts | 29 +++-- backend/src/releaseManager.ts | 67 +++++++---- backend/src/server.ts | 12 +- backend/src/services/logRotation.ts | 61 +++++----- backend/src/services/pullRequests.ts | 110 +++++++++--------- backend/test/releaseDeployment.test.ts | 154 +++++++++++++------------ backend/test/releaseManager.test.ts | 44 ++++++- backend/test/serviceBehavior.test.ts | 136 ++++++++++------------ backend/test/support/releaseFixture.ts | 70 +++++++++++ backend/test/utilityBehavior.test.ts | 12 ++ docs/setup/production-deploy.md | 104 +++++++++++------ systemd/mira-dashboard-worker.service | 2 +- systemd/mira-dashboard.service | 2 +- 15 files changed, 505 insertions(+), 321 deletions(-) create mode 100644 backend/test/support/releaseFixture.ts diff --git a/backend/src/lib/safePath.ts b/backend/src/lib/safePath.ts index b4004ca7a..66719ae33 100644 --- a/backend/src/lib/safePath.ts +++ b/backend/src/lib/safePath.ts @@ -38,6 +38,19 @@ function isFilesystemRoot(rootPath: string): boolean { return path.parse(rootPath).root === rootPath; } +/** Resolves an absolute path while rejecting empty, null-byte, relative, and root paths. */ +export function resolveAbsoluteNonRootPath(value: string, label: string): string { + const trimmed = value.trim(); + if (!trimmed || trimmed.includes("\0") || !path.isAbsolute(trimmed)) { + throw new TypeError(`${label} must be an absolute non-root path`); + } + const resolved = path.resolve(trimmed); + if (isFilesystemRoot(resolved)) { + throw new TypeError(`${label} must be an absolute non-root path`); + } + return resolved; +} + function isWithinCanonicalRoot(candidate: string, root: string, normalizedRoot: string) { return candidate === root || candidate.startsWith(normalizedRoot); } diff --git a/backend/src/lib/values.ts b/backend/src/lib/values.ts index a3ade33de..680670bba 100644 --- a/backend/src/lib/values.ts +++ b/backend/src/lib/values.ts @@ -10,6 +10,16 @@ export function nonEmptyEnvironmentFallback(name: string, fallback: string): str return value && value.length > 0 ? value : fallback; } +/** Returns the validated effective Dashboard listen port. */ +export function resolveDashboardPort(value = process.env.PORT): number { + const trimmed = value?.trim() ?? ""; + if (!/^\d+$/u.test(trimmed)) { + return 3100; + } + const port = Number(trimmed); + return port > 0 && port <= 65_535 ? port : 3100; +} + /** Converts optional values to strings while preserving empty/undefined fallback behavior. */ export function stringFallback(value?: unknown, fallback = ""): string { return String(value ?? fallback); diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index 63483a1bc..6a291d92f 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -4,6 +4,7 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { runProcess } from "./lib/processes.ts"; +import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; import { type DashboardReleaseRetentionResult, ensureDashboardReleaseLayout, @@ -32,6 +33,13 @@ export const MANAGED_DASHBOARD_UNITS = { "mira-dashboard-worker.service": "dist/workerStart.js", "mira-dashboard.service": "dist/serverStart.js", } as const; +export const MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT = [ + "MIRA_DASHBOARD_DB_PATH", + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", + "MIRA_DASHBOARD_OPENCLAW_HOME", + "MIRA_DASHBOARD_RELEASE_ROOT", + "MIRA_DASHBOARD_RELEASES_ROOT", +] as const; export interface DashboardReleaseCommandResult { stderr: string; @@ -77,18 +85,6 @@ function assertFullCommitSha(commitSha: string): string { return commitSha; } -function resolveAbsoluteNonRootPath(value: string, label: string): string { - const trimmed = value.trim(); - if (!trimmed || trimmed.includes("\0") || !path.isAbsolute(trimmed)) { - throw new TypeError(`${label} must be an absolute non-root path`); - } - const resolved = path.resolve(trimmed); - if (resolved === path.parse(resolved).root) { - throw new TypeError(`${label} must be an absolute non-root path`); - } - return resolved; -} - function hasExactEnvironmentAssignment( serializedEnvironment: string, assignment: string @@ -356,6 +352,14 @@ export function assertManagedDashboardUnitProperties( if (!hasExactSerializedToken(execStart, MANAGED_DASHBOARD_UNITS[unit])) { throw new Error(`${unit} has an unexpected managed release entrypoint`); } + const preservedEnvironment = `--preserve-env=${MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT.join( + "," + )}`; + if (!hasExactSerializedToken(execStart, preservedEnvironment)) { + throw new Error( + `${unit} must preserve managed release environment through Doppler` + ); + } const environment = actual.get("Environment") ?? ""; const missingEnvironment = expectedEnvironment.filter( (entry) => !hasExactEnvironmentAssignment(environment, entry) @@ -413,6 +417,7 @@ export async function stageDashboardRelease( MIRA_DASHBOARD_DB_PATH: contract.databasePath, MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE: contract.logRotationLockFile, MIRA_DASHBOARD_OPENCLAW_HOME: contract.openClawHome, + MIRA_DASHBOARD_RELEASE_ROOT: worktreePath, MIRA_DASHBOARD_RELEASES_ROOT: contract.releasesRoot, NODE_ENV: "production", }; diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index 3ab686b87..e3724265e 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -13,6 +13,7 @@ import { import { readAppliedDatabaseMigrationHistory } from "./databaseMigrationRunner.ts"; import type { DatabaseMigrationIdentity } from "./databaseMigrations/index.ts"; import { guardedPath, writeTextNoFollowGuarded } from "./lib/guardedOps.ts"; +import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; import { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY, type DashboardReleaseManifest, @@ -58,6 +59,7 @@ export interface DashboardReleaseState { export interface DashboardReleaseRetentionResult { removed: string[]; retained: string[]; + warnings: string[]; } export interface DashboardReleaseManagerOptions { @@ -100,17 +102,6 @@ function assertReleaseCommitSha(commitSha: string): void { } } -function assertAbsoluteNonRootPath(value: string): string { - if (!value || value.includes("\0") || !path.isAbsolute(value)) { - throw new TypeError("Dashboard releases root must be an absolute non-root path"); - } - const resolved = path.resolve(value); - if (resolved === path.parse(resolved).root) { - throw new TypeError("Dashboard releases root must be an absolute non-root path"); - } - return resolved; -} - async function assertRealDirectory(directoryPath: string): Promise { const stat = await fsp.lstat(directoryPath); if (!stat.isDirectory() || stat.isSymbolicLink()) { @@ -332,13 +323,13 @@ export function resolveDashboardReleasesRoot( configuredRoot = process.env.MIRA_DASHBOARD_RELEASES_ROOT ?? DEFAULT_DASHBOARD_RELEASES_ROOT ): string { - return assertAbsoluteNonRootPath(configuredRoot.trim()); + return resolveAbsoluteNonRootPath(configuredRoot, "Dashboard releases root"); } export async function ensureDashboardReleaseLayout( configuredRoot = resolveDashboardReleasesRoot() ): Promise { - const root = assertAbsoluteNonRootPath(configuredRoot); + const root = resolveAbsoluteNonRootPath(configuredRoot, "Dashboard releases root"); const releasesPath = path.join(root, MANAGED_RELEASES_DIRECTORY_NAME); await assertRealDirectory(path.dirname(root)); try { @@ -363,7 +354,7 @@ export async function ensureDashboardReleaseLayout( export function managedReleasePath(releasesRoot: string, commitSha: string): string { assertReleaseCommitSha(commitSha); return path.join( - assertAbsoluteNonRootPath(releasesRoot), + resolveAbsoluteNonRootPath(releasesRoot, "Dashboard releases root"), MANAGED_RELEASES_DIRECTORY_NAME, commitSha ); @@ -1229,7 +1220,11 @@ export async function pruneDashboardReleases( withFileTypes: true, }); let hasFilesystemChanges = false; - const releases: ManagedDashboardRelease[] = []; + const releases: Array<{ + publishedAtNs: bigint; + release: ManagedDashboardRelease; + }> = []; + const warnings: string[] = []; for (const entry of entries) { if (RETIRED_RELEASE_DIRECTORY_PATTERN.test(entry.name)) { if (!entry.isDirectory() || entry.isSymbolicLink()) { @@ -1251,17 +1246,43 @@ export async function pruneDashboardReleases( `Managed release entry must be a real directory: ${entry.name}` ); } - releases.push(await loadManagedReleaseFromLayout(layout, entry.name)); + try { + const release = await loadManagedReleaseFromLayout(layout, entry.name); + const directoryStat = await fsp.lstat(release.path, { bigint: true }); + if ( + !directoryStat.isDirectory() || + directoryStat.isSymbolicLink() || + releaseDirectoryIdentity(directoryStat) !== release.directoryIdentity + ) { + throw new Error( + `Managed release changed before retention ordering: ${entry.name}` + ); + } + releases.push({ + publishedAtNs: directoryStat.birthtimeNs, + release, + }); + } catch (error) { + if (protectedCommits.has(entry.name)) { + throw error; + } + warnings.push(`Skipped unverifiable release ${entry.name}`); + } } const newestFirst = releases.toSorted((left, right) => { - const builtAtComparison = right.manifest.builtAt.localeCompare( - left.manifest.builtAt - ); - return builtAtComparison || right.commitSha.localeCompare(left.commitSha); + if (left.release.manifest.builtAt !== right.release.manifest.builtAt) { + return left.release.manifest.builtAt < right.release.manifest.builtAt + ? 1 + : -1; + } + if (left.publishedAtNs === right.publishedAtNs) { + return 0; + } + return left.publishedAtNs < right.publishedAtNs ? 1 : -1; }); const retained = new Set(protectedCommits); - for (const release of newestFirst) { + for (const { release } of newestFirst) { if (retained.size >= retainCount) { break; } @@ -1269,7 +1290,7 @@ export async function pruneDashboardReleases( } const removed: string[] = []; - for (const release of newestFirst.toReversed()) { + for (const { release } of newestFirst.toReversed()) { if (retained.has(release.commitSha)) { continue; } @@ -1310,8 +1331,10 @@ export async function pruneDashboardReleases( return { removed, retained: newestFirst + .map(({ release }) => release) .filter((release) => retained.has(release.commitSha)) .map((release) => release.commitSha), + warnings: warnings.toSorted(compareStrings), }; }); } diff --git a/backend/src/server.ts b/backend/src/server.ts index 965d6f8e2..0e4aa6842 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -15,6 +15,7 @@ import type { DashboardSocket } from "./dashboardSocket.ts"; import { resolveFrontendPath } from "./frontendAssets.ts"; import gateway from "./gateway.ts"; import { isAllowedDashboardOrigin, sessionIdFromCookie } from "./http.ts"; +import { resolveDashboardPort } from "./lib/values.ts"; import { requiresRecentMfaForGatewayMethod } from "./requestPolicy.ts"; import { withRequestSecurity } from "./requestSecurity.ts"; import { routes } from "./routes.ts"; @@ -76,14 +77,7 @@ function hasHiddenStaticSegment(relativePath: string): boolean { return relativePath.split(path.sep).some((segment) => segment.startsWith(".")); } -export function resolveListenPort(value = process.env.PORT): number { - const trimmed = value?.trim() ?? ""; - if (!/^\d+$/u.test(trimmed)) { - return 3100; - } - const port = Number(trimmed); - return port > 0 && port <= 65_535 ? port : 3100; -} +export { resolveDashboardPort as resolveListenPort } from "./lib/values.ts"; function dashboardSocketFromBun( ws: ServerWebSocket @@ -106,7 +100,7 @@ function dashboardSocketFromBun( }; } -export function createServer(port = resolveListenPort()): Server { +export function createServer(port = resolveDashboardPort()): Server { validateAuthenticationConfig(); validateStoredSecretConfig(); validateAutomationCredentials(); diff --git a/backend/src/services/logRotation.ts b/backend/src/services/logRotation.ts index a91af96dc..3d2d47e1d 100644 --- a/backend/src/services/logRotation.ts +++ b/backend/src/services/logRotation.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { database } from "../database.ts"; import { runProcess } from "../lib/processes.ts"; +import { resolveAbsoluteNonRootPath } from "../lib/safePath.ts"; import { writeCacheSuccess } from "./cacheEntryWriter.ts"; import { getScheduledJob, @@ -68,23 +69,21 @@ const ELEVATED_LOG_ROTATION_MAX_BUFFER = 16 * 1024 * 1024; const LOG_ROTATION_JOB_ID = "ops.log-rotation"; const LOG_ROTATION_FAILURE_OUTPUT_MAX_CHARS = 100_000; const BUN_EXECUTABLE = process.env.BUN_BINARY || "bun"; +const ELEVATED_LOG_ROTATION_PRESERVED_ENVIRONMENT = [ + "LANG", + "NODE_ENV", + "TZ", + "MIRA_DASHBOARD_DB_PATH", + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", +] as const; + function resolveLogRotationLockFile(): string { - const configured = - process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE?.trim() || DEFAULT_LOCK_FILE; - if ( - configured.includes("\0") || - !path.isAbsolute(configured) || - path.resolve(configured) === path.parse(path.resolve(configured)).root - ) { - throw new TypeError( - "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE must be an absolute non-root path" - ); - } - return path.resolve(configured); + return resolveAbsoluteNonRootPath( + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE?.trim() || DEFAULT_LOCK_FILE, + "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE" + ); } -const logRotationLockFile = resolveLogRotationLockFile(); - type ExecFileRunner = ( file: string, arguments_: readonly string[], @@ -1486,15 +1485,22 @@ async function processRotationCandidate({ } } -async function acquireLogRotationLock(isDryRun: boolean) { +interface LogRotationLock { + file: fs.FileHandle; + path: string; +} + +async function acquireLogRotationLock( + isDryRun: boolean +): Promise { if (isDryRun) return; - const lockFile = logRotationLockFile; + const lockFile = resolveLogRotationLockFile(); await fs.mkdir(path.dirname(lockFile), { recursive: true }); const openLock = async () => { const handle = await fs.open(lockFile, "wx"); try { await handle.writeFile(`${process.pid}\n`); - return handle; + return { file: handle, path: lockFile }; } catch (error) { await ignoreRejection(handle.close()); await ignoreRejection(fs.unlink(lockFile)); @@ -1517,8 +1523,8 @@ async function acquireLogRotationLock(isDryRun: boolean) { async function reclaimStaleLogRotationLock( lockFile: string, - openLock: () => Promise -) { + openLock: () => Promise +): Promise { const reclaimDirectory = `${lockFile}.reclaim`; try { await fs.mkdir(reclaimDirectory); @@ -1619,17 +1625,16 @@ function isProcessRunning(pid: number): boolean { } } -async function releaseLogRotationLock(handle: fs.FileHandle | undefined) { - if (!handle) return; - const lockFile = logRotationLockFile; +async function releaseLogRotationLock(lock: LogRotationLock | undefined) { + if (!lock) return; try { - const heldStat = await handle.stat(); - const pathStat = await fs.stat(lockFile); + const heldStat = await lock.file.stat(); + const pathStat = await fs.stat(lock.path); if (pathStat && pathStat.dev === heldStat.dev && pathStat.ino === heldStat.ino) { - await fs.unlink(lockFile); + await fs.unlink(lock.path); } } finally { - await handle.close(); + await lock.file.close(); } } @@ -2034,7 +2039,7 @@ function buildElevatedLogRotationCliArguments( ].join("\n"); return [ "-n", - "-E", + `--preserve-env=${ELEVATED_LOG_ROTATION_PRESERVED_ENVIRONMENT.join(",")}`, resolveBunExecutable(), "--input-type=module", "--eval", @@ -2056,7 +2061,7 @@ function elevatedLogRotationEnvironment(): NodeJS.ProcessEnv { "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", ]; const environment: NodeJS.ProcessEnv = {}; - // Keep sudo -E narrow: only runtime lookup, home/locale, mode, and stable state paths. + // Keep sudo environment preservation narrow: runtime lookup, locale, and state paths. for (const key of allowed) { if (process.env[key] !== undefined) { environment[key] = process.env[key]; diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 987241ec4..17de20093 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -8,7 +8,7 @@ import { runProcess, spawnProcess, } from "../lib/processes.ts"; -import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; +import { nonEmptyEnvironmentFallback, resolveDashboardPort } from "../lib/values.ts"; import { assertManagedDashboardUnitProperties, MANAGED_DASHBOARD_UNITS, @@ -16,10 +16,8 @@ import { stageDashboardRelease, } from "../releaseDeployment.ts"; import { - activateDashboardRelease, readDashboardReleaseState, resolveDashboardReleasesRoot, - rollbackDashboardRelease, } from "../releaseManager.ts"; import { enqueueJobExecution, @@ -1487,12 +1485,19 @@ async function assertManagedDashboardServiceContract( /** Schedules detached service restart, commit-bound readiness, and rollback. */ async function scheduleReleaseCutover( job: DeploymentJob, + candidateCommit: string, rollbackCommit: string, signal?: AbortSignal ): Promise { if (!job.commit || !/^[\da-f]{8}$/u.test(job.commit)) { throw new TypeError("Release cutover requires an eight-character commit"); } + if ( + !/^[\da-f]{40}$/u.test(candidateCommit) || + candidateCommit.slice(0, 8) !== job.commit + ) { + throw new TypeError("Release cutover requires the matching full candidate SHA"); + } if (!/^[\da-f]{8}$/u.test(rollbackCommit)) { throw new TypeError( "Release cutover requires an eight-character rollback commit" @@ -1506,6 +1511,14 @@ async function scheduleReleaseCutover( "dist", "releaseLifecycle.js" ); + const readinessUrl = `http://127.0.0.1:${resolveDashboardPort()}/api/health/ready`; + const lifecycleEnvironment = [ + `MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)}`, + `MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())}`, + "NODE_ENV=production", + shellQuote(resolveBunExecutable()), + shellQuote(lifecycleCommand), + ].join(" "); const okJob: DeploymentJob = { ...job, status: "isOk", @@ -1528,13 +1541,19 @@ async function scheduleReleaseCutover( updatedAt: dateToISOString(new Date()), note: `Release readiness failed and automatic rollback to ${rollbackCommit} failed`, }; + const activationFailedJob: DeploymentJob = { + ...job, + status: "failed", + updatedAt: dateToISOString(new Date()), + note: "Release activation failed before restart; guardian left current unchanged", + }; const script = [ "sleep 2", "ready_for_commit() {", ' expected_commit="$1"', " for attempt in {1..30}; do", - " response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 http://127.0.0.1:3100/api/health/ready 2>/dev/null || true)", + ` response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 ${shellQuote(readinessUrl)} 2>/dev/null || true)`, ' if printf "%s" "$response" | /usr/bin/jq --exit-status --arg expected "$expected_commit" \'.status == "isReady" and .checks.release.ready == true and .checks.release.backendCommit == $expected and .checks.release.frontendCommit == $expected and .checks.worker.ready == true\' >/dev/null 2>&1; then', " return 0", " fi", @@ -1545,18 +1564,22 @@ async function scheduleReleaseCutover( "restart_services() {", ` /usr/bin/systemctl --user restart ${DASHBOARD_SERVICES.join(" ")}`, "}", - `if restart_services && ready_for_commit ${shellQuote(job.commit)}; then`, - ` if MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)} MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())} NODE_ENV=production ${shellQuote(resolveBunExecutable())} ${shellQuote(lifecycleCommand)} prune 3; then`, - ` ${deploymentJobUpdateCommand(okJob)}`, + `if ${lifecycleEnvironment} activate ${shellQuote(candidateCommit)}; then`, + ` if restart_services && ready_for_commit ${shellQuote(job.commit)}; then`, + ` if ${lifecycleEnvironment} prune 3; then`, + ` ${deploymentJobUpdateCommand(okJob)}`, + " else", + ` ${deploymentJobUpdateCommand(okWithRetentionWarningJob)}`, + " fi", " else", - ` ${deploymentJobUpdateCommand(okWithRetentionWarningJob)}`, + ` if ${lifecycleEnvironment} rollback && restart_services && ready_for_commit ${shellQuote(rollbackCommit)}; then`, + ` ${deploymentJobUpdateCommand(rolledBackJob)}`, + " else", + ` ${deploymentJobUpdateCommand(rollbackFailedJob)}`, + " fi", " fi", "else", - ` if MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)} MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())} NODE_ENV=production ${shellQuote(resolveBunExecutable())} ${shellQuote(lifecycleCommand)} rollback && restart_services && ready_for_commit ${shellQuote(rollbackCommit)}; then`, - ` ${deploymentJobUpdateCommand(rolledBackJob)}`, - " else", - ` ${deploymentJobUpdateCommand(rollbackFailedJob)}`, - " fi", + ` ${deploymentJobUpdateCommand(activationFailedJob)}`, "fi", ].join("\n"); @@ -1618,50 +1641,31 @@ async function runDeploymentJob( worktreeRoot: getDashboardWorktreeRoot(), }); currentJob = refreshDeploymentHeartbeat(currentJob); - const activated = await activateDashboardRelease(expectedCommit, releasesRoot); - const didActivateNewRelease = currentState.current.commitSha !== expectedCommit; - if ( - activated.current?.commitSha !== expectedCommit || - !activated.previous || - activated.previous.commitSha === expectedCommit - ) { - if (didActivateNewRelease) { - await rollbackDashboardRelease(releasesRoot); - } + const rollbackRelease = + currentState.current.commitSha === expectedCommit + ? currentState.previous + : currentState.current; + if (!rollbackRelease || rollbackRelease.commitSha === expectedCommit) { throw new Error( - "Managed release activation did not preserve a distinct rollback release" + "Managed deployment requires a distinct verified rollback release" ); } - try { - const restartScheduled: DeploymentJob = { - ...currentJob, - status: "restart-scheduled", - updatedAt: dateToISOString(new Date()), - commit: candidate.manifest.commitShort, - commitTitle: candidate.manifest.commitTitle, - note: "Immutable release published. Atomic restart and rollback check scheduled", - }; - writeDeploymentJob(restartScheduled); - await scheduleReleaseCutover( - restartScheduled, - activated.previous.manifest.commitShort, - signal - ); - } catch (error) { - if (didActivateNewRelease) { - try { - await rollbackDashboardRelease(releasesRoot); - } catch (rollbackError) { - throw new AggregateError( - [error, rollbackError], - "Deployment scheduling failed and release-link rollback also failed", - { cause: rollbackError } - ); - } - } - throw error; - } + const restartScheduled: DeploymentJob = { + ...currentJob, + status: "restart-scheduled", + updatedAt: dateToISOString(new Date()), + commit: candidate.manifest.commitShort, + commitTitle: candidate.manifest.commitTitle, + note: "Immutable release published. Detached activation and rollback check scheduled", + }; + writeDeploymentJob(restartScheduled); + await scheduleReleaseCutover( + restartScheduled, + expectedCommit, + rollbackRelease.manifest.commitShort, + signal + ); return true; } catch (error) { const failed: DeploymentJob = { diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts index c221180b8..ec00874d3 100644 --- a/backend/test/releaseDeployment.test.ts +++ b/backend/test/releaseDeployment.test.ts @@ -5,7 +5,6 @@ import { readdirSync, readFileSync, rmSync, - writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -15,82 +14,45 @@ import { afterEach, describe, expect, it } from "bun:test"; import { assertManagedDashboardUnitProperties, type DashboardReleaseCommandRunner, + MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT, + MANAGED_DASHBOARD_UNITS, managedDashboardUnitContract, runReleaseDeploymentCommand, stageDashboardRelease, } from "../src/releaseDeployment.ts"; import { managedReleasePath } from "../src/releaseManager.ts"; -import { writeReleaseManifest } from "../src/releaseManifest.ts"; +import { createReleaseFixture } from "./support/releaseFixture.ts"; const COMMIT_SHA = "a".repeat(40); const OTHER_COMMIT_SHA = "b".repeat(40); const temporaryRoots: string[] = []; +function managedUnitProperties(unitContents: string): string { + const lines = unitContents.split("\n"); + const environment = lines + .filter((line) => line.startsWith("Environment=")) + .map((line) => line.slice("Environment=".length)); + const execStart = lines.find((line) => line.startsWith("ExecStart=")); + const workingDirectory = lines.find((line) => line.startsWith("WorkingDirectory=")); + if (!execStart || !workingDirectory) { + throw new Error("Managed unit fixture is missing required service properties"); + } + return [`Environment=${environment.join(" ")}`, execStart, workingDirectory].join( + "\n" + ); +} + +async function concurrentBuildTimeout(): Promise { + await Bun.sleep(5000); + throw new Error("Concurrent release build barrier timed out"); +} + function temporaryRoot(label: string): string { const root = mkdtempSync(path.join(tmpdir(), `${label}-`)); temporaryRoots.push(root); return root; } -async function createBuiltRelease( - releaseRoot: string, - commitSha = COMMIT_SHA -): Promise { - mkdirSync(path.join(releaseRoot, "backend", "config"), { recursive: true }); - mkdirSync(path.join(releaseRoot, "backend", "dist"), { recursive: true }); - mkdirSync(path.join(releaseRoot, "dist", "assets"), { recursive: true }); - writeFileSync(path.join(releaseRoot, "package.json"), "{}\n"); - writeFileSync(path.join(releaseRoot, "bun.lock"), "root-lock\n"); - writeFileSync(path.join(releaseRoot, "backend", "package.json"), "{}\n"); - writeFileSync(path.join(releaseRoot, "backend", "bun.lock"), "backend-lock\n"); - writeFileSync( - path.join(releaseRoot, "backend", "config", "log-rotation.json"), - '{"jobs":[]}\n' - ); - writeFileSync(path.join(releaseRoot, "dist", "index.html"), "
ok
\n"); - writeFileSync( - path.join(releaseRoot, "dist", "assets", "app.js"), - "export const ok = true;\n" - ); - writeFileSync( - path.join(releaseRoot, "dist", "build-identity.json"), - `${JSON.stringify({ - bunVersion: Bun.version, - commitSha, - component: "frontend", - formatVersion: 1, - })}\n` - ); - writeFileSync( - path.join(releaseRoot, "backend", "dist", "build-identity.json"), - `${JSON.stringify({ - bunVersion: Bun.version, - commitSha, - component: "backend", - formatVersion: 1, - })}\n` - ); - for (const entrypoint of [ - "databasePreflight", - "releaseLifecycle", - "resetDashboardPassword", - "serverStart", - "workerStart", - ]) { - writeFileSync( - path.join(releaseRoot, "backend", "dist", `${entrypoint}.js`), - `export const commit = "${commitSha}";\n` - ); - } - writeFileSync(path.join(releaseRoot, "not-a-release-artifact.txt"), "ignore me\n"); - await writeReleaseManifest({ - builtAt: new Date("2026-07-26T01:30:00.000Z"), - commitSha, - commitTitle: `Release ${commitSha.slice(0, 8)}`, - releaseRoot, - }); -} - function stagingOptions() { const base = temporaryRoot("mira-release-deployment-test"); const sourceRoot = path.join(base, "source"); @@ -118,15 +80,30 @@ afterEach(() => { }); describe("immutable release deployment", () => { - it("keeps tracked production state outside source and release directories", () => { - for (const unitName of [ - "mira-dashboard.service", - "mira-dashboard-worker.service", - ]) { + it("keeps shipped managed units aligned with the production contract", () => { + const releasesRoot = "/home/ubuntu/projects/mira-dashboard-releases"; + const contract = { + databasePath: "/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db", + logRotationLockFile: + "/home/ubuntu/projects/mira-dashboard-state/log-rotation.lock", + openClawHome: "/home/ubuntu/projects/mira-dashboard-state/openclaw-client", + releaseRoot: `${releasesRoot}/current`, + releasesRoot, + }; + for (const unitName of Object.keys(MANAGED_DASHBOARD_UNITS) as Array< + keyof typeof MANAGED_DASHBOARD_UNITS + >) { const unit = readFileSync( path.resolve(import.meta.dirname, "../../systemd", unitName), "utf8" ); + expect(() => + assertManagedDashboardUnitProperties( + unitName, + managedUnitProperties(unit), + contract + ) + ).not.toThrow(); expect(unit).toContain( "WorkingDirectory=/home/ubuntu/projects/mira-dashboard-releases/current/backend" ); @@ -151,6 +128,7 @@ describe("immutable release deployment", () => { arguments_: readonly string[]; command: string; cwd: string; + releaseRoot: string | undefined; }> = []; const progress: string[] = []; const runner: DashboardReleaseCommandRunner = async ( @@ -158,12 +136,17 @@ describe("immutable release deployment", () => { arguments_, commandOptions ) => { - calls.push({ arguments_, command, cwd: commandOptions.cwd }); + calls.push({ + arguments_, + command, + cwd: commandOptions.cwd, + releaseRoot: commandOptions.environment.MIRA_DASHBOARD_RELEASE_ROOT, + }); if (command === "git" && arguments_[0] === "worktree") { if (arguments_[1] === "add") { const worktreePath = String(arguments_[3]); mkdirSync(worktreePath); - await createBuiltRelease(worktreePath); + await createReleaseFixture(worktreePath, COMMIT_SHA); } else if (arguments_[1] === "remove") { rmSync(String(arguments_[3]), { force: true, recursive: true }); } @@ -210,13 +193,16 @@ describe("immutable release deployment", () => { "Building and preflighting release", "Publishing verified immutable release", ]); + const buildReleaseRoots = new Set(calls.map(({ releaseRoot }) => releaseRoot)); + expect(buildReleaseRoots.size).toBe(1); + expect([...buildReleaseRoots][0]).toStartWith(`${options.worktreeRoot}/release-`); }); it("reuses an already verified immutable release without running commands", async () => { const options = stagingOptions(); const buildRoot = path.join(options.worktreeRoot, "prepared"); mkdirSync(buildRoot); - await createBuiltRelease(buildRoot); + await createReleaseFixture(buildRoot, COMMIT_SHA); const initialRunner: DashboardReleaseCommandRunner = async ( command, arguments_ @@ -225,7 +211,7 @@ describe("immutable release deployment", () => { if (arguments_[1] === "add") { const worktreePath = String(arguments_[3]); mkdirSync(worktreePath); - await createBuiltRelease(worktreePath); + await createReleaseFixture(worktreePath, COMMIT_SHA); } else { rmSync(String(arguments_[3]), { force: true, recursive: true }); } @@ -261,7 +247,7 @@ describe("immutable release deployment", () => { if (arguments_[1] === "add") { const worktreePath = String(arguments_[3]); mkdirSync(worktreePath); - await createBuiltRelease(worktreePath); + await createReleaseFixture(worktreePath, COMMIT_SHA); } else { rmSync(String(arguments_[3]), { force: true, recursive: true }); } @@ -275,7 +261,7 @@ describe("immutable release deployment", () => { if (buildsReady === 2) { releaseBuilds(); } - await buildsReleased; + await Promise.race([buildsReleased, concurrentBuildTimeout()]); } return { stderr: "", @@ -382,7 +368,7 @@ describe("immutable release deployment", () => { if (arguments_[1] === "add") { const worktreePath = String(arguments_[3]); mkdirSync(worktreePath); - await createBuiltRelease(worktreePath); + await createReleaseFixture(worktreePath, COMMIT_SHA); } else if (arguments_[1] === "remove") { throw new Error("registered worktree removal failed"); } @@ -412,7 +398,7 @@ describe("immutable release deployment", () => { if (arguments_[1] === "add") { const worktreePath = String(arguments_[3]); mkdirSync(worktreePath); - await createBuiltRelease(worktreePath, OTHER_COMMIT_SHA); + await createReleaseFixture(worktreePath, OTHER_COMMIT_SHA); } else { rmSync(String(arguments_[3]), { force: true, recursive: true }); } @@ -449,6 +435,9 @@ describe("immutable release deployment", () => { options.openClawHome ) ).toThrow("database path must be an absolute non-root path"); + expect(() => + managedDashboardUnitContract("/", options.databasePath, options.openClawHome) + ).toThrow("releases root must be an absolute non-root path"); const contract = managedDashboardUnitContract( options.releasesRoot, options.databasePath, @@ -457,7 +446,7 @@ describe("immutable release deployment", () => { const properties = [ `WorkingDirectory=${contract.releaseRoot}/backend`, `Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${contract.databasePath} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${contract.releaseRoot} MIRA_DASHBOARD_RELEASES_ROOT=${contract.releasesRoot}`, - "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run -- bun dist/serverStart.js ; }", + `ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run --preserve-env=${MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT.join(",")} -- bun dist/serverStart.js ; }`, ].join("\n"); expect(() => assertManagedDashboardUnitProperties( @@ -466,6 +455,16 @@ describe("immutable release deployment", () => { contract ) ).not.toThrow(); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + ` --preserve-env=${MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT.join(",")}`, + "" + ), + contract + ) + ).toThrow("must preserve managed release environment"); expect(() => assertManagedDashboardUnitProperties( "mira-dashboard-worker.service", @@ -522,6 +521,9 @@ describe("immutable release deployment", () => { await expect( runReleaseDeploymentCommand(["unknown"], options.releasesRoot) ).rejects.toThrow("Usage"); + await expect( + runReleaseDeploymentCommand(["stage"], options.releasesRoot) + ).rejects.toThrow("stage requires a commit SHA"); await expect( runReleaseDeploymentCommand(["prune", "1"], options.releasesRoot) ).rejects.toThrow("retention must be between 2 and 20"); @@ -530,6 +532,6 @@ describe("immutable release deployment", () => { ).rejects.toThrow("unexpected arguments"); expect( await runReleaseDeploymentCommand(["prune"], options.releasesRoot) - ).toEqual({ removed: [], retained: [] }); + ).toEqual({ removed: [], retained: [], warnings: [] }); }); }); diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index 49f658f1d..16ed9fdce 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -115,7 +115,8 @@ async function createManagedRelease( releasesRoot: string, directoryCommit: string, manifestCommit = directoryCommit, - bunVersion = Bun.version + bunVersion = Bun.version, + builtAt = new Date("2026-07-25T17:00:00.000Z") ): Promise { await ensureDashboardReleaseLayout(releasesRoot); const releasePath = managedReleasePath(releasesRoot, directoryCommit); @@ -169,7 +170,7 @@ async function createManagedRelease( ); } await writeReleaseManifest({ - builtAt: new Date("2026-07-25T17:00:00.000Z"), + builtAt, bunVersion, commitSha: manifestCommit, commitTitle: `Release ${manifestCommit.slice(0, 8)}`, @@ -388,6 +389,7 @@ describe("Dashboard immutable release manager", () => { await expect(runReleaseLifecycleCommand(["prune"], root)).resolves.toEqual({ removed: [], retained: [SECOND_COMMIT, FIRST_COMMIT], + warnings: [], }); await expect(runReleaseLifecycleCommand(["prune", "1"], root)).rejects.toThrow( "retention must be between 2 and 20" @@ -829,10 +831,34 @@ describe("Dashboard immutable release manager", () => { it("prunes old releases while preserving current and previous", async () => { const root = temporaryReleasesRoot(); - await createManagedRelease(root, FIRST_COMMIT); - await createManagedRelease(root, SECOND_COMMIT); - await createManagedRelease(root, THIRD_COMMIT); - await createManagedRelease(root, FOURTH_COMMIT); + await createManagedRelease( + root, + FIRST_COMMIT, + FIRST_COMMIT, + Bun.version, + new Date("2026-07-25T17:00:00.000Z") + ); + await createManagedRelease( + root, + SECOND_COMMIT, + SECOND_COMMIT, + Bun.version, + new Date("2026-07-25T17:01:00.000Z") + ); + await createManagedRelease( + root, + THIRD_COMMIT, + THIRD_COMMIT, + Bun.version, + new Date("2026-07-25T17:02:00.000Z") + ); + await createManagedRelease( + root, + FOURTH_COMMIT, + FOURTH_COMMIT, + Bun.version, + new Date("2026-07-25T17:03:00.000Z") + ); await activateDashboardRelease(SECOND_COMMIT, root, SCHEMA_6_OPTIONS); await activateDashboardRelease(THIRD_COMMIT, root, SCHEMA_6_OPTIONS); const interruptedRetirementPath = path.join( @@ -842,17 +868,23 @@ describe("Dashboard immutable release manager", () => { ); mkdirSync(interruptedRetirementPath); writeFileSync(path.join(interruptedRetirementPath, "stale"), "stale\n"); + const unverifiableCommit = "e".repeat(40); + const unverifiablePath = managedReleasePath(root, unverifiableCommit); + mkdirSync(unverifiablePath); + writeFileSync(path.join(unverifiablePath, "invalid"), "invalid\n"); const result = await pruneDashboardReleases(3, root); expect(result).toEqual({ removed: [FIRST_COMMIT], retained: [FOURTH_COMMIT, THIRD_COMMIT, SECOND_COMMIT], + warnings: [`Skipped unverifiable release ${unverifiableCommit}`], }); expect(existsSync(managedReleasePath(root, FIRST_COMMIT))).toBe(false); expect(existsSync(managedReleasePath(root, SECOND_COMMIT))).toBe(true); expect(existsSync(managedReleasePath(root, THIRD_COMMIT))).toBe(true); expect(existsSync(managedReleasePath(root, FOURTH_COMMIT))).toBe(true); + expect(existsSync(unverifiablePath)).toBe(true); expect(existsSync(interruptedRetirementPath)).toBe(false); const state = await readDashboardReleaseState(root); expect(state.current?.commitSha).toBe(THIRD_COMMIT); diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index e1dc92aba..b989a0244 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -24,7 +24,7 @@ import { ensureDashboardReleaseLayout, managedReleasePath, } from "../src/releaseManager.ts"; -import { writeReleaseManifest } from "../src/releaseManifest.ts"; +import { createReleaseFixture } from "./support/releaseFixture.ts"; const cleanupCallbacks: Array<() => Promise | void> = []; @@ -47,66 +47,6 @@ function createTemporaryRoot(prefix: string): string { return root; } -async function createDeploymentReleaseFixture( - releaseRoot: string, - commitSha: string, - commitTitle: string -): Promise { - mkdirSync(path.join(releaseRoot, "backend", "config"), { recursive: true }); - mkdirSync(path.join(releaseRoot, "backend", "dist"), { recursive: true }); - mkdirSync(path.join(releaseRoot, "dist", "assets"), { recursive: true }); - writeFileSync(path.join(releaseRoot, "package.json"), "{}\n"); - writeFileSync(path.join(releaseRoot, "bun.lock"), "root-lock\n"); - writeFileSync(path.join(releaseRoot, "backend", "package.json"), "{}\n"); - writeFileSync(path.join(releaseRoot, "backend", "bun.lock"), "backend-lock\n"); - writeFileSync( - path.join(releaseRoot, "backend", "config", "log-rotation.json"), - '{"jobs":[]}\n' - ); - writeFileSync(path.join(releaseRoot, "dist", "index.html"), "
ready
\n"); - writeFileSync( - path.join(releaseRoot, "dist", "assets", "app.js"), - `export const commit = "${commitSha}";\n` - ); - writeFileSync( - path.join(releaseRoot, "not-a-release-artifact.txt"), - "must not publish\n" - ); - for (const component of ["frontend", "backend"] as const) { - const componentRoot = - component === "frontend" - ? path.join(releaseRoot, "dist") - : path.join(releaseRoot, "backend", "dist"); - writeFileSync( - path.join(componentRoot, "build-identity.json"), - `${JSON.stringify({ - bunVersion: Bun.version, - commitSha, - component, - formatVersion: 1, - })}\n` - ); - } - for (const entrypoint of [ - "databasePreflight", - "releaseLifecycle", - "resetDashboardPassword", - "serverStart", - "workerStart", - ]) { - writeFileSync( - path.join(releaseRoot, "backend", "dist", `${entrypoint}.js`), - `export const commit = "${commitSha}";\n` - ); - } - await writeReleaseManifest({ - builtAt: new Date("2026-07-26T02:00:00.000Z"), - commitSha, - commitTitle, - releaseRoot, - }); -} - function readableUtf8Stream(value: string): ReadableStream { return new ReadableStream({ start(controller) { @@ -1876,13 +1816,14 @@ describe("backend service behavior", () => { } }); - it("publishes and activates an immutable release before detached cutover", async () => { + it("publishes an immutable release and hands activation to detached cutover", async () => { rememberEnvironment("PATH"); rememberEnvironment("MIRA_DASHBOARD_ROOT"); rememberEnvironment("MIRA_DASHBOARD_WORKTREE_ROOT"); rememberEnvironment("MIRA_DASHBOARD_RELEASES_ROOT"); rememberEnvironment("MIRA_DASHBOARD_OPENCLAW_HOME"); rememberEnvironment("MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE"); + rememberEnvironment("PORT"); const fakeRoot = createTemporaryRoot("mira-pr-deploy-root-"); const fakeBin = createTemporaryRoot("mira-pr-deploy-bin-"); const worktreeRoot = path.join(fakeRoot, "worktrees"); @@ -1900,18 +1841,14 @@ describe("backend service behavior", () => { mkdirSync(worktreeRoot); mkdirSync(path.dirname(openClawHome), { recursive: true }); mkdirSync(candidateTemplate); - await createDeploymentReleaseFixture( - candidateTemplate, - candidateCommit, - "Deployable dashboard commit" - ); + await createReleaseFixture(candidateTemplate, candidateCommit, { + commitTitle: "Deployable dashboard commit", + }); await ensureDashboardReleaseLayout(releasesRoot); const oldReleasePath = managedReleasePath(releasesRoot, oldCommit); - await createDeploymentReleaseFixture( - oldReleasePath, - oldCommit, - "Previous dashboard commit" - ); + await createReleaseFixture(oldReleasePath, oldCommit, { + commitTitle: "Previous dashboard commit", + }); symlinkSync(`releases/${oldCommit}`, path.join(releasesRoot, "current"), "dir"); writeFileSync( path.join(fakeBin, "git"), @@ -1972,7 +1909,7 @@ else fi printf '%s\n' \ 'Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${getMiraDatabasePath()} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${releasesRoot}/current MIRA_DASHBOARD_RELEASES_ROOT=${releasesRoot}' \ - "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run -- bun $entrypoint ; }" \ + "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- bun $entrypoint ; }" \ 'WorkingDirectory=${releasesRoot}/current/backend' ` ); @@ -1980,6 +1917,7 @@ printf '%s\n' \ path.join(fakeBin, "systemd-run"), String.raw`#!/usr/bin/env bash set -euo pipefail +/bin/bash -n <<<"$6" printf '%s\n' "$*" >> ${JSON.stringify(systemdLog)} printf 'scheduled\n' ` @@ -1994,6 +1932,7 @@ printf 'scheduled\n' process.env.MIRA_DASHBOARD_RELEASES_ROOT = releasesRoot; process.env.MIRA_DASHBOARD_OPENCLAW_HOME = openClawHome; process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE = logRotationLockFile; + process.env.PORT = "4310"; const { registerPullRequestExecutionActions, startDeployLatest } = await import("../src/services/pullRequests.ts"); @@ -2056,7 +1995,7 @@ printf 'scheduled\n' expect(row).toEqual({ commit_sha: candidateCommit.slice(0, 8), commit_title: "Deployable dashboard commit", - note: "Immutable release published. Atomic restart and rollback check scheduled", + note: "Immutable release published. Detached activation and rollback check scheduled", status: "restart-scheduled", }); await expect(Bun.file(gitLog).text()).resolves.toContain( @@ -2075,20 +2014,22 @@ printf 'scheduled\n' `mira-dashboard-deploy-${job.id}` ); const restartCommand = await Bun.file(systemdLog).text(); - expect(restartCommand).toContain("/api/health/ready"); + expect(restartCommand).toContain("http://127.0.0.1:4310/api/health/ready"); expect(restartCommand).toContain("--connect-timeout 2 --max-time 5"); expect(restartCommand).toContain("for attempt in {1..30}"); expect(restartCommand).toContain(".checks.release.backendCommit"); expect(restartCommand).toContain("releaseLifecycle.js"); + expect(restartCommand).toContain(`activate '${candidateCommit}'`); + expect(restartCommand.indexOf(`activate '${candidateCommit}'`)).toBeLessThan( + restartCommand.indexOf("if restart_services") + ); expect(restartCommand).toContain("rollback"); expect(restartCommand).toContain("prune 3"); expect(restartCommand).not.toContain("/api/job-executions"); expect(readlinkSync(path.join(releasesRoot, "current"))).toBe( - `releases/${candidateCommit}` - ); - expect(readlinkSync(path.join(releasesRoot, "previous"))).toBe( `releases/${oldCommit}` ); + expect(existsSync(path.join(releasesRoot, "previous"))).toBe(false); const publishedReleasePath = managedReleasePath( releasesRoot, candidateCommit @@ -5064,6 +5005,13 @@ fi }); it("normalizes elevated log rotation command output and failures", async () => { + rememberEnvironment("MIRA_DASHBOARD_DB_PATH"); + rememberEnvironment("MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE"); + const stateRoot = createTemporaryRoot("mira-elevated-log-rotation-"); + const configuredLockFile = path.join(stateRoot, "log-rotation.lock"); + const configuredDatabasePath = path.join(stateRoot, "mira-dashboard.db"); + process.env.MIRA_DASHBOARD_DB_PATH = configuredDatabasePath; + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE = configuredLockFile; const { runElevatedLogRotationService } = await import("../src/services/logRotation.ts"); const runProcessSpy = jest @@ -5094,6 +5042,16 @@ fi result: { checkedFiles: 2, isOk: true }, stderr: "sudo notice", }); + const [sudoCommand, sudoArguments, sudoOptions] = + runProcessSpy.mock.calls[0] ?? []; + expect(sudoCommand).toBe("sudo"); + expect(sudoArguments).toContain( + "--preserve-env=LANG,NODE_ENV,TZ,MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE" + ); + expect(sudoOptions?.env).toMatchObject({ + MIRA_DASHBOARD_DB_PATH: configuredDatabasePath, + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE: configuredLockFile, + }); await expect( runElevatedLogRotationService({ isDryRun: false }) ).resolves.toMatchObject({ @@ -5121,6 +5079,28 @@ fi }); }); + it("uses the configured lock for non-elevated log rotation", async () => { + rememberEnvironment("MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE"); + const stateRoot = createTemporaryRoot("mira-log-rotation-lock-"); + const configuredLockFile = path.join(stateRoot, "custom.lock"); + const configPath = path.join(stateRoot, "log-rotation.json"); + writeFileSync(configPath, '{"groups":[],"version":1}\n'); + writeFileSync(configuredLockFile, `${process.pid}\n`); + process.env.MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE = configuredLockFile; + const { runLogRotationService } = await import("../src/services/logRotation.ts"); + + const summary = await runLogRotationService({ + config: configPath, + isDryRun: false, + }); + + expect(summary).toMatchObject({ + errors: [{ message: "Log rotation is already running" }], + isOk: false, + }); + expect(readFileSync(configuredLockFile, "utf8")).toBe(`${process.pid}\n`); + }); + it("records scheduled log-rotation failures in cache state", async () => { const { registerLogRotationScheduledJobs } = await import("../src/services/logRotation.ts"); diff --git a/backend/test/support/releaseFixture.ts b/backend/test/support/releaseFixture.ts new file mode 100644 index 000000000..a1704b59c --- /dev/null +++ b/backend/test/support/releaseFixture.ts @@ -0,0 +1,70 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +import { writeReleaseManifest } from "../../src/releaseManifest.ts"; + +interface ReleaseFixtureOptions { + builtAt?: Date; + commitTitle?: string; +} + +/** Creates the complete artifact set shared by immutable release tests. */ +export async function createReleaseFixture( + releaseRoot: string, + commitSha: string, + options: ReleaseFixtureOptions = {} +): Promise { + mkdirSync(path.join(releaseRoot, "backend", "config"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "backend", "dist"), { recursive: true }); + mkdirSync(path.join(releaseRoot, "dist", "assets"), { recursive: true }); + writeFileSync(path.join(releaseRoot, "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "bun.lock"), "root-lock\n"); + writeFileSync(path.join(releaseRoot, "backend", "package.json"), "{}\n"); + writeFileSync(path.join(releaseRoot, "backend", "bun.lock"), "backend-lock\n"); + writeFileSync( + path.join(releaseRoot, "backend", "config", "log-rotation.json"), + '{"jobs":[]}\n' + ); + writeFileSync(path.join(releaseRoot, "dist", "index.html"), "
ready
\n"); + writeFileSync( + path.join(releaseRoot, "dist", "assets", "app.js"), + `export const commit = "${commitSha}";\n` + ); + writeFileSync( + path.join(releaseRoot, "not-a-release-artifact.txt"), + "must not publish\n" + ); + for (const component of ["frontend", "backend"] as const) { + const componentRoot = + component === "frontend" + ? path.join(releaseRoot, "dist") + : path.join(releaseRoot, "backend", "dist"); + writeFileSync( + path.join(componentRoot, "build-identity.json"), + `${JSON.stringify({ + bunVersion: Bun.version, + commitSha, + component, + formatVersion: 1, + })}\n` + ); + } + for (const entrypoint of [ + "databasePreflight", + "releaseLifecycle", + "resetDashboardPassword", + "serverStart", + "workerStart", + ]) { + writeFileSync( + path.join(releaseRoot, "backend", "dist", `${entrypoint}.js`), + `export const commit = "${commitSha}";\n` + ); + } + await writeReleaseManifest({ + builtAt: options.builtAt ?? new Date("2026-07-26T02:00:00.000Z"), + commitSha, + commitTitle: options.commitTitle ?? `Release ${commitSha.slice(0, 8)}`, + releaseRoot, + }); +} diff --git a/backend/test/utilityBehavior.test.ts b/backend/test/utilityBehavior.test.ts index fa8090428..94a08579c 100644 --- a/backend/test/utilityBehavior.test.ts +++ b/backend/test/utilityBehavior.test.ts @@ -19,6 +19,7 @@ import { loadOrCreateDeviceIdentity } from "../src/lib/openclawGatewayClient.ts" import { pipeProcessOutput, runProcess } from "../src/lib/processes.ts"; import { prepareSafeWriteTargetWithinRoot, + resolveAbsoluteNonRootPath, safePathWithinRoot, sanitizeFilename, } from "../src/lib/safePath.ts"; @@ -28,6 +29,7 @@ import { nonEmptyEnvironmentFallback, nullableString, objectFallback, + resolveDashboardPort, stringFallback, } from "../src/lib/values.ts"; import { resetRequestPolicyForTests, withRequestPolicy } from "../src/requestPolicy.ts"; @@ -405,6 +407,10 @@ describe("backend service utilities", () => { ).toEqual({}); expect(arrayFallback(["a"])).toEqual(["a"]); expect(arrayFallback("not-array", ["fallback"])).toEqual(["fallback"]); + expect(resolveDashboardPort(" 4310 ")).toBe(4310); + expect(resolveDashboardPort("0")).toBe(3100); + expect(resolveDashboardPort("65536")).toBe(3100); + expect(resolveDashboardPort("not-a-port")).toBe(3100); } finally { if (originalValue === undefined) { delete process.env.MIRA_TEST_OPTIONAL_VALUE; @@ -442,6 +448,12 @@ describe("backend service utilities", () => { expect(safePathWithinRoot("../escape.txt", root)).toBeUndefined(); expect(safePathWithinRoot("outside-link/escape.txt", root)).toBeUndefined(); expect(safePathWithinRoot("bad\0name", root)).toBeUndefined(); + expect(resolveAbsoluteNonRootPath(` ${root} `, "Test path")).toBe(root); + for (const invalidPath of ["", "relative", "/", "bad\0path"]) { + expect(() => + resolveAbsoluteNonRootPath(invalidPath, "Test path") + ).toThrow("Test path must be an absolute non-root path"); + } const writeTarget = path.join(root, "nested", "report.txt"); expect( diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 51567bb0a..b00df4e29 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -50,6 +50,9 @@ MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/curren MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases ``` +Their Doppler command selectively preserves these five values, so production +secrets cannot replace unit-owned state or release paths. + The OpenClaw home preserves the signed Gateway device identity across releases. Secrets remain in Doppler `rajohan/prd`; tracked unit files contain no secret values. @@ -67,8 +70,9 @@ The Dashboard worker owns the deployment: every checksummed artifact. 6. Copy only declared artifacts to a hidden directory and atomically publish it as `releases/`. -7. Atomically switch `current`; retain the old release as `previous`. -8. Restart web and worker. +7. Start a detached cutover guardian, which atomically switches `current` and + retains the old release as `previous`. +8. Restart web and worker from inside that guardian. 9. Require `/api/health/ready` to report the exact expected frontend/backend commit and a fresh worker heartbeat from that commit. 10. On failure, switch back to `previous`, restart both units, verify the old @@ -82,9 +86,11 @@ the running release. ## One-Time Managed Cutover -Run this once after the atomic-executor change has been merged and built by the -old in-place deployment. The Jobs queue must be idle. PR #333 is the known-good -format-2 bootstrap release. +Run this once immediately after the atomic-executor change has been merged into +the control checkout. Do **not** use the old in-place deploy executor for this +change. The old services stay online while the merged control-checkout scripts +stage the managed releases. The Jobs queue must be idle. PR #333 is the +known-good format-2 bootstrap release. ### 1. Stage both releases while the old services remain online @@ -109,6 +115,18 @@ OPENCLAW_CLIENT_HOME="$STATE_ROOT/openclaw-client" LOG_ROTATION_LOCK="$STATE_ROOT/log-rotation.lock" BOOTSTRAP_SHA=4aca68e0cffed68c42a630c4221e11e725ab294b CANDIDATE_SHA="$(git rev-parse HEAD)" +DASHBOARD_PORT="$( + /usr/local/bin/doppler run --config prd --project rajohan -- \ + /bin/sh -c 'printf "%s" "${PORT:-3100}"' +)" +DASHBOARD_PORT="$( + printf '%s' "$DASHBOARD_PORT" | + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +)" +if ! [[ "$DASHBOARD_PORT" =~ ^[0-9]+$ ]] || + (( DASHBOARD_PORT < 1 || DASHBOARD_PORT > 65535 )); then + DASHBOARD_PORT=3100 +fi env \ MIRA_DASHBOARD_DB_PATH="$OLD_DATABASE_PATH" \ @@ -180,7 +198,7 @@ ready_for_commit() { local response for attempt in {1..30}; do response="$(curl --fail --silent --show-error \ - http://127.0.0.1:3100/api/health/ready || true)" + "http://127.0.0.1:${DASHBOARD_PORT}/api/health/ready" || true)" if jq --exit-status --arg expected "$expected" \ '.status == "isReady" and .checks.release.backendCommit == $expected @@ -193,24 +211,30 @@ ready_for_commit() { return 1 } -ready_for_commit "$BOOTSTRAP_SHA" -``` - -If bootstrap readiness fails, stop both units, restore the saved unit files, -move the state directory back, reload systemd, and restart the old deployment: +recover_legacy_deployment() { + systemctl --user stop \ + mira-dashboard-worker.service mira-dashboard.service || return 1 + install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard.service" \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service || return 1 + install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard-worker.service" \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service || return 1 + mv --no-target-directory "$STATE_ROOT" "$OLD_STATE_ROOT" || return 1 + systemctl --user daemon-reload || return 1 + systemctl --user restart \ + mira-dashboard-worker.service mira-dashboard.service +} -```bash -systemctl --user stop mira-dashboard-worker.service mira-dashboard.service -install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard.service" \ - /home/ubuntu/.config/systemd/user/mira-dashboard.service -install -m 0644 "$CUTOVER_UNIT_BACKUP/mira-dashboard-worker.service" \ - /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service -mv --no-target-directory "$STATE_ROOT" "$OLD_STATE_ROOT" -systemctl --user daemon-reload -systemctl --user restart mira-dashboard-worker.service mira-dashboard.service +if ! ready_for_commit "$BOOTSTRAP_SHA"; then + echo "Bootstrap readiness failed; restoring the legacy deployment" >&2 + if ! recover_legacy_deployment; then + echo "Legacy deployment recovery also failed; manual recovery is required" >&2 + fi + exit 1 +fi ``` -Investigate before retrying. Do not continue to candidate activation. +Any failed bootstrap check now runs recovery and exits nonzero. Investigate +before retrying; never continue to candidate activation after this branch. ### 4. Activate and verify the candidate @@ -224,21 +248,31 @@ env \ systemctl --user restart mira-dashboard-worker.service mira-dashboard.service ``` -Require `ready_for_commit "$CANDIDATE_SHA"`. If it fails: - ```bash -env \ - MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ - MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ - NODE_ENV=production \ - bun "$RELEASES_ROOT/releases/$CANDIDATE_SHA/backend/dist/releaseLifecycle.js" \ - rollback -systemctl --user restart mira-dashboard-worker.service mira-dashboard.service -ready_for_commit "$BOOTSTRAP_SHA" +if ! ready_for_commit "$CANDIDATE_SHA"; then + echo "Candidate readiness failed; rolling back to bootstrap" >&2 + if ! env \ + MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ + MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ + NODE_ENV=production \ + bun "$RELEASES_ROOT/releases/$CANDIDATE_SHA/backend/dist/releaseLifecycle.js" \ + rollback; then + echo "Candidate rollback failed; manual recovery is required" >&2 + exit 1 + fi + systemctl --user restart \ + mira-dashboard-worker.service mira-dashboard.service + if ! ready_for_commit "$BOOTSTRAP_SHA"; then + echo "Bootstrap was restored but did not become ready" >&2 + exit 1 + fi + echo "Bootstrap restored; investigate candidate failure before retrying" >&2 + exit 1 +fi ``` -Do not complete the cutover unless the restored bootstrap is ready. After a -successful candidate check: +The failure branch always exits nonzero, including after a verified rollback. +After a successful candidate check: ```bash env \ @@ -261,7 +295,7 @@ systemctl --user status mira-dashboard-worker.service --no-pager journalctl --user -u mira-dashboard.service -n 120 --no-pager journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager curl --fail --silent --show-error \ - http://127.0.0.1:3100/api/health/ready | jq + "http://127.0.0.1:${DASHBOARD_PORT:-3100}/api/health/ready" | jq ``` Direct loopback is transport, not authentication. Tokenless protected API calls @@ -285,7 +319,7 @@ env MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" rollback systemctl --user restart mira-dashboard-worker.service mira-dashboard.service curl --fail --silent --show-error \ - http://127.0.0.1:3100/api/health/ready | jq + "http://127.0.0.1:${DASHBOARD_PORT:-3100}/api/health/ready" | jq ``` Git reset and rebuilding in the control checkout are not production rollback diff --git a/systemd/mira-dashboard-worker.service b/systemd/mira-dashboard-worker.service index 54b44efa8..b83f59cd7 100644 --- a/systemd/mira-dashboard-worker.service +++ b/systemd/mira-dashboard-worker.service @@ -16,7 +16,7 @@ Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-das Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases -ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/workerStart.js +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/workerStart.js Restart=on-failure RestartSec=5 KillMode=control-group diff --git a/systemd/mira-dashboard.service b/systemd/mira-dashboard.service index b7dd20f88..a8dfadf04 100644 --- a/systemd/mira-dashboard.service +++ b/systemd/mira-dashboard.service @@ -16,7 +16,7 @@ Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-das Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases -ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/serverStart.js +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/serverStart.js Restart=on-failure RestartSec=5 KillMode=control-group From 2bb231386e0e75ccb21bbe51178cae434485dd8e Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 06:37:33 +0200 Subject: [PATCH 03/11] fix(ops): close managed cutover recovery gaps --- backend/package.json | 2 +- backend/src/server.ts | 3 +++ backend/test/bunNativeServerBehavior.test.ts | 4 ++++ backend/test/releaseDeployment.test.ts | 13 +++++++++++++ docs/operations/runbooks.md | 10 +++++++--- docs/setup/production-deploy.md | 1 + 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/backend/package.json b/backend/package.json index a9503522e..e2b8014eb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -9,7 +9,7 @@ "build": "bun node_modules/@typescript/native/bin/tsc --noEmit && bun scripts/build.ts", "db:preflight": "bun dist/databasePreflight.js", "deploy:prepare": "bun run build && bun run db:preflight", - "auth:reset-password": "NODE_ENV=production doppler run --config prd --project rajohan -- bun dist/resetDashboardPassword.js", + "auth:reset-password": "MIRA_DASHBOARD_DB_PATH=${MIRA_DASHBOARD_DB_PATH:-/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db} NODE_ENV=production doppler run --config prd --project rajohan --preserve-env=MIRA_DASHBOARD_DB_PATH -- bun dist/resetDashboardPassword.js", "start": "NODE_ENV=production doppler run --config prd --project rajohan -- bun dist/serverStart.js", "start:worker": "NODE_ENV=production doppler run --config prd --project rajohan -- bun dist/workerStart.js", "dev": "MIRA_DASHBOARD_DISABLE_SCHEDULER=1 doppler run --config prd --project rajohan -- bun --watch src/serverStart.ts", diff --git a/backend/src/server.ts b/backend/src/server.ts index 0e4aa6842..d27bb15f0 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -231,6 +231,9 @@ async function staticResponse(pathname: string): Promise { if (decodedPathname === "/api" || decodedPathname.startsWith("/api/")) { return Response.json({ error: "Not found" }, { status: 404 }); } + if (decodedPathname === "/health") { + return new Response("Not found", { status: 404 }); + } const frontendPath = resolveFrontendPath(); const indexPath = path.join(frontendPath, "index.html"); diff --git a/backend/test/bunNativeServerBehavior.test.ts b/backend/test/bunNativeServerBehavior.test.ts index 67544748e..079ab5382 100644 --- a/backend/test/bunNativeServerBehavior.test.ts +++ b/backend/test/bunNativeServerBehavior.test.ts @@ -335,6 +335,10 @@ describe("Bun-native dashboard backend", () => { }); expect(liveHead).toEqual({ body: undefined, status: 200 }); + const retiredHealth = await fetch(`${state.baseUrl}/health`); + expect(retiredHealth.status).toBe(404); + expect(await retiredHealth.text()).toBe("Not found"); + const bootstrap = await api<{ hasGatewayToken: boolean; isBootstrapRequired: boolean; diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts index ec00874d3..514439bc8 100644 --- a/backend/test/releaseDeployment.test.ts +++ b/backend/test/releaseDeployment.test.ts @@ -122,6 +122,19 @@ describe("immutable release deployment", () => { } }); + it("keeps host-local password reset on the stable production database", () => { + const backendPackage = JSON.parse( + readFileSync(path.resolve(import.meta.dirname, "../package.json"), "utf8") + ) as { scripts?: Record }; + const resetCommand = backendPackage.scripts?.["auth:reset-password"]; + expect(resetCommand).toContain( + "MIRA_DASHBOARD_DB_PATH=${MIRA_DASHBOARD_DB_PATH:-/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db}" + ); + expect(resetCommand).toContain( + "--preserve-env=MIRA_DASHBOARD_DB_PATH -- bun dist/resetDashboardPassword.js" + ); + }); + it("builds in an isolated worktree and atomically publishes only artifacts", async () => { const options = stagingOptions(); const calls: Array<{ diff --git a/docs/operations/runbooks.md b/docs/operations/runbooks.md index 71aefc0c9..59b86658e 100644 --- a/docs/operations/runbooks.md +++ b/docs/operations/runbooks.md @@ -85,11 +85,14 @@ host-local interactive command from an SSH/console TTY: ```bash cd /home/ubuntu/projects/mira-dashboard-releases/current/backend -bun run auth:reset-password -- --username +MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ + bun run auth:reset-password -- --username ``` The single standalone `--` ends Bun script options; `--username` is passed to -the reset program. The program reads the new password twice with terminal echo +the reset program. The package script preserves the explicit stable database +path through Doppler and defaults to the same production path when it is not +already set. The program reads the new password twice with terminal echo disabled, preserves MFA, revokes every session and pending ceremony, clears authentication cooldowns, and appends an audit event. It never accepts password material through command arguments or environment variables. @@ -98,7 +101,8 @@ Only when all registered second factors are also lost, run the deliberate break-glass variant: ```bash -bun run auth:reset-password -- --username --reset-mfa +MIRA_DASHBOARD_DB_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db \ + bun run auth:reset-password -- --username --reset-mfa ``` `--reset-mfa` deletes registered WebAuthn credentials, encrypted TOTP factors, diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index b00df4e29..2152acd6b 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -198,6 +198,7 @@ ready_for_commit() { local response for attempt in {1..30}; do response="$(curl --fail --silent --show-error \ + --connect-timeout 2 --max-time 5 \ "http://127.0.0.1:${DASHBOARD_PORT}/api/health/ready" || true)" if jq --exit-status --arg expected "$expected" \ '.status == "isReady" From 337d3568ca24296bea475c21af1e9b581b0d8c4d Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 07:17:25 +0200 Subject: [PATCH 04/11] fix(deploy): persist cutover safety across worker restart --- backend/src/services/logRotation.ts | 12 ++++-------- backend/src/services/pullRequests.ts | 16 +++++++++++++--- backend/src/services/scheduledJobs.ts | 19 +++++++++++++++++++ backend/test/serviceBehavior.test.ts | 19 ++++++++++++++++--- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/backend/src/services/logRotation.ts b/backend/src/services/logRotation.ts index 3d2d47e1d..090a363ea 100644 --- a/backend/src/services/logRotation.ts +++ b/backend/src/services/logRotation.ts @@ -69,7 +69,7 @@ const ELEVATED_LOG_ROTATION_MAX_BUFFER = 16 * 1024 * 1024; const LOG_ROTATION_JOB_ID = "ops.log-rotation"; const LOG_ROTATION_FAILURE_OUTPUT_MAX_CHARS = 100_000; const BUN_EXECUTABLE = process.env.BUN_BINARY || "bun"; -const ELEVATED_LOG_ROTATION_PRESERVED_ENVIRONMENT = [ +const ELEVATED_LOG_ROTATION_FORWARDED_ENVIRONMENT = [ "LANG", "NODE_ENV", "TZ", @@ -2039,7 +2039,7 @@ function buildElevatedLogRotationCliArguments( ].join("\n"); return [ "-n", - `--preserve-env=${ELEVATED_LOG_ROTATION_PRESERVED_ENVIRONMENT.join(",")}`, + `--preserve-env=${ELEVATED_LOG_ROTATION_FORWARDED_ENVIRONMENT.join(",")}`, resolveBunExecutable(), "--input-type=module", "--eval", @@ -2054,12 +2054,8 @@ function elevatedLogRotationEnvironment(): NodeJS.ProcessEnv { const allowed = [ "PATH", "HOME", - "LANG", - "NODE_ENV", - "TZ", - "MIRA_DASHBOARD_DB_PATH", - "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", - ]; + ...ELEVATED_LOG_ROTATION_FORWARDED_ENVIRONMENT, + ] as const; const environment: NodeJS.ProcessEnv = {}; // Keep sudo environment preservation narrow: runtime lookup, locale, and state paths. for (const key of allowed) { diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 17de20093..24baab331 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -8,7 +8,7 @@ import { runProcess, spawnProcess, } from "../lib/processes.ts"; -import { nonEmptyEnvironmentFallback, resolveDashboardPort } from "../lib/values.ts"; +import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; import { assertManagedDashboardUnitProperties, MANAGED_DASHBOARD_UNITS, @@ -1511,7 +1511,6 @@ async function scheduleReleaseCutover( "dist", "releaseLifecycle.js" ); - const readinessUrl = `http://127.0.0.1:${resolveDashboardPort()}/api/health/ready`; const lifecycleEnvironment = [ `MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)}`, `MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())}`, @@ -1550,10 +1549,21 @@ async function scheduleReleaseCutover( const script = [ "sleep 2", + "resolve_dashboard_port() {", + ' dashboard_port=$(/usr/local/bin/doppler run --config prd --project rajohan -- /bin/sh -c \'printf "%s" "${PORT:-3100}"\' 2>/dev/null || true)', + ' case "$dashboard_port" in', + ' ""|*[!0-9]*) dashboard_port=3100 ;;', + " esac", + " if ((10#$dashboard_port < 1 || 10#$dashboard_port > 65535)); then", + " dashboard_port=3100", + " fi", + ' printf "%s" "$dashboard_port"', + "}", "ready_for_commit() {", ' expected_commit="$1"', + ' dashboard_port="$(resolve_dashboard_port)"', " for attempt in {1..30}; do", - ` response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 ${shellQuote(readinessUrl)} 2>/dev/null || true)`, + ' response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 "http://127.0.0.1:${dashboard_port}/api/health/ready" 2>/dev/null || true)', ' if printf "%s" "$response" | /usr/bin/jq --exit-status --arg expected "$expected_commit" \'.status == "isReady" and .checks.release.ready == true and .checks.release.backendCommit == $expected and .checks.release.frontendCommit == $expected and .checks.worker.ready == true\' >/dev/null 2>&1; then', " return 0", " fi", diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index fe0c83a5a..98261e30d 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -1251,6 +1251,19 @@ function resetExecutorClaimPause(): void { scheduledJobRuntimeState.isExecutorClaimingPaused = false; } +function hasPendingDeploymentCutover(): boolean { + return Boolean( + database + .query( + `SELECT 1 + FROM deployment_jobs + WHERE status = 'restart-scheduled' + LIMIT 1` + ) + .get() + ); +} + function executorTick(): void { if ( !scheduledJobRuntimeState.executor || @@ -1262,6 +1275,12 @@ function executorTick(): void { } scheduledJobRuntimeState.isExecutorTickRunning = true; try { + // The in-memory pause is lost when the deployment restarts this worker. + // Keep replacement workers idle until the detached guardian records a + // terminal deployment status in the shared database. + if (hasPendingDeploymentCutover()) { + return; + } const execution = claimNextJobExecution( scheduledJobRuntimeState.workerId, executorCapacity diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index b989a0244..7e242fe88 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1938,8 +1938,11 @@ printf 'scheduled\n' await import("../src/services/pullRequests.ts"); const { enqueueJobExecution, getJobExecution } = await import("../src/services/jobExecutionQueue.ts"); - const { registerScheduledJobAction } = - await import("../src/services/scheduledJobs.ts"); + const { + registerScheduledJobAction, + startScheduledJobExecutor, + stopScheduledJobExecutor, + } = await import("../src/services/scheduledJobs.ts"); registerPullRequestExecutionActions(); registerScheduledJobAction("test.after-deploy", async () => ({})); await startTestScheduledExecutor(); @@ -1980,6 +1983,8 @@ printf 'scheduled\n' () => getJobExecution(deploymentExecution.id)?.status === "success", 5000 ); + await stopScheduledJobExecutor(); + startScheduledJobExecutor(); await Bun.sleep(25); const row = database @@ -2014,7 +2019,15 @@ printf 'scheduled\n' `mira-dashboard-deploy-${job.id}` ); const restartCommand = await Bun.file(systemdLog).text(); - expect(restartCommand).toContain("http://127.0.0.1:4310/api/health/ready"); + expect(restartCommand).toContain( + "/usr/local/bin/doppler run --config prd --project rajohan" + ); + expect(restartCommand).toContain( + "http://127.0.0.1:${dashboard_port}/api/health/ready" + ); + expect(restartCommand).not.toContain( + "http://127.0.0.1:4310/api/health/ready" + ); expect(restartCommand).toContain("--connect-timeout 2 --max-time 5"); expect(restartCommand).toContain("for attempt in {1..30}"); expect(restartCommand).toContain(".checks.release.backendCommit"); From ef9315758c9cf551fb0a3e06959f5d9346320c3d Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 07:58:44 +0200 Subject: [PATCH 05/11] fix(deploy): recover interrupted cutovers safely --- backend/src/releaseDeployment.ts | 54 ++++++++++------ backend/src/releaseManager.ts | 7 +- backend/src/services/pullRequests.ts | 4 ++ backend/src/services/scheduledJobs.ts | 90 ++++++++++++++++++++++++++ backend/test/jobExecutionQueue.test.ts | 68 ++++++++++++++++++- backend/test/releaseDeployment.test.ts | 42 +++++++++++- backend/test/serviceBehavior.test.ts | 8 +++ 7 files changed, 250 insertions(+), 23 deletions(-) diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index 6a291d92f..5bacd2d50 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -78,6 +78,21 @@ export interface ManagedDashboardUnitContract { type ManagedDashboardUnitName = keyof typeof MANAGED_DASHBOARD_UNITS; +function managedReleaseEnvironment( + contract: ManagedDashboardUnitContract, + releaseRoot: string +): NodeJS.ProcessEnv { + return { + ...process.env, + MIRA_DASHBOARD_DB_PATH: contract.databasePath, + MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE: contract.logRotationLockFile, + MIRA_DASHBOARD_OPENCLAW_HOME: contract.openClawHome, + MIRA_DASHBOARD_RELEASE_ROOT: releaseRoot, + MIRA_DASHBOARD_RELEASES_ROOT: contract.releasesRoot, + NODE_ENV: "production", + }; +} + function assertFullCommitSha(commitSha: string): string { if (!RELEASE_COMMIT_SHA_PATTERN.test(commitSha)) { throw new TypeError("Release staging requires a full lowercase Git SHA"); @@ -382,13 +397,32 @@ export async function stageDashboardRelease( options.releasesRoot ?? resolveDashboardReleasesRoot(), "Dashboard releases root" ); + const commandRunner = options.commandRunner ?? defaultCommandRunner; + const contract = managedDashboardUnitContract( + releasesRoot, + options.databasePath ?? + process.env.MIRA_DASHBOARD_DB_PATH ?? + DEFAULT_DASHBOARD_DATABASE_PATH, + options.openClawHome + ); + let existingRelease: ManagedDashboardRelease | undefined; try { - return await loadManagedRelease(releasesRoot, expectedCommit); + existingRelease = await loadManagedRelease(releasesRoot, expectedCommit); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { throw error; } } + if (existingRelease) { + options.onProgress?.("Preflighting existing immutable release"); + await commandRunner("bun", ["dist/databasePreflight.js"], { + cwd: path.join(existingRelease.path, "backend"), + environment: managedReleaseEnvironment(contract, existingRelease.path), + signal: options.signal, + timeoutMs: 120_000, + }); + return existingRelease; + } const sourceRoot = resolveAbsoluteNonRootPath( options.sourceRoot ?? DEFAULT_DASHBOARD_SOURCE_ROOT, @@ -400,27 +434,11 @@ export async function stageDashboardRelease( ); await assertRealDirectory(sourceRoot, "Dashboard source root"); await assertRealDirectory(worktreeRoot, "Dashboard worktree root"); - const commandRunner = options.commandRunner ?? defaultCommandRunner; const worktreePath = path.join( worktreeRoot, `release-${expectedCommit.slice(0, 12)}-${randomUUID()}` ); - const contract = managedDashboardUnitContract( - releasesRoot, - options.databasePath ?? - process.env.MIRA_DASHBOARD_DB_PATH ?? - DEFAULT_DASHBOARD_DATABASE_PATH, - options.openClawHome - ); - const environment: NodeJS.ProcessEnv = { - ...process.env, - MIRA_DASHBOARD_DB_PATH: contract.databasePath, - MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE: contract.logRotationLockFile, - MIRA_DASHBOARD_OPENCLAW_HOME: contract.openClawHome, - MIRA_DASHBOARD_RELEASE_ROOT: worktreePath, - MIRA_DASHBOARD_RELEASES_ROOT: contract.releasesRoot, - NODE_ENV: "production", - }; + const environment = managedReleaseEnvironment(contract, worktreePath); let isWorktreeCreated = false; let stagedRelease: ManagedDashboardRelease | undefined; let stagingError: unknown; diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index e3724265e..1ebf898bd 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -1277,7 +1277,12 @@ export async function pruneDashboardReleases( : -1; } if (left.publishedAtNs === right.publishedAtNs) { - return 0; + // Once the build and filesystem publication timestamps tie, + // the SHA is a deterministic fallback rather than a recency signal. + if (left.release.commitSha === right.release.commitSha) { + return 0; + } + return left.release.commitSha < right.release.commitSha ? 1 : -1; } return left.publishedAtNs < right.publishedAtNs ? 1 : -1; }); diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 24baab331..cf2fd6e0e 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -1551,9 +1551,13 @@ async function scheduleReleaseCutover( "sleep 2", "resolve_dashboard_port() {", ' dashboard_port=$(/usr/local/bin/doppler run --config prd --project rajohan -- /bin/sh -c \'printf "%s" "${PORT:-3100}"\' 2>/dev/null || true)', + ' dashboard_port="$(printf "%s" "$dashboard_port" | /usr/bin/sed -e \'s/^[[:space:]]*//\' -e \'s/[[:space:]]*$//\')"', ' case "$dashboard_port" in', ' ""|*[!0-9]*) dashboard_port=3100 ;;', " esac", + " if ((${#dashboard_port} > 5)); then", + " dashboard_port=3100", + " fi", " if ((10#$dashboard_port < 1 || 10#$dashboard_port > 65535)); then", " dashboard_port=3100", " fi", diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index 98261e30d..1fab40d95 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -33,8 +33,10 @@ const latestRunsJobIdChunkSize = 900; const executorTickMs = 1000; const executorHeartbeatMs = 1000; const executorCapacity = 1; +const deploymentCutoverReconcileIntervalMs = 5000; const interruptedHandlerGraceMs = 30_000; const RELEASE_COMMIT_PATTERN = /^(?:[\da-f]{8,40}|development)$/u; +const DEPLOYMENT_GUARDIAN_UNIT_PREFIX = "mira-dashboard-deploy-"; const actionHandlers = new Map(); const interruptedHandlerSettled = new WeakMap< ScheduledJobInterruptionError, @@ -51,6 +53,7 @@ const scheduledJobRuntimeState: { isSchedulerTickRunning: boolean; isExecutorClaimingPaused: boolean; isExecutorTickRunning: boolean; + nextDeploymentCutoverReconcileAt: number; workerId: string; } = { scheduler: undefined, @@ -60,9 +63,13 @@ const scheduledJobRuntimeState: { isSchedulerTickRunning: false, isExecutorClaimingPaused: false, isExecutorTickRunning: false, + nextDeploymentCutoverReconcileAt: 0, workerId: "", }; +type DeploymentGuardianState = "active" | "inactive" | "unknown"; +type DeploymentGuardianStateReader = (jobId: string) => DeploymentGuardianState; + export type ScheduledJobScheduleType = "interval" | "daily" | "cron"; export type ScheduledJobRunStatus = "queued" | "running" | "success" | "failed" | "cancelled"; @@ -1249,9 +1256,92 @@ function pauseExecutorClaims(): () => void { function resetExecutorClaimPause(): void { scheduledJobRuntimeState.executorClaimPauseGeneration += 1; scheduledJobRuntimeState.isExecutorClaimingPaused = false; + scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt = 0; +} + +function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { + const result = Bun.spawnSync({ + cmd: [ + "systemctl", + "--user", + "is-active", + `${DEPLOYMENT_GUARDIAN_UNIT_PREFIX}${jobId}.service`, + ], + env: process.env, + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + const state = new TextDecoder().decode(result.stdout).trim(); + if (["active", "activating", "deactivating", "reloading"].includes(state)) { + return "active"; + } + if (["inactive", "failed", "unknown"].includes(state)) { + return "inactive"; + } + return "unknown"; +} + +export function reconcileOrphanedDeploymentCutovers( + timestamp = nowIso(), + readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState +): number { + const pending = database + .query( + `SELECT id + FROM deployment_jobs + WHERE status = 'restart-scheduled'` + ) + .all() as Array<{ id: string }>; + const failOrphanedCutover = database.transaction((jobId: string) => { + const result = database + .prepare( + `UPDATE deployment_jobs + SET status = 'failed', + updated_at = ?, + note = 'Detached release cutover guardian is no longer active' + WHERE id = ? + AND status = 'restart-scheduled'` + ) + .run(timestamp, jobId); + if (result.changes === 1) { + database + .prepare("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") + .run(jobId); + } + return result.changes; + }); + let recovered = 0; + for (const { id } of pending) { + let state: DeploymentGuardianState; + try { + state = readGuardianState(id); + } catch (error) { + console.warn( + "[ScheduledJobs] Failed to inspect detached deployment guardian:", + error + ); + continue; + } + if (state === "inactive") { + recovered += failOrphanedCutover(id); + } + } + return recovered; } function hasPendingDeploymentCutover(): boolean { + const now = Date.now(); + if (now >= scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt) { + scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt = + now + deploymentCutoverReconcileIntervalMs; + const recovered = reconcileOrphanedDeploymentCutovers(); + if (recovered > 0) { + console.warn("[ScheduledJobs] Recovered orphaned deployment cutovers", { + recovered, + }); + } + } return Boolean( database .query( diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 689b1b4a3..892f2cdf3 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "bun:test"; +import { afterEach, describe, expect, it, jest } from "bun:test"; import { database } from "../src/database.ts"; import { @@ -24,6 +24,7 @@ import { import { waitForJobExecution } from "../src/services/queuedJobExecution.ts"; import { enqueueScheduledJob, + reconcileOrphanedDeploymentCutovers, recoverOrphanedScheduledJobRuns, registerScheduledJobAction, removeScheduledJobsNotInAction, @@ -36,6 +37,7 @@ import { const testJobIds = new Set(); const testExecutionIds = new Set(); +const testDeploymentIds = new Set(); afterEach(async () => { await stopScheduledJobExecutor(); @@ -46,8 +48,15 @@ afterEach(async () => { database.prepare("DELETE FROM scheduled_job_runs WHERE job_id = ?").run(jobId); database.prepare("DELETE FROM scheduled_jobs WHERE id = ?").run(jobId); } + for (const deploymentId of testDeploymentIds) { + database + .prepare("DELETE FROM deployment_lock WHERE job_id = ?") + .run(deploymentId); + database.prepare("DELETE FROM deployment_jobs WHERE id = ?").run(deploymentId); + } testExecutionIds.clear(); testJobIds.clear(); + testDeploymentIds.clear(); }); function createScheduledTestJob( @@ -69,6 +78,63 @@ function createScheduledTestJob( } describe("persistent job execution queue", () => { + it("fails orphaned detached cutovers before they can pause worker claims", () => { + const deploymentId = `test-orphaned-cutover-${Bun.randomUUIDv7()}`; + const startedAt = "2026-07-26T03:00:00.000Z"; + const recoveredAt = "2026-07-26T03:01:00.000Z"; + testDeploymentIds.add(deploymentId); + database + .prepare( + `INSERT INTO deployment_jobs ( + id, status, started_at, updated_at, note, stdout, stderr + ) VALUES (?, 'restart-scheduled', ?, ?, ?, '', '')` + ) + .run(deploymentId, startedAt, startedAt, "Waiting for guardian"); + database + .prepare( + "INSERT INTO deployment_lock (id, job_id, updated_at) VALUES (1, ?, ?)" + ) + .run(deploymentId, startedAt); + + expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "active")).toBe(0); + expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "unknown")).toBe(0); + const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); + try { + expect( + reconcileOrphanedDeploymentCutovers(recoveredAt, () => { + throw new Error("systemd unavailable"); + }) + ).toBe(0); + expect(warning).toHaveBeenCalledWith( + "[ScheduledJobs] Failed to inspect detached deployment guardian:", + expect.any(Error) + ); + } finally { + warning.mockRestore(); + } + expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "inactive")).toBe( + 1 + ); + expect( + database + .prepare( + `SELECT status, updated_at AS updatedAt, note + FROM deployment_jobs + WHERE id = ?` + ) + .get(deploymentId) + ).toEqual({ + note: "Detached release cutover guardian is no longer active", + status: "failed", + updatedAt: recoveredAt, + }); + expect( + database + .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") + .get(deploymentId) + ).toBeNull(); + }); + it("persists worker progress and structured action failures", async () => { const actionKey = `test.worker-${Bun.randomUUIDv7()}`; registerScheduledJobAction(actionKey, async (_job, _signal, context) => { diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts index 514439bc8..ce94a9695 100644 --- a/backend/test/releaseDeployment.test.ts +++ b/backend/test/releaseDeployment.test.ts @@ -211,7 +211,7 @@ describe("immutable release deployment", () => { expect([...buildReleaseRoots][0]).toStartWith(`${options.worktreeRoot}/release-`); }); - it("reuses an already verified immutable release without running commands", async () => { + it("reruns database preflight when reusing a verified immutable release", async () => { const options = stagingOptions(); const buildRoot = path.join(options.worktreeRoot, "prepared"); mkdirSync(buildRoot); @@ -240,14 +240,50 @@ describe("immutable release deployment", () => { commandRunner: initialRunner, }); + const calls: Array<{ + arguments_: readonly string[]; + command: string; + cwd: string; + databasePath: string | undefined; + releaseRoot: string | undefined; + }> = []; const reused = await stageDashboardRelease(COMMIT_SHA, { ...options, - commandRunner: async () => { - throw new Error("command runner should not be called"); + commandRunner: async (command, arguments_, commandOptions) => { + calls.push({ + arguments_, + command, + cwd: commandOptions.cwd, + databasePath: commandOptions.environment.MIRA_DASHBOARD_DB_PATH, + releaseRoot: commandOptions.environment.MIRA_DASHBOARD_RELEASE_ROOT, + }); + return { stderr: "", stdout: "" }; }, }); expect(reused.commitSha).toBe(COMMIT_SHA); + expect(calls).toEqual([ + { + arguments_: ["dist/databasePreflight.js"], + command: "bun", + cwd: path.join(reused.path, "backend"), + databasePath: options.databasePath, + releaseRoot: reused.path, + }, + ]); + await expect( + stageDashboardRelease(COMMIT_SHA, { + ...options, + commandRunner: async () => { + throw Object.assign( + new Error("database preflight executable missing"), + { + code: "ENOENT", + } + ); + }, + }) + ).rejects.toThrow("database preflight executable missing"); }); it("accepts a concurrently published copy of the same verified release", async () => { diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 7e242fe88..d30c7182c 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1898,6 +1898,10 @@ fi String.raw`#!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >> ${JSON.stringify(systemctlLog)} +if [[ "$*" == "--user is-active mira-dashboard-deploy-"*".service" ]]; then + printf 'active\n' + exit 0 +fi if [[ "$*" != *"--user show"* ]]; then echo "unexpected systemctl args: $*" >&2 exit 2 @@ -2015,6 +2019,9 @@ printf 'scheduled\n' await expect(Bun.file(systemctlLog).text()).resolves.toContain( "show mira-dashboard.service" ); + await expect(Bun.file(systemctlLog).text()).resolves.toContain( + `--user is-active mira-dashboard-deploy-${job.id}.service` + ); await expect(Bun.file(systemdLog).text()).resolves.toContain( `mira-dashboard-deploy-${job.id}` ); @@ -2022,6 +2029,7 @@ printf 'scheduled\n' expect(restartCommand).toContain( "/usr/local/bin/doppler run --config prd --project rajohan" ); + expect(restartCommand).toContain("/usr/bin/sed"); expect(restartCommand).toContain( "http://127.0.0.1:${dashboard_port}/api/health/ready" ); From efb03a3c6a72271259c0cd2dd820c2a6d3878f32 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 08:14:32 +0200 Subject: [PATCH 06/11] fix(deploy): bound guardian recovery state --- backend/src/releaseManager.ts | 4 +- backend/src/services/scheduledJobs.ts | 84 ++++++++++++++++++++------ backend/test/jobExecutionQueue.test.ts | 76 ++++++++++++++++------- backend/test/releaseDeployment.test.ts | 4 +- backend/test/releaseManager.test.ts | 12 ++-- backend/test/serviceBehavior.test.ts | 6 +- 6 files changed, 132 insertions(+), 54 deletions(-) diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index 1ebf898bd..daf2bf397 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -1203,8 +1203,8 @@ export async function pruneDashboardReleases( retainCount = 3, releasesRoot = resolveDashboardReleasesRoot() ): Promise { - if (!Number.isSafeInteger(retainCount) || retainCount < 2 || retainCount > 20) { - throw new TypeError("Managed release retention must be between 2 and 20"); + if (!Number.isSafeInteger(retainCount) || retainCount < 3 || retainCount > 20) { + throw new TypeError("Managed release retention must be between 3 and 20"); } const layout = await ensureDashboardReleaseLayout(releasesRoot); diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index 1fab40d95..c300b2b20 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -34,6 +34,7 @@ const executorTickMs = 1000; const executorHeartbeatMs = 1000; const executorCapacity = 1; const deploymentCutoverReconcileIntervalMs = 5000; +const deploymentCutoverMaximumUnknownMs = 10 * 60 * 1000; const interruptedHandlerGraceMs = 30_000; const RELEASE_COMMIT_PATTERN = /^(?:[\da-f]{8,40}|development)$/u; const DEPLOYMENT_GUARDIAN_UNIT_PREFIX = "mira-dashboard-deploy-"; @@ -1264,56 +1265,91 @@ function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { cmd: [ "systemctl", "--user", - "is-active", + "show", `${DEPLOYMENT_GUARDIAN_UNIT_PREFIX}${jobId}.service`, + "--property=ActiveState", + "--property=LoadState", + "--no-pager", ], env: process.env, stderr: "pipe", stdin: "ignore", stdout: "pipe", }); - const state = new TextDecoder().decode(result.stdout).trim(); - if (["active", "activating", "deactivating", "reloading"].includes(state)) { + const stderr = new TextDecoder().decode(result.stderr).trim(); + if (stderr || result.exitCode !== 0) { + return "unknown"; + } + const properties = new Map( + new TextDecoder() + .decode(result.stdout) + .trim() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return separator === -1 + ? [line, ""] + : [line.slice(0, separator), line.slice(separator + 1)]; + }) + ); + if (properties.get("LoadState") !== "loaded") { + return "unknown"; + } + const state = properties.get("ActiveState"); + if (state && ["active", "activating", "deactivating", "reloading"].includes(state)) { return "active"; } - if (["inactive", "failed", "unknown"].includes(state)) { + if (state && ["inactive", "failed"].includes(state)) { return "inactive"; } return "unknown"; } +function isDeploymentCutoverReconciliationExpired( + updatedAt: string, + timestamp: string +): boolean { + const updatedAtMs = Date.parse(updatedAt); + const timestampMs = Date.parse(timestamp); + return ( + !Number.isFinite(updatedAtMs) || + !Number.isFinite(timestampMs) || + timestampMs - updatedAtMs >= deploymentCutoverMaximumUnknownMs + ); +} + export function reconcileOrphanedDeploymentCutovers( timestamp = nowIso(), readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState ): number { const pending = database .query( - `SELECT id + `SELECT id, updated_at AS updatedAt FROM deployment_jobs WHERE status = 'restart-scheduled'` ) - .all() as Array<{ id: string }>; - const failOrphanedCutover = database.transaction((jobId: string) => { + .all() as Array<{ id: string; updatedAt: string }>; + const failOrphanedCutover = database.transaction((jobId: string, note: string) => { const result = database - .prepare( + .query( `UPDATE deployment_jobs - SET status = 'failed', - updated_at = ?, - note = 'Detached release cutover guardian is no longer active' - WHERE id = ? - AND status = 'restart-scheduled'` + SET status = 'failed', + updated_at = ?, + note = ? + WHERE id = ? + AND status = 'restart-scheduled'` ) - .run(timestamp, jobId); + .run(timestamp, note, jobId); if (result.changes === 1) { database - .prepare("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") + .query("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") .run(jobId); } return result.changes; }); let recovered = 0; - for (const { id } of pending) { - let state: DeploymentGuardianState; + for (const { id, updatedAt } of pending) { + let state: DeploymentGuardianState = "unknown"; try { state = readGuardianState(id); } catch (error) { @@ -1321,10 +1357,20 @@ export function reconcileOrphanedDeploymentCutovers( "[ScheduledJobs] Failed to inspect detached deployment guardian:", error ); - continue; } if (state === "inactive") { - recovered += failOrphanedCutover(id); + recovered += failOrphanedCutover( + id, + "Detached release cutover guardian is no longer active" + ); + } else if ( + state === "unknown" && + isDeploymentCutoverReconciliationExpired(updatedAt, timestamp) + ) { + recovered += failOrphanedCutover( + id, + "Detached release cutover guardian could not be confirmed within ten minutes" + ); } } return recovered; diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 892f2cdf3..b68c5e906 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -77,34 +77,69 @@ function createScheduledTestJob( return id; } +function createRestartScheduledDeployment(updatedAt: string): string { + const deploymentId = `test-orphaned-cutover-${Bun.randomUUIDv7()}`; + testDeploymentIds.add(deploymentId); + database + .prepare( + `INSERT INTO deployment_jobs ( + id, status, started_at, updated_at, note, stdout, stderr + ) VALUES (?, 'restart-scheduled', ?, ?, ?, '', '')` + ) + .run(deploymentId, updatedAt, updatedAt, "Waiting for guardian"); + database + .prepare("INSERT INTO deployment_lock (id, job_id, updated_at) VALUES (1, ?, ?)") + .run(deploymentId, updatedAt); + return deploymentId; +} + describe("persistent job execution queue", () => { it("fails orphaned detached cutovers before they can pause worker claims", () => { - const deploymentId = `test-orphaned-cutover-${Bun.randomUUIDv7()}`; const startedAt = "2026-07-26T03:00:00.000Z"; const recoveredAt = "2026-07-26T03:01:00.000Z"; - testDeploymentIds.add(deploymentId); - database - .prepare( - `INSERT INTO deployment_jobs ( - id, status, started_at, updated_at, note, stdout, stderr - ) VALUES (?, 'restart-scheduled', ?, ?, ?, '', '')` - ) - .run(deploymentId, startedAt, startedAt, "Waiting for guardian"); - database - .prepare( - "INSERT INTO deployment_lock (id, job_id, updated_at) VALUES (1, ?, ?)" - ) - .run(deploymentId, startedAt); + const deploymentId = createRestartScheduledDeployment(startedAt); expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "active")).toBe(0); - expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "unknown")).toBe(0); + expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "inactive")).toBe( + 1 + ); + expect( + database + .prepare( + `SELECT status, updated_at AS updatedAt, note + FROM deployment_jobs + WHERE id = ?` + ) + .get(deploymentId) + ).toEqual({ + note: "Detached release cutover guardian is no longer active", + status: "failed", + updatedAt: recoveredAt, + }); + expect( + database + .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") + .get(deploymentId) + ).toBeNull(); + }); + + it("bounds claim pauses when the guardian state cannot be confirmed", () => { + const startedAt = "2026-07-26T03:00:00.000Z"; + const deploymentId = createRestartScheduledDeployment(startedAt); + + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:01:00.000Z", + () => "unknown" + ) + ).toBe(0); const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); try { expect( - reconcileOrphanedDeploymentCutovers(recoveredAt, () => { + reconcileOrphanedDeploymentCutovers("2026-07-26T03:11:00.000Z", () => { throw new Error("systemd unavailable"); }) - ).toBe(0); + ).toBe(1); expect(warning).toHaveBeenCalledWith( "[ScheduledJobs] Failed to inspect detached deployment guardian:", expect.any(Error) @@ -112,9 +147,6 @@ describe("persistent job execution queue", () => { } finally { warning.mockRestore(); } - expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "inactive")).toBe( - 1 - ); expect( database .prepare( @@ -124,9 +156,9 @@ describe("persistent job execution queue", () => { ) .get(deploymentId) ).toEqual({ - note: "Detached release cutover guardian is no longer active", + note: "Detached release cutover guardian could not be confirmed within ten minutes", status: "failed", - updatedAt: recoveredAt, + updatedAt: "2026-07-26T03:11:00.000Z", }); expect( database diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts index ce94a9695..d78eab40a 100644 --- a/backend/test/releaseDeployment.test.ts +++ b/backend/test/releaseDeployment.test.ts @@ -574,8 +574,8 @@ describe("immutable release deployment", () => { runReleaseDeploymentCommand(["stage"], options.releasesRoot) ).rejects.toThrow("stage requires a commit SHA"); await expect( - runReleaseDeploymentCommand(["prune", "1"], options.releasesRoot) - ).rejects.toThrow("retention must be between 2 and 20"); + runReleaseDeploymentCommand(["prune", "2"], options.releasesRoot) + ).rejects.toThrow("retention must be between 3 and 20"); await expect( runReleaseDeploymentCommand(["prune", "3", "extra"], options.releasesRoot) ).rejects.toThrow("unexpected arguments"); diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index 16ed9fdce..e8495c3da 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -391,8 +391,8 @@ describe("Dashboard immutable release manager", () => { retained: [SECOND_COMMIT, FIRST_COMMIT], warnings: [], }); - await expect(runReleaseLifecycleCommand(["prune", "1"], root)).rejects.toThrow( - "retention must be between 2 and 20" + await expect(runReleaseLifecycleCommand(["prune", "2"], root)).rejects.toThrow( + "retention must be between 3 and 20" ); await expect( runReleaseLifecycleCommand(["prune", "3", "extra"], root) @@ -893,14 +893,14 @@ describe("Dashboard immutable release manager", () => { it("validates release retention bounds", async () => { const root = temporaryReleasesRoot(); - await expect(pruneDashboardReleases(1, root)).rejects.toThrow( - "retention must be between 2 and 20" + await expect(pruneDashboardReleases(2, root)).rejects.toThrow( + "retention must be between 3 and 20" ); await expect(pruneDashboardReleases(21, root)).rejects.toThrow( - "retention must be between 2 and 20" + "retention must be between 3 and 20" ); await expect(pruneDashboardReleases(NaN, root)).rejects.toThrow( - "retention must be between 2 and 20" + "retention must be between 3 and 20" ); }); }); diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index d30c7182c..30117cf88 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1898,8 +1898,8 @@ fi String.raw`#!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >> ${JSON.stringify(systemctlLog)} -if [[ "$*" == "--user is-active mira-dashboard-deploy-"*".service" ]]; then - printf 'active\n' +if [[ "$*" == "--user show mira-dashboard-deploy-"*".service --property=ActiveState --property=LoadState --no-pager" ]]; then + printf 'LoadState=loaded\nActiveState=active\n' exit 0 fi if [[ "$*" != *"--user show"* ]]; then @@ -2020,7 +2020,7 @@ printf 'scheduled\n' "show mira-dashboard.service" ); await expect(Bun.file(systemctlLog).text()).resolves.toContain( - `--user is-active mira-dashboard-deploy-${job.id}.service` + `--user show mira-dashboard-deploy-${job.id}.service --property=ActiveState --property=LoadState --no-pager` ); await expect(Bun.file(systemdLog).text()).resolves.toContain( `mira-dashboard-deploy-${job.id}` From 862b474de72c9e374398ffa14144f6bfb60c839f Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 09:05:45 +0200 Subject: [PATCH 07/11] fix: harden atomic release cutover recovery --- backend/src/releaseDeployment.ts | 19 ++ backend/src/releaseManager.ts | 48 ++++ backend/src/services/jobExecutionQueue.ts | 8 +- backend/src/services/pullRequests.ts | 275 +++++++++++++++++----- backend/src/services/scheduledJobs.ts | 125 ++++++---- backend/test/jobExecutionQueue.test.ts | 64 +++-- backend/test/releaseDeployment.test.ts | 12 +- backend/test/releaseManager.test.ts | 19 ++ backend/test/serviceBehavior.test.ts | 52 +++- docs/setup/production-deploy.md | 15 +- docs/setup/secrets-and-env.md | 7 +- src/pages/PullRequests.tsx | 2 +- systemd/mira-dashboard-worker.service | 2 +- systemd/mira-dashboard.service | 2 +- 14 files changed, 513 insertions(+), 137 deletions(-) diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index 5bacd2d50..a173037e4 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -34,12 +34,30 @@ export const MANAGED_DASHBOARD_UNITS = { "mira-dashboard.service": "dist/serverStart.js", } as const; export const MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT = [ + "NODE_ENV", + "MIRA_DASHBOARD_EXECUTION_ROLE", + "MIRA_DASHBOARD_ENABLE_JOB_SCOPES", + "MIRA_DASHBOARD_JOB_SCOPE_OWNER", "MIRA_DASHBOARD_DB_PATH", "MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE", "MIRA_DASHBOARD_OPENCLAW_HOME", "MIRA_DASHBOARD_RELEASE_ROOT", "MIRA_DASHBOARD_RELEASES_ROOT", ] as const; +const MANAGED_DASHBOARD_UNIT_POLICY_ENVIRONMENT = { + "mira-dashboard-worker.service": [ + "NODE_ENV=production", + "MIRA_DASHBOARD_EXECUTION_ROLE=worker", + "MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1", + "MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard-worker.service", + ], + "mira-dashboard.service": [ + "NODE_ENV=production", + "MIRA_DASHBOARD_EXECUTION_ROLE=web", + "MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1", + "MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard.service", + ], +} as const satisfies Record; export interface DashboardReleaseCommandResult { stderr: string; @@ -340,6 +358,7 @@ export function assertManagedDashboardUnitProperties( contract = managedDashboardUnitContract() ): void { const expectedEnvironment = [ + ...MANAGED_DASHBOARD_UNIT_POLICY_ENVIRONMENT[unit], `MIRA_DASHBOARD_DB_PATH=${contract.databasePath}`, `MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile}`, `MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome}`, diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index daf2bf397..a9f04d457 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -32,6 +32,9 @@ const RELEASE_TRANSITION_JOURNAL_FILE_NAME = ".release-transition.json"; export const RELEASE_TRANSITION_LOCK_FILE_NAME = ".release-transition.lock"; const RETIRED_RELEASE_DIRECTORY_PATTERN = /^\.retired-[\da-f]{40}-[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; +const STAGING_RELEASE_DIRECTORY_PATTERN = + /^\.staging-([\da-f]{40})-[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; +const STALE_STAGING_RELEASE_AGE_MS = 24 * 60 * 60 * 1000; const MAX_RELEASE_TRANSITION_FILE_BYTES = 4096; export const RELEASE_TRANSITION_LOCK_PROGRAM = "/usr/bin/flock"; @@ -1238,6 +1241,51 @@ export async function pruneDashboardReleases( hasFilesystemChanges = true; continue; } + const stagingMatch = STAGING_RELEASE_DIRECTORY_PATTERN.exec(entry.name); + if (stagingMatch) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new TypeError( + `Staging release entry must be a real directory: ${entry.name}` + ); + } + const stagingPath = path.join(layout.releasesPath, entry.name); + const stagingStat = await fsp.lstat(stagingPath, { bigint: true }); + const staleBeforeNs = + BigInt(Math.max(0, Date.now() - STALE_STAGING_RELEASE_AGE_MS)) * + 1_000_000n; + if (stagingStat.mtimeNs > staleBeforeNs) { + continue; + } + const currentStat = await fsp.lstat(stagingPath, { bigint: true }); + if ( + !currentStat.isDirectory() || + currentStat.isSymbolicLink() || + !isSameReleaseDirectoryInode(stagingStat, currentStat) + ) { + throw new Error( + `Staging release changed before cleanup: ${entry.name}` + ); + } + const retiredPath = path.join( + layout.releasesPath, + `.retired-${stagingMatch[1]}-${randomUUID()}` + ); + await fsp.rename(stagingPath, retiredPath); + await syncDirectory(layout.releasesPath); + const retiredStat = await fsp.lstat(retiredPath, { bigint: true }); + if ( + !retiredStat.isDirectory() || + retiredStat.isSymbolicLink() || + !isSameReleaseDirectoryInode(currentStat, retiredStat) + ) { + throw new Error( + `Staging release changed during cleanup: ${entry.name}` + ); + } + await fsp.rm(retiredPath, { recursive: true }); + hasFilesystemChanges = true; + continue; + } if (!RELEASE_COMMIT_SHA_PATTERN.test(entry.name)) { continue; } diff --git a/backend/src/services/jobExecutionQueue.ts b/backend/src/services/jobExecutionQueue.ts index f60dc5a4a..743ed9200 100644 --- a/backend/src/services/jobExecutionQueue.ts +++ b/backend/src/services/jobExecutionQueue.ts @@ -14,7 +14,7 @@ import { const DEFAULT_LEASE_MS = 2 * 60 * 1000; const MAX_EXECUTION_LIST_LIMIT = 200; -const WORKER_HEARTBEAT_MAX_AGE_MS = 30_000; +export const JOB_WORKER_HEARTBEAT_MAX_AGE_MS = 30_000; const RELEASE_COMMIT_PATTERN = /^[\da-f]{8,40}$/u; export type JobExecutionStatus = @@ -432,7 +432,7 @@ export function getJobExecutionSummary(timestamp = Date.now()): JobExecutionSumm const oldestQueuedAt = fromSqlNullable(counts.oldest_queued_at); const parsedOldestQueuedAt = oldestQueuedAt ? Date.parse(oldestQueuedAt) : NaN; const workerFreshAfter = new Date( - timestamp - WORKER_HEARTBEAT_MAX_AGE_MS + timestamp - JOB_WORKER_HEARTBEAT_MAX_AGE_MS ).toISOString(); const worker = database .prepare( @@ -471,7 +471,9 @@ export function isJobWorkerReleaseReady( if (!RELEASE_COMMIT_PATTERN.test(releaseCommit)) { return false; } - const freshAfter = new Date(timestamp - WORKER_HEARTBEAT_MAX_AGE_MS).toISOString(); + const freshAfter = new Date( + timestamp - JOB_WORKER_HEARTBEAT_MAX_AGE_MS + ).toISOString(); const row = database .prepare( `SELECT 1 diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index cf2fd6e0e..778c4fbf6 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -21,6 +21,7 @@ import { } from "../releaseManager.ts"; import { enqueueJobExecution, + JOB_WORKER_HEARTBEAT_MAX_AGE_MS, type JobExecution, registerExpiredJobExecutionHandler, registerQueuedJobCancellationHandler, @@ -30,6 +31,8 @@ import { waitForJobExecution, } from "./queuedJobExecution.ts"; import { + type OrphanedDeploymentCutover, + registerDeploymentCutoverRecoveryHandler, registerScheduledJobAction, type ScheduledJob, type ScheduledJobActionContext, @@ -66,6 +69,8 @@ const MAX_JSON_LINE_LENGTH = 1024 * 1024; const PR_LIST_TIMEOUT_MS = 180_000; const DEPLOYMENT_RESTART_STATUS_POLL_MS = 1000; const DEPLOYMENT_RESTART_CLAIM_PAUSE_TIMEOUT_MS = 2 * 60 * 1000; +const DEPLOYMENT_WORKER_STABILITY_SECONDS = + Math.ceil(JOB_WORKER_HEARTBEAT_MAX_AGE_MS / 1000) + 1; const PASSING_CHECK_VALUES = new Set(["success", "successful", "neutral", "skipped"]); const OPINIONATED_REVIEW_STATES = new Set(["APPROVED", "CHANGES_REQUESTED", "DISMISSED"]); const ACTIVE_DEPLOYMENT_STATUSES = new Set(["building", "restart-scheduled"]); @@ -1459,6 +1464,77 @@ try { ].join(" "); } +function releaseLifecycleInvocation( + releasesRoot: string, + lifecycleCommand: string +): string { + return [ + `MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)}`, + `MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())}`, + "NODE_ENV=production", + shellQuote(resolveBunExecutable()), + shellQuote(lifecycleCommand), + ].join(" "); +} + +function releaseCutoverShellFunctions(): string[] { + return [ + "resolve_dashboard_port() {", + ' dashboard_port=$(/usr/local/bin/doppler run --config prd --project rajohan -- /bin/sh -c \'printf "%s" "${PORT:-3100}"\' 2>/dev/null || true)', + " dashboard_port=\"$(printf \"%s\" \"$dashboard_port\" | /usr/bin/sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/^0*//')\"", + ' [ -n "$dashboard_port" ] || dashboard_port=0', + ' case "$dashboard_port" in', + " *[!0-9]*) dashboard_port=3100 ;;", + " esac", + " if ((${#dashboard_port} > 5)); then", + " dashboard_port=3100", + " fi", + " if ((10#$dashboard_port < 1 || 10#$dashboard_port > 65535)); then", + " dashboard_port=3100", + " fi", + ' printf "%s" "$dashboard_port"', + "}", + "worker_identity() {", + " worker_properties=$(/usr/bin/systemctl --user show mira-dashboard-worker.service --property=ActiveState --property=SubState --property=MainPID --property=ExecMainStartTimestampMonotonic --no-pager 2>/dev/null) || return 1", + String.raw` worker_active="$(printf "%s\n" "$worker_properties" | /usr/bin/sed -n 's/^ActiveState=//p')"`, + String.raw` worker_substate="$(printf "%s\n" "$worker_properties" | /usr/bin/sed -n 's/^SubState=//p')"`, + String.raw` worker_pid="$(printf "%s\n" "$worker_properties" | /usr/bin/sed -n 's/^MainPID=//p')"`, + String.raw` worker_started="$(printf "%s\n" "$worker_properties" | /usr/bin/sed -n 's/^ExecMainStartTimestampMonotonic=//p')"`, + ' [ "$worker_active" = active ] || return 1', + ' [ "$worker_substate" = running ] || return 1', + ' case "$worker_pid:$worker_started" in', + " *[!0-9:]*|0:*|*:0|:*|*:) return 1 ;;", + " esac", + ' printf "%s:%s" "$worker_pid" "$worker_started"', + "}", + "readiness_matches() {", + ' expected_commit="$1"', + ' dashboard_port="$(resolve_dashboard_port)"', + ' response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 "http://127.0.0.1:${dashboard_port}/api/health/ready" 2>/dev/null || true)', + ' printf "%s" "$response" | /usr/bin/jq --exit-status --arg expected "$expected_commit" \'.status == "isReady" and .checks.release.ready == true and .checks.release.backendCommit == $expected and .checks.release.frontendCommit == $expected and .checks.worker.ready == true\' >/dev/null 2>&1', + "}", + "ready_for_commit() {", + ' expected_commit="$1"', + ' initial_worker_identity=""', + " for attempt in {1..30}; do", + ' if readiness_matches "$expected_commit"; then', + ' initial_worker_identity="$(worker_identity || true)"', + ' [ -n "$initial_worker_identity" ] && break', + " fi", + " sleep 1", + " done", + ' [ -n "$initial_worker_identity" ] || return 1', + ` sleep ${DEPLOYMENT_WORKER_STABILITY_SECONDS}`, + ' current_worker_identity="$(worker_identity || true)"', + ' [ "$current_worker_identity" = "$initial_worker_identity" ] || return 1', + ' readiness_matches "$expected_commit"', + "}", + "restart_services() {", + ` /usr/bin/systemctl --user restart ${DASHBOARD_SERVICES.join(" ")}`, + "}", + ]; +} + async function assertManagedDashboardServiceContract( signal?: AbortSignal ): Promise { @@ -1486,38 +1562,37 @@ async function assertManagedDashboardServiceContract( async function scheduleReleaseCutover( job: DeploymentJob, candidateCommit: string, + preActivationCommit: string, rollbackCommit: string, signal?: AbortSignal ): Promise { - if (!job.commit || !/^[\da-f]{8}$/u.test(job.commit)) { - throw new TypeError("Release cutover requires an eight-character commit"); + if (!job.commit || !/^[\da-f]{40}$/u.test(job.commit)) { + throw new TypeError("Release cutover requires a full candidate commit"); } - if ( - !/^[\da-f]{40}$/u.test(candidateCommit) || - candidateCommit.slice(0, 8) !== job.commit - ) { + if (!/^[\da-f]{40}$/u.test(candidateCommit) || candidateCommit !== job.commit) { throw new TypeError("Release cutover requires the matching full candidate SHA"); } - if (!/^[\da-f]{8}$/u.test(rollbackCommit)) { - throw new TypeError( - "Release cutover requires an eight-character rollback commit" - ); + if (!/^[\da-f]{40}$/u.test(preActivationCommit)) { + throw new TypeError("Release cutover requires a full pre-activation commit"); + } + if (rollbackCommit === candidateCommit || !/^[\da-f]{40}$/u.test(rollbackCommit)) { + throw new TypeError("Release cutover requires a distinct full rollback commit"); } const releasesRoot = resolveDashboardReleasesRoot(); - const releaseRoot = path.join(releasesRoot, "current"); const lifecycleCommand = path.join( - releaseRoot, + releasesRoot, + "releases", + preActivationCommit, "backend", "dist", "releaseLifecycle.js" ); - const lifecycleEnvironment = [ - `MIRA_DASHBOARD_RELEASES_ROOT=${shellQuote(releasesRoot)}`, - `MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())}`, - "NODE_ENV=production", - shellQuote(resolveBunExecutable()), - shellQuote(lifecycleCommand), - ].join(" "); + const lifecycleEnvironment = releaseLifecycleInvocation( + releasesRoot, + lifecycleCommand + ); + const candidateShort = candidateCommit.slice(0, 8); + const rollbackShort = rollbackCommit.slice(0, 8); const okJob: DeploymentJob = { ...job, status: "isOk", @@ -1532,13 +1607,13 @@ async function scheduleReleaseCutover( ...job, status: "failed", updatedAt: dateToISOString(new Date()), - note: `Release readiness failed; automatic rollback restored ${rollbackCommit}`, + note: `Release readiness failed; automatic rollback restored ${rollbackShort}`, }; const rollbackFailedJob: DeploymentJob = { ...job, status: "failed", updatedAt: dateToISOString(new Date()), - note: `Release readiness failed and automatic rollback to ${rollbackCommit} failed`, + note: `Release readiness failed and automatic rollback to ${rollbackShort} failed`, }; const activationFailedJob: DeploymentJob = { ...job, @@ -1549,44 +1624,16 @@ async function scheduleReleaseCutover( const script = [ "sleep 2", - "resolve_dashboard_port() {", - ' dashboard_port=$(/usr/local/bin/doppler run --config prd --project rajohan -- /bin/sh -c \'printf "%s" "${PORT:-3100}"\' 2>/dev/null || true)', - ' dashboard_port="$(printf "%s" "$dashboard_port" | /usr/bin/sed -e \'s/^[[:space:]]*//\' -e \'s/[[:space:]]*$//\')"', - ' case "$dashboard_port" in', - ' ""|*[!0-9]*) dashboard_port=3100 ;;', - " esac", - " if ((${#dashboard_port} > 5)); then", - " dashboard_port=3100", - " fi", - " if ((10#$dashboard_port < 1 || 10#$dashboard_port > 65535)); then", - " dashboard_port=3100", - " fi", - ' printf "%s" "$dashboard_port"', - "}", - "ready_for_commit() {", - ' expected_commit="$1"', - ' dashboard_port="$(resolve_dashboard_port)"', - " for attempt in {1..30}; do", - ' response=$(/usr/bin/curl --fail --silent --show-error --connect-timeout 2 --max-time 5 "http://127.0.0.1:${dashboard_port}/api/health/ready" 2>/dev/null || true)', - ' if printf "%s" "$response" | /usr/bin/jq --exit-status --arg expected "$expected_commit" \'.status == "isReady" and .checks.release.ready == true and .checks.release.backendCommit == $expected and .checks.release.frontendCommit == $expected and .checks.worker.ready == true\' >/dev/null 2>&1; then', - " return 0", - " fi", - " sleep 1", - " done", - " return 1", - "}", - "restart_services() {", - ` /usr/bin/systemctl --user restart ${DASHBOARD_SERVICES.join(" ")}`, - "}", + ...releaseCutoverShellFunctions(), `if ${lifecycleEnvironment} activate ${shellQuote(candidateCommit)}; then`, - ` if restart_services && ready_for_commit ${shellQuote(job.commit)}; then`, + ` if restart_services && ready_for_commit ${shellQuote(candidateShort)}; then`, ` if ${lifecycleEnvironment} prune 3; then`, ` ${deploymentJobUpdateCommand(okJob)}`, " else", ` ${deploymentJobUpdateCommand(okWithRetentionWarningJob)}`, " fi", " else", - ` if ${lifecycleEnvironment} rollback && restart_services && ready_for_commit ${shellQuote(rollbackCommit)}; then`, + ` if ${lifecycleEnvironment} rollback && restart_services && ready_for_commit ${shellQuote(rollbackShort)}; then`, ` ${deploymentJobUpdateCommand(rolledBackJob)}`, " else", ` ${deploymentJobUpdateCommand(rollbackFailedJob)}`, @@ -1601,6 +1648,7 @@ async function scheduleReleaseCutover( "systemd-run", [ "--user", + "--collect", `--unit=mira-dashboard-deploy-${job.id}`, "--description=Mira Dashboard atomic release cutover", "/bin/bash", @@ -1611,6 +1659,123 @@ async function scheduleReleaseCutover( ); } +function didScheduleOrphanedReleaseCutoverRecovery( + cutover: OrphanedDeploymentCutover +): boolean { + const job = readDeploymentJob(cutover.id); + if (!job || job.status !== "restart-scheduled") { + return false; + } + const candidateCommit = cutover.candidateCommit ?? job.commit; + if ( + !candidateCommit || + !/^[\da-f]{40}$/u.test(candidateCommit) || + job.commit !== candidateCommit + ) { + throw new Error( + "Orphaned release cutover recovery requires its persisted full candidate SHA" + ); + } + + const releasesRoot = resolveDashboardReleasesRoot(); + const rolledBackJob: DeploymentJob = { + ...job, + status: "failed", + updatedAt: dateToISOString(new Date()), + note: "Interrupted release cutover recovered; automatic rollback restored the previous verified release", + }; + const activationNotAppliedJob: DeploymentJob = { + ...job, + status: "failed", + updatedAt: dateToISOString(new Date()), + note: "Interrupted release cutover recovered before candidate activation; current verified release remains ready", + }; + const script = [ + "sleep 1", + ...releaseCutoverShellFunctions(), + `releases_root=${shellQuote(releasesRoot)}`, + `candidate_commit=${shellQuote(candidateCommit)}`, + `bun_executable=${shellQuote(resolveBunExecutable())}`, + "resolve_trusted_lifecycle() {", + ' current_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/current") || return 1', + ' current_commit="$(/usr/bin/basename -- "$current_release")"', + ' [[ "$current_commit" =~ ^[0-9a-f]{40}$ ]] || return 1', + ' [ "$current_release" = "$releases_root/releases/$current_commit" ] || return 1', + ' if [ "$current_commit" = "$candidate_commit" ]; then', + ' trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous") || return 1', + " else", + ' trusted_release="$current_release"', + " fi", + ' trusted_commit="$(/usr/bin/basename -- "$trusted_release")"', + ' [[ "$trusted_commit" =~ ^[0-9a-f]{40}$ ]] || return 1', + ' [ "$trusted_release" = "$releases_root/releases/$trusted_commit" ] || return 1', + ' trusted_lifecycle="$trusted_release/backend/dist/releaseLifecycle.js"', + ' [ -f "$trusted_lifecycle" ] && [ ! -L "$trusted_lifecycle" ]', + "}", + "run_lifecycle() {", + ' MIRA_DASHBOARD_RELEASES_ROOT="$releases_root" \\', + ` MIRA_DASHBOARD_DB_PATH=${shellQuote(getMiraDatabasePath())} \\`, + " NODE_ENV=production \\", + ' "$bun_executable" "$trusted_lifecycle" "$@"', + "}", + "resolve_trusted_lifecycle", + 'if activation_output="$(run_lifecycle activate "$candidate_commit")"; then', + ' rollback_commit="$(printf "%s" "$activation_output" | /usr/bin/jq --raw-output \'.previous.commitSha // empty\')"', + ' [[ "$rollback_commit" =~ ^[0-9a-f]{40}$ ]] || exit 1', + ' [ "$rollback_commit" != "$candidate_commit" ] || exit 1', + ' if run_lifecycle rollback && restart_services && ready_for_commit "${rollback_commit:0:8}"; then', + ` ${deploymentJobUpdateCommand(rolledBackJob)}`, + " else", + " exit 1", + " fi", + "else", + ' status_output="$(run_lifecycle status)" || exit 1', + ' current_commit="$(printf "%s" "$status_output" | /usr/bin/jq --raw-output \'.current.commitSha // empty\')"', + ' [[ "$current_commit" =~ ^[0-9a-f]{40}$ ]] || exit 1', + ' [ "$current_commit" != "$candidate_commit" ] || exit 1', + ' if ready_for_commit "${current_commit:0:8}" || { restart_services && ready_for_commit "${current_commit:0:8}"; }; then', + ` ${deploymentJobUpdateCommand(activationNotAppliedJob)}`, + " else", + " exit 1", + " fi", + "fi", + ].join("\n"); + + writeDeploymentJob({ + ...job, + updatedAt: dateToISOString(new Date()), + note: "Detached release guardian ended without a terminal result; automatic rollback recovery scheduled", + }); + const result = Bun.spawnSync({ + cmd: [ + "systemd-run", + "--user", + "--collect", + `--unit=mira-dashboard-deploy-recovery-${job.id}`, + "--description=Mira Dashboard orphaned release rollback", + "/bin/bash", + "-lc", + script, + ], + cwd: getDashboardRoot(), + env: buildCommandEnvironment(), + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + if (result.exitCode !== 0) { + const diagnostic = + new TextDecoder().decode(result.stderr).trim() || + new TextDecoder().decode(result.stdout).trim(); + throw new Error( + `systemd-run failed to schedule orphaned release rollback: ${ + diagnostic || `exit ${result.exitCode}` + }` + ); + } + return true; +} + /** Runs deployment work after the API has returned a job to the caller. */ async function runDeploymentJob( job: DeploymentJob, @@ -1669,7 +1834,7 @@ async function runDeploymentJob( ...currentJob, status: "restart-scheduled", updatedAt: dateToISOString(new Date()), - commit: candidate.manifest.commitShort, + commit: candidate.manifest.commitSha, commitTitle: candidate.manifest.commitTitle, note: "Immutable release published. Detached activation and rollback check scheduled", }; @@ -1677,7 +1842,8 @@ async function runDeploymentJob( await scheduleReleaseCutover( restartScheduled, expectedCommit, - rollbackRelease.manifest.commitShort, + currentState.current.commitSha, + rollbackRelease.commitSha, signal ); return true; @@ -2055,6 +2221,7 @@ async function executePullRequestMerge( /** Registers every mutating GitHub/deploy action exclusively in the worker. */ export function registerPullRequestExecutionActions(): void { registerPullRequestJobLifecycleHandlers(); + registerDeploymentCutoverRecoveryHandler(didScheduleOrphanedReleaseCutoverRecovery); registerScheduledJobAction("dashboard.deploy", async (job, signal, context) => { const deploymentId = job.actionPayload.deploymentId; if (typeof deploymentId !== "string" || deploymentId.trim() === "") { diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index c300b2b20..f3099c446 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -38,6 +38,7 @@ const deploymentCutoverMaximumUnknownMs = 10 * 60 * 1000; const interruptedHandlerGraceMs = 30_000; const RELEASE_COMMIT_PATTERN = /^(?:[\da-f]{8,40}|development)$/u; const DEPLOYMENT_GUARDIAN_UNIT_PREFIX = "mira-dashboard-deploy-"; +const DEPLOYMENT_RECOVERY_UNIT_PREFIX = "mira-dashboard-deploy-recovery-"; const actionHandlers = new Map(); const interruptedHandlerSettled = new WeakMap< ScheduledJobInterruptionError, @@ -46,11 +47,23 @@ const interruptedHandlerSettled = new WeakMap< const activeExecutionControllers = new Map(); const activeExecutionRuns = new Map>(); +type DeploymentGuardianState = "active" | "inactive" | "unknown"; +type DeploymentGuardianStateReader = (jobId: string) => DeploymentGuardianState; +export interface OrphanedDeploymentCutover { + candidateCommit?: string; + id: string; + updatedAt: string; +} +export type DeploymentCutoverRecoveryHandler = ( + cutover: OrphanedDeploymentCutover +) => boolean; + const scheduledJobRuntimeState: { scheduler: NodeJS.Timeout | undefined; executor: NodeJS.Timeout | undefined; workerHeartbeat: NodeJS.Timeout | undefined; executorClaimPauseGeneration: number; + deploymentCutoverRecoveryHandler: DeploymentCutoverRecoveryHandler | undefined; isSchedulerTickRunning: boolean; isExecutorClaimingPaused: boolean; isExecutorTickRunning: boolean; @@ -61,6 +74,7 @@ const scheduledJobRuntimeState: { executor: undefined, workerHeartbeat: undefined, executorClaimPauseGeneration: 0, + deploymentCutoverRecoveryHandler: undefined, isSchedulerTickRunning: false, isExecutorClaimingPaused: false, isExecutorTickRunning: false, @@ -68,9 +82,6 @@ const scheduledJobRuntimeState: { workerId: "", }; -type DeploymentGuardianState = "active" | "inactive" | "unknown"; -type DeploymentGuardianStateReader = (jobId: string) => DeploymentGuardianState; - export type ScheduledJobScheduleType = "interval" | "daily" | "cron"; export type ScheduledJobRunStatus = "queued" | "running" | "success" | "failed" | "cancelled"; @@ -1260,13 +1271,15 @@ function resetExecutorClaimPause(): void { scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt = 0; } -function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { +type SystemdUnitState = DeploymentGuardianState | "missing"; + +function readSystemdUnitState(unit: string): SystemdUnitState { const result = Bun.spawnSync({ cmd: [ "systemctl", "--user", "show", - `${DEPLOYMENT_GUARDIAN_UNIT_PREFIX}${jobId}.service`, + unit, "--property=ActiveState", "--property=LoadState", "--no-pager", @@ -1292,6 +1305,9 @@ function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { : [line.slice(0, separator), line.slice(separator + 1)]; }) ); + if (properties.get("LoadState") === "not-found") { + return "missing"; + } if (properties.get("LoadState") !== "loaded") { return "unknown"; } @@ -1305,6 +1321,25 @@ function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { return "unknown"; } +function readDeploymentGuardianState(jobId: string): DeploymentGuardianState { + const guardian = readSystemdUnitState( + `${DEPLOYMENT_GUARDIAN_UNIT_PREFIX}${jobId}.service` + ); + if (guardian === "active") { + return "active"; + } + const recovery = readSystemdUnitState( + `${DEPLOYMENT_RECOVERY_UNIT_PREFIX}${jobId}.service` + ); + if (recovery === "active") { + return "active"; + } + if (guardian === "unknown" || recovery === "unknown") { + return "unknown"; + } + return "inactive"; +} + function isDeploymentCutoverReconciliationExpired( updatedAt: string, timestamp: string @@ -1320,62 +1355,66 @@ function isDeploymentCutoverReconciliationExpired( export function reconcileOrphanedDeploymentCutovers( timestamp = nowIso(), - readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState + readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState, + recoverCutover: + | DeploymentCutoverRecoveryHandler + | undefined = scheduledJobRuntimeState.deploymentCutoverRecoveryHandler ): number { - const pending = database + const pendingRows = database .query( - `SELECT id, updated_at AS updatedAt + `SELECT id, commit_sha AS candidateCommit, updated_at AS updatedAt FROM deployment_jobs WHERE status = 'restart-scheduled'` ) - .all() as Array<{ id: string; updatedAt: string }>; - const failOrphanedCutover = database.transaction((jobId: string, note: string) => { - const result = database - .query( - `UPDATE deployment_jobs - SET status = 'failed', - updated_at = ?, - note = ? - WHERE id = ? - AND status = 'restart-scheduled'` - ) - .run(timestamp, note, jobId); - if (result.changes === 1) { - database - .query("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") - .run(jobId); - } - return result.changes; - }); + .all() as Array<{ + candidateCommit: string | null; + id: string; + updatedAt: string; + }>; + const pending: OrphanedDeploymentCutover[] = pendingRows.map((row) => ({ + ...(row.candidateCommit && { candidateCommit: row.candidateCommit }), + id: row.id, + updatedAt: row.updatedAt, + })); let recovered = 0; - for (const { id, updatedAt } of pending) { + for (const cutover of pending) { let state: DeploymentGuardianState = "unknown"; try { - state = readGuardianState(id); + state = readGuardianState(cutover.id); } catch (error) { console.warn( "[ScheduledJobs] Failed to inspect detached deployment guardian:", error ); } - if (state === "inactive") { - recovered += failOrphanedCutover( - id, - "Detached release cutover guardian is no longer active" - ); - } else if ( - state === "unknown" && - isDeploymentCutoverReconciliationExpired(updatedAt, timestamp) - ) { - recovered += failOrphanedCutover( - id, - "Detached release cutover guardian could not be confirmed within ten minutes" + const shouldRecover = + state === "inactive" || + (state === "unknown" && + isDeploymentCutoverReconciliationExpired(cutover.updatedAt, timestamp)); + if (!shouldRecover || !recoverCutover) { + continue; + } + try { + if (recoverCutover(cutover)) { + recovered += 1; + } + } catch (error) { + console.warn( + "[ScheduledJobs] Failed to schedule orphaned deployment rollback:", + error ); } } return recovered; } +/** Registers the detached rollback scheduler used for orphaned release cutovers. */ +export function registerDeploymentCutoverRecoveryHandler( + didScheduleRecovery: DeploymentCutoverRecoveryHandler +): void { + scheduledJobRuntimeState.deploymentCutoverRecoveryHandler = didScheduleRecovery; +} + function hasPendingDeploymentCutover(): boolean { const now = Date.now(); if (now >= scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt) { @@ -1383,8 +1422,8 @@ function hasPendingDeploymentCutover(): boolean { now + deploymentCutoverReconcileIntervalMs; const recovered = reconcileOrphanedDeploymentCutovers(); if (recovered > 0) { - console.warn("[ScheduledJobs] Recovered orphaned deployment cutovers", { - recovered, + console.warn("[ScheduledJobs] Scheduled orphaned deployment rollbacks", { + scheduled: recovered, }); } } diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index b68c5e906..40cbd1edc 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -79,14 +79,15 @@ function createScheduledTestJob( function createRestartScheduledDeployment(updatedAt: string): string { const deploymentId = `test-orphaned-cutover-${Bun.randomUUIDv7()}`; + const candidateCommit = "c".repeat(40); testDeploymentIds.add(deploymentId); database .prepare( `INSERT INTO deployment_jobs ( - id, status, started_at, updated_at, note, stdout, stderr - ) VALUES (?, 'restart-scheduled', ?, ?, ?, '', '')` + id, status, started_at, updated_at, commit_sha, note, stdout, stderr + ) VALUES (?, 'restart-scheduled', ?, ?, ?, ?, '', '')` ) - .run(deploymentId, updatedAt, updatedAt, "Waiting for guardian"); + .run(deploymentId, updatedAt, updatedAt, candidateCommit, "Waiting for guardian"); database .prepare("INSERT INTO deployment_lock (id, job_id, updated_at) VALUES (1, ?, ?)") .run(deploymentId, updatedAt); @@ -94,15 +95,24 @@ function createRestartScheduledDeployment(updatedAt: string): string { } describe("persistent job execution queue", () => { - it("fails orphaned detached cutovers before they can pause worker claims", () => { + it("schedules rollback recovery for an inactive detached cutover", () => { const startedAt = "2026-07-26T03:00:00.000Z"; const recoveredAt = "2026-07-26T03:01:00.000Z"; const deploymentId = createRestartScheduledDeployment(startedAt); + const recovery = jest.fn(() => true); - expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "active")).toBe(0); - expect(reconcileOrphanedDeploymentCutovers(recoveredAt, () => "inactive")).toBe( - 1 - ); + expect( + reconcileOrphanedDeploymentCutovers(recoveredAt, () => "active", recovery) + ).toBe(0); + expect(recovery).not.toHaveBeenCalled(); + expect( + reconcileOrphanedDeploymentCutovers(recoveredAt, () => "inactive", recovery) + ).toBe(1); + expect(recovery).toHaveBeenCalledWith({ + candidateCommit: "c".repeat(40), + id: deploymentId, + updatedAt: startedAt, + }); expect( database .prepare( @@ -112,33 +122,40 @@ describe("persistent job execution queue", () => { ) .get(deploymentId) ).toEqual({ - note: "Detached release cutover guardian is no longer active", - status: "failed", - updatedAt: recoveredAt, + note: "Waiting for guardian", + status: "restart-scheduled", + updatedAt: startedAt, }); expect( database .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") .get(deploymentId) - ).toBeNull(); + ).toEqual({ job_id: deploymentId }); }); - it("bounds claim pauses when the guardian state cannot be confirmed", () => { + it("bounds unknown guardian inspection by scheduling rollback recovery", () => { const startedAt = "2026-07-26T03:00:00.000Z"; const deploymentId = createRestartScheduledDeployment(startedAt); + const recovery = jest.fn(() => true); expect( reconcileOrphanedDeploymentCutovers( "2026-07-26T03:01:00.000Z", - () => "unknown" + () => "unknown", + recovery ) ).toBe(0); + expect(recovery).not.toHaveBeenCalled(); const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); try { expect( - reconcileOrphanedDeploymentCutovers("2026-07-26T03:11:00.000Z", () => { - throw new Error("systemd unavailable"); - }) + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:11:00.000Z", + () => { + throw new Error("systemd unavailable"); + }, + recovery + ) ).toBe(1); expect(warning).toHaveBeenCalledWith( "[ScheduledJobs] Failed to inspect detached deployment guardian:", @@ -147,6 +164,11 @@ describe("persistent job execution queue", () => { } finally { warning.mockRestore(); } + expect(recovery).toHaveBeenCalledWith({ + candidateCommit: "c".repeat(40), + id: deploymentId, + updatedAt: startedAt, + }); expect( database .prepare( @@ -156,15 +178,15 @@ describe("persistent job execution queue", () => { ) .get(deploymentId) ).toEqual({ - note: "Detached release cutover guardian could not be confirmed within ten minutes", - status: "failed", - updatedAt: "2026-07-26T03:11:00.000Z", + note: "Waiting for guardian", + status: "restart-scheduled", + updatedAt: startedAt, }); expect( database .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") .get(deploymentId) - ).toBeNull(); + ).toEqual({ job_id: deploymentId }); }); it("persists worker progress and structured action failures", async () => { diff --git a/backend/test/releaseDeployment.test.ts b/backend/test/releaseDeployment.test.ts index d78eab40a..4facd8d70 100644 --- a/backend/test/releaseDeployment.test.ts +++ b/backend/test/releaseDeployment.test.ts @@ -494,7 +494,7 @@ describe("immutable release deployment", () => { ); const properties = [ `WorkingDirectory=${contract.releaseRoot}/backend`, - `Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${contract.databasePath} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${contract.releaseRoot} MIRA_DASHBOARD_RELEASES_ROOT=${contract.releasesRoot}`, + `Environment=NODE_ENV=production MIRA_DASHBOARD_EXECUTION_ROLE=web MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard.service MIRA_DASHBOARD_DB_PATH=${contract.databasePath} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${contract.logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${contract.openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${contract.releaseRoot} MIRA_DASHBOARD_RELEASES_ROOT=${contract.releasesRoot}`, `ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run --preserve-env=${MANAGED_DASHBOARD_PRESERVED_ENVIRONMENT.join(",")} -- bun dist/serverStart.js ; }`, ].join("\n"); expect(() => @@ -521,6 +521,16 @@ describe("immutable release deployment", () => { contract ) ).toThrow("unexpected managed release entrypoint"); + expect(() => + assertManagedDashboardUnitProperties( + "mira-dashboard.service", + properties.replace( + "MIRA_DASHBOARD_EXECUTION_ROLE=web", + "MIRA_DASHBOARD_EXECUTION_ROLE=worker" + ), + contract + ) + ).toThrow("missing stable managed release environment"); expect(() => assertManagedDashboardUnitProperties( "mira-dashboard.service", diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index e8495c3da..f18d5391c 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -10,6 +10,7 @@ import { renameSync, rmSync, symlinkSync, + utimesSync, writeFileSync, } from "node:fs"; import fsp from "node:fs/promises"; @@ -868,6 +869,22 @@ describe("Dashboard immutable release manager", () => { ); mkdirSync(interruptedRetirementPath); writeFileSync(path.join(interruptedRetirementPath, "stale"), "stale\n"); + const staleStagingPath = path.join( + root, + "releases", + `.staging-${FIRST_COMMIT}-00000000-0000-4000-8000-000000000001` + ); + mkdirSync(staleStagingPath); + writeFileSync(path.join(staleStagingPath, "partial"), "partial\n"); + const staleTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000); + utimesSync(staleStagingPath, staleTimestamp, staleTimestamp); + const activeStagingPath = path.join( + root, + "releases", + `.staging-${FOURTH_COMMIT}-00000000-0000-4000-8000-000000000002` + ); + mkdirSync(activeStagingPath); + writeFileSync(path.join(activeStagingPath, "partial"), "active\n"); const unverifiableCommit = "e".repeat(40); const unverifiablePath = managedReleasePath(root, unverifiableCommit); mkdirSync(unverifiablePath); @@ -886,6 +903,8 @@ describe("Dashboard immutable release manager", () => { expect(existsSync(managedReleasePath(root, FOURTH_COMMIT))).toBe(true); expect(existsSync(unverifiablePath)).toBe(true); expect(existsSync(interruptedRetirementPath)).toBe(false); + expect(existsSync(staleStagingPath)).toBe(false); + expect(existsSync(activeStagingPath)).toBe(true); const state = await readDashboardReleaseState(root); expect(state.current?.commitSha).toBe(THIRD_COMMIT); expect(state.previous?.commitSha).toBe(SECOND_COMMIT); diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 30117cf88..15638078c 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1908,12 +1908,16 @@ if [[ "$*" != *"--user show"* ]]; then fi if [[ "$*" == *"mira-dashboard-worker.service"* ]]; then entrypoint="dist/workerStart.js" + execution_role="worker" + scope_owner="mira-dashboard-worker.service" else entrypoint="dist/serverStart.js" + execution_role="web" + scope_owner="mira-dashboard.service" fi printf '%s\n' \ - 'Environment=NODE_ENV=production MIRA_DASHBOARD_DB_PATH=${getMiraDatabasePath()} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${releasesRoot}/current MIRA_DASHBOARD_RELEASES_ROOT=${releasesRoot}' \ - "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- bun $entrypoint ; }" \ + "Environment=NODE_ENV=production MIRA_DASHBOARD_EXECUTION_ROLE=$execution_role MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 MIRA_DASHBOARD_JOB_SCOPE_OWNER=$scope_owner MIRA_DASHBOARD_DB_PATH=${getMiraDatabasePath()} MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=${logRotationLockFile} MIRA_DASHBOARD_OPENCLAW_HOME=${openClawHome} MIRA_DASHBOARD_RELEASE_ROOT=${releasesRoot}/current MIRA_DASHBOARD_RELEASES_ROOT=${releasesRoot}" \ + "ExecStart={ path=/usr/local/bin/doppler ; argv[]=/usr/local/bin/doppler run --preserve-env=NODE_ENV,MIRA_DASHBOARD_EXECUTION_ROLE,MIRA_DASHBOARD_ENABLE_JOB_SCOPES,MIRA_DASHBOARD_JOB_SCOPE_OWNER,MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- bun $entrypoint ; }" \ 'WorkingDirectory=${releasesRoot}/current/backend' ` ); @@ -1921,7 +1925,8 @@ printf '%s\n' \ path.join(fakeBin, "systemd-run"), String.raw`#!/usr/bin/env bash set -euo pipefail -/bin/bash -n <<<"$6" +script="${"$"}{!#}" +/bin/bash -n <<<"$script" printf '%s\n' "$*" >> ${JSON.stringify(systemdLog)} printf 'scheduled\n' ` @@ -1943,6 +1948,7 @@ printf 'scheduled\n' const { enqueueJobExecution, getJobExecution } = await import("../src/services/jobExecutionQueue.ts"); const { + reconcileOrphanedDeploymentCutovers, registerScheduledJobAction, startScheduledJobExecutor, stopScheduledJobExecutor, @@ -2002,7 +2008,7 @@ printf 'scheduled\n' status: string; }; expect(row).toEqual({ - commit_sha: candidateCommit.slice(0, 8), + commit_sha: candidateCommit, commit_title: "Deployable dashboard commit", note: "Immutable release published. Detached activation and rollback check scheduled", status: "restart-scheduled", @@ -2026,10 +2032,12 @@ printf 'scheduled\n' `mira-dashboard-deploy-${job.id}` ); const restartCommand = await Bun.file(systemdLog).text(); + expect(restartCommand).toContain("--collect"); expect(restartCommand).toContain( "/usr/local/bin/doppler run --config prd --project rajohan" ); expect(restartCommand).toContain("/usr/bin/sed"); + expect(restartCommand).toContain("s/^0*//"); expect(restartCommand).toContain( "http://127.0.0.1:${dashboard_port}/api/health/ready" ); @@ -2038,8 +2046,14 @@ printf 'scheduled\n' ); expect(restartCommand).toContain("--connect-timeout 2 --max-time 5"); expect(restartCommand).toContain("for attempt in {1..30}"); + expect(restartCommand).toContain("worker_identity"); + expect(restartCommand).toContain("ExecMainStartTimestampMonotonic"); + expect(restartCommand).toContain("sleep 31"); expect(restartCommand).toContain(".checks.release.backendCommit"); expect(restartCommand).toContain("releaseLifecycle.js"); + expect(restartCommand).toContain( + `${releasesRoot}/releases/${oldCommit}/backend/dist/releaseLifecycle.js` + ); expect(restartCommand).toContain(`activate '${candidateCommit}'`); expect(restartCommand.indexOf(`activate '${candidateCommit}'`)).toBeLessThan( restartCommand.indexOf("if restart_services") @@ -2071,6 +2085,36 @@ printf 'scheduled\n' expect(getJobExecution(followUpExecution.id)).toMatchObject({ status: "queued", }); + expect( + reconcileOrphanedDeploymentCutovers( + new Date().toISOString(), + () => "inactive" + ) + ).toBe(1); + const recoveryCommand = await Bun.file(systemdLog).text(); + expect(recoveryCommand).toContain(`mira-dashboard-deploy-recovery-${job.id}`); + expect(recoveryCommand).toContain( + 'current_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/current")' + ); + expect(recoveryCommand).toContain( + 'trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous")' + ); + expect(recoveryCommand).toContain( + "run_lifecycle rollback && restart_services" + ); + expect( + database + .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") + .get(job.id) + ).toEqual({ + note: "Detached release guardian ended without a terminal result; automatic rollback recovery scheduled", + status: "restart-scheduled", + }); + expect( + database + .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") + .get(job.id) + ).toEqual({ job_id: job.id }); database .prepare( `UPDATE deployment_jobs diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 2152acd6b..93cec5b00 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -50,8 +50,10 @@ MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/curren MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases ``` -Their Doppler command selectively preserves these five values, so production -secrets cannot replace unit-owned state or release paths. +Their Doppler command selectively preserves these five values plus `NODE_ENV`, +`MIRA_DASHBOARD_EXECUTION_ROLE`, `MIRA_DASHBOARD_ENABLE_JOB_SCOPES`, and +`MIRA_DASHBOARD_JOB_SCOPE_OWNER`, so production secrets cannot replace +unit-owned state, release paths, or orchestration policy. The OpenClaw home preserves the signed Gateway device identity across releases. Secrets remain in Doppler `rajohan/prd`; tracked unit files contain no secret @@ -240,12 +242,15 @@ before retrying; never continue to candidate activation after this branch. ### 4. Activate and verify the candidate ```bash -env \ +if ! env \ MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ NODE_ENV=production \ bun "$RELEASES_ROOT/releases/$BOOTSTRAP_SHA/backend/dist/releaseLifecycle.js" \ - activate "$CANDIDATE_SHA" + activate "$CANDIDATE_SHA"; then + echo "Candidate activation failed; bootstrap remains active" >&2 + exit 1 +fi systemctl --user restart mira-dashboard-worker.service mira-dashboard.service ``` @@ -256,7 +261,7 @@ if ! ready_for_commit "$CANDIDATE_SHA"; then MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ NODE_ENV=production \ - bun "$RELEASES_ROOT/releases/$CANDIDATE_SHA/backend/dist/releaseLifecycle.js" \ + bun "$RELEASES_ROOT/releases/$BOOTSTRAP_SHA/backend/dist/releaseLifecycle.js" \ rollback; then echo "Candidate rollback failed; manual recovery is required" >&2 exit 1 diff --git a/docs/setup/secrets-and-env.md b/docs/setup/secrets-and-env.md index ef2087cf6..b409f976f 100644 --- a/docs/setup/secrets-and-env.md +++ b/docs/setup/secrets-and-env.md @@ -86,9 +86,10 @@ password-hashed recovery validators need no equivalent decryption key. | `MIRA_DASHBOARD_JOB_SCOPE_OWNER` | owning service unit | Binds transient scopes to their service lifecycle so restarts terminate orphaned children. | | `MIRA_DASHBOARD_DISABLE_SCHEDULER` | unset in production | Development/test escape hatch; `1` disables scheduler/executor startup. | -The tracked systemd units set these orchestration values directly. Doppler -remains the source of auth, origin, provider, and credential values. Production -actions run in the worker, so their child scopes bind to +The tracked systemd units set these orchestration values directly and preserve +them, together with `NODE_ENV` and the managed state/release paths, through +Doppler. Doppler remains the source of auth, origin, provider, and credential +values. Production actions run in the worker, so their child scopes bind to `mira-dashboard-worker.service`; restarting only the web unit leaves them untouched. diff --git a/src/pages/PullRequests.tsx b/src/pages/PullRequests.tsx index d45f6a86e..059780982 100644 --- a/src/pages/PullRequests.tsx +++ b/src/pages/PullRequests.tsx @@ -299,7 +299,7 @@ function SectionHeader({ /** Renders the deployment commit title and commit reference. */ function deploymentCommitLabel(deployment: DeploymentJob): ReactNode { - const commit = deployment.commit || deployment.id; + const commit = deployment.commit?.slice(0, 8) || deployment.id; if (!deployment.commitTitle) return commit; return ( diff --git a/systemd/mira-dashboard-worker.service b/systemd/mira-dashboard-worker.service index b83f59cd7..344dc197e 100644 --- a/systemd/mira-dashboard-worker.service +++ b/systemd/mira-dashboard-worker.service @@ -16,7 +16,7 @@ Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-das Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases -ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/workerStart.js +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=NODE_ENV,MIRA_DASHBOARD_EXECUTION_ROLE,MIRA_DASHBOARD_ENABLE_JOB_SCOPES,MIRA_DASHBOARD_JOB_SCOPE_OWNER,MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/workerStart.js Restart=on-failure RestartSec=5 KillMode=control-group diff --git a/systemd/mira-dashboard.service b/systemd/mira-dashboard.service index a8dfadf04..3705a2dbd 100644 --- a/systemd/mira-dashboard.service +++ b/systemd/mira-dashboard.service @@ -16,7 +16,7 @@ Environment=MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE=/home/ubuntu/projects/mira-das Environment=MIRA_DASHBOARD_OPENCLAW_HOME=/home/ubuntu/projects/mira-dashboard-state/openclaw-client Environment=MIRA_DASHBOARD_RELEASE_ROOT=/home/ubuntu/projects/mira-dashboard-releases/current Environment=MIRA_DASHBOARD_RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases -ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/serverStart.js +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan --preserve-env=NODE_ENV,MIRA_DASHBOARD_EXECUTION_ROLE,MIRA_DASHBOARD_ENABLE_JOB_SCOPES,MIRA_DASHBOARD_JOB_SCOPE_OWNER,MIRA_DASHBOARD_DB_PATH,MIRA_DASHBOARD_LOG_ROTATION_LOCK_FILE,MIRA_DASHBOARD_OPENCLAW_HOME,MIRA_DASHBOARD_RELEASE_ROOT,MIRA_DASHBOARD_RELEASES_ROOT -- /home/ubuntu/.bun/bin/bun dist/serverStart.js Restart=on-failure RestartSec=5 KillMode=control-group From 21f7242f4638e7d79ae7afa72e2e48fca46f57ad Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 09:26:48 +0200 Subject: [PATCH 08/11] fix: serialize immutable release publication --- backend/src/releaseDeployment.ts | 123 +------------------- backend/src/releaseManager.ts | 172 ++++++++++++++++++++++++---- backend/test/releaseManager.test.ts | 28 +++++ 3 files changed, 181 insertions(+), 142 deletions(-) diff --git a/backend/src/releaseDeployment.ts b/backend/src/releaseDeployment.ts index a173037e4..548578ac0 100644 --- a/backend/src/releaseDeployment.ts +++ b/backend/src/releaseDeployment.ts @@ -1,5 +1,4 @@ import { randomUUID } from "node:crypto"; -import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; @@ -7,19 +6,12 @@ import { runProcess } from "./lib/processes.ts"; import { resolveAbsoluteNonRootPath } from "./lib/safePath.ts"; import { type DashboardReleaseRetentionResult, - ensureDashboardReleaseLayout, loadManagedRelease, type ManagedDashboardRelease, - managedReleasePath, pruneDashboardReleases, + publishVerifiedDashboardRelease, resolveDashboardReleasesRoot, } from "./releaseManager.ts"; -import { - loadReleaseManifest, - RELEASE_MANIFEST_FILE_NAME, - verifyReleaseArtifacts, - verifyReleaseBuildIdentities, -} from "./releaseManifest.ts"; const RELEASE_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; const DEFAULT_DASHBOARD_SOURCE_ROOT = "/home/ubuntu/projects/mira-dashboard"; @@ -194,27 +186,6 @@ async function assertRealDirectory(directoryPath: string, label: string): Promis } } -async function syncFile(filePath: string): Promise { - const file = await fsp.open(filePath, fs.constants.O_RDONLY); - try { - await file.sync(); - } finally { - await file.close(); - } -} - -async function syncDirectory(directoryPath: string): Promise { - const directory = await fsp.open( - directoryPath, - fs.constants.O_RDONLY | fs.constants.O_DIRECTORY - ); - try { - await directory.sync(); - } finally { - await directory.close(); - } -} - async function defaultCommandRunner( command: string, arguments_: readonly string[], @@ -242,96 +213,6 @@ async function defaultCommandRunner( return { stderr: result.stderr, stdout: result.stdout }; } -async function copyVerifiedRelease( - buildRoot: string, - commitSha: string, - releasesRoot: string -): Promise { - const layout = await ensureDashboardReleaseLayout(releasesRoot); - const finalPath = managedReleasePath(releasesRoot, commitSha); - try { - return await loadManagedRelease(releasesRoot, commitSha); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - - const manifest = await loadReleaseManifest(buildRoot); - await verifyReleaseArtifacts(buildRoot, manifest); - await verifyReleaseBuildIdentities(buildRoot, manifest); - if (manifest.commitSha !== commitSha) { - throw new Error( - `Built release identity ${manifest.commitSha} does not match ${commitSha}` - ); - } - - const stagingPath = path.join( - layout.releasesPath, - `.staging-${commitSha}-${randomUUID()}` - ); - await fsp.mkdir(stagingPath, { mode: 0o755 }); - try { - const files = [ - ...manifest.artifacts.map((artifact) => artifact.path), - RELEASE_MANIFEST_FILE_NAME, - ]; - const createdDirectories = new Set([stagingPath]); - for (const relativePath of files) { - const sourcePath = path.join(buildRoot, relativePath); - const destinationPath = path.join(stagingPath, relativePath); - const destinationDirectory = path.dirname(destinationPath); - await fsp.mkdir(destinationDirectory, { mode: 0o755, recursive: true }); - for ( - let directory = destinationDirectory; - directory.startsWith(`${stagingPath}${path.sep}`); - directory = path.dirname(directory) - ) { - createdDirectories.add(directory); - } - await fsp.copyFile(sourcePath, destinationPath, fs.constants.COPYFILE_EXCL); - await syncFile(destinationPath); - } - - const stagedManifest = await loadReleaseManifest(stagingPath); - await verifyReleaseArtifacts(stagingPath, stagedManifest); - await verifyReleaseBuildIdentities(stagingPath, stagedManifest); - if (stagedManifest.commitSha !== commitSha) { - throw new Error( - `Staged release identity ${stagedManifest.commitSha} does not match ${commitSha}` - ); - } - const deepestFirst = [...createdDirectories].toSorted( - (left, right) => right.length - left.length - ); - for (const directory of deepestFirst) { - await syncDirectory(directory); - } - try { - await fsp.rename(stagingPath, finalPath); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "EEXIST" && code !== "ENOTEMPTY") { - throw error; - } - // A concurrent publisher may have won the same immutable SHA. - // Accept it only after full manifest, artifact, and identity verification. - const concurrentlyPublished = await loadManagedRelease( - releasesRoot, - commitSha - ); - await fsp.rm(stagingPath, { recursive: true }); - await syncDirectory(layout.releasesPath); - return concurrentlyPublished; - } - await syncDirectory(layout.releasesPath); - } catch (error) { - await fsp.rm(stagingPath, { force: true, recursive: true }); - throw error; - } - return loadManagedRelease(releasesRoot, commitSha); -} - export function managedDashboardUnitContract( releasesRoot = resolveDashboardReleasesRoot(), databasePath = process.env.MIRA_DASHBOARD_DB_PATH ?? DEFAULT_DASHBOARD_DATABASE_PATH, @@ -506,7 +387,7 @@ export async function stageDashboardRelease( timeoutMs: 12 * 60 * 1000, }); options.onProgress?.("Publishing verified immutable release"); - stagedRelease = await copyVerifiedRelease( + stagedRelease = await publishVerifiedDashboardRelease( worktreePath, expectedCommit, contract.releasesRoot diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index a9f04d457..d70f7e424 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -18,6 +18,7 @@ import { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY, type DashboardReleaseManifest, loadReleaseManifest, + RELEASE_MANIFEST_FILE_NAME, verifyReleaseArtifacts, verifyReleaseBuildIdentities, } from "./releaseManifest.ts"; @@ -35,6 +36,8 @@ const RETIRED_RELEASE_DIRECTORY_PATTERN = const STAGING_RELEASE_DIRECTORY_PATTERN = /^\.staging-([\da-f]{40})-[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; const STALE_STAGING_RELEASE_AGE_MS = 24 * 60 * 60 * 1000; +const RELEASE_PUBLICATION_LOCK_WAIT_MS = 2 * 60 * 1000; +const RELEASE_TRANSITION_LOCK_RETRY_MS = 50; const MAX_RELEASE_TRANSITION_FILE_BYTES = 4096; export const RELEASE_TRANSITION_LOCK_PROGRAM = "/usr/bin/flock"; @@ -132,6 +135,15 @@ async function syncDirectory(directoryPath: string): Promise { } } +async function syncFile(filePath: string): Promise { + const file = await fsp.open(filePath, fs.constants.O_RDONLY); + try { + await file.sync(); + } finally { + await file.close(); + } +} + function isPlainRecord(value: unknown): value is Record { if (!value || typeof value !== "object" || Array.isArray(value)) { return false; @@ -957,41 +969,50 @@ export function assertReleaseTransitionLockCommandSucceeded( async function acquireReleaseTransitionLock( layout: DashboardReleaseLayout, - lockMode: "exclusive" | "shared" + lockMode: "exclusive" | "shared", + waitTimeoutMs = 0 ): Promise { - const lockFile = await openReleaseTransitionLockFile(layout); - const result = spawnSync( - RELEASE_TRANSITION_LOCK_PROGRAM, - [ - lockMode === "exclusive" ? "--exclusive" : "--shared", - "--nonblock", - "--conflict-exit-code", - "75", - "3", - ], - { - stdio: ["ignore", "ignore", "pipe", lockFile.fd], + const deadline = Date.now() + waitTimeoutMs; + while (true) { + const lockFile = await openReleaseTransitionLockFile(layout); + const result = spawnSync( + RELEASE_TRANSITION_LOCK_PROGRAM, + [ + lockMode === "exclusive" ? "--exclusive" : "--shared", + "--nonblock", + "--conflict-exit-code", + "75", + "3", + ], + { + stdio: ["ignore", "ignore", "pipe", lockFile.fd], + } + ); + if (result.status === 0 && !result.error) { + return lockFile; + } + await lockFile.close(); + if (result.status === 75 && Date.now() < deadline) { + await Bun.sleep( + Math.min(RELEASE_TRANSITION_LOCK_RETRY_MS, deadline - Date.now()) + ); + continue; } - ); - try { assertReleaseTransitionLockCommandSucceeded( result.error as NodeJS.ErrnoException | undefined, result.status, result.stderr?.toString("utf8") ?? "" ); - } catch (error) { - await lockFile.close(); - throw error; } - return lockFile; } async function withReleaseTransitionLock( layout: DashboardReleaseLayout, lockMode: "exclusive" | "shared", - transition: () => Promise + transition: () => Promise, + waitTimeoutMs = 0 ): Promise { - const lockFile = await acquireReleaseTransitionLock(layout, lockMode); + const lockFile = await acquireReleaseTransitionLock(layout, lockMode, waitTimeoutMs); let result: T | undefined; let transitionError: unknown; try { @@ -1017,6 +1038,115 @@ async function withReleaseTransitionLock( return result as T; } +/** + * Copies a verified build into the immutable release store while excluding + * activation, rollback, pruning, and another publisher from its staging path. + */ +export async function publishVerifiedDashboardRelease( + buildRoot: string, + commitSha: string, + releasesRoot = resolveDashboardReleasesRoot() +): Promise { + assertReleaseCommitSha(commitSha); + const layout = await ensureDashboardReleaseLayout(releasesRoot); + const manifest = await loadReleaseManifest(buildRoot); + await verifyReleaseArtifacts(buildRoot, manifest); + await verifyReleaseBuildIdentities(buildRoot, manifest); + if (manifest.commitSha !== commitSha) { + throw new Error( + `Built release identity ${manifest.commitSha} does not match ${commitSha}` + ); + } + + return withReleaseTransitionLock( + layout, + "exclusive", + async () => { + await recoverInterruptedReleaseTransition(layout); + try { + return await loadManagedReleaseFromLayout(layout, commitSha); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + + const finalPath = path.join(layout.releasesPath, commitSha); + const stagingPath = path.join( + layout.releasesPath, + `.staging-${commitSha}-${randomUUID()}` + ); + await fsp.mkdir(stagingPath, { mode: 0o755 }); + try { + const files = [ + ...manifest.artifacts.map((artifact) => artifact.path), + RELEASE_MANIFEST_FILE_NAME, + ]; + const createdDirectories = new Set([stagingPath]); + for (const relativePath of files) { + const sourcePath = path.join(buildRoot, relativePath); + const destinationPath = path.join(stagingPath, relativePath); + const destinationDirectory = path.dirname(destinationPath); + await fsp.mkdir(destinationDirectory, { + mode: 0o755, + recursive: true, + }); + for ( + let directory = destinationDirectory; + directory.startsWith(`${stagingPath}${path.sep}`); + directory = path.dirname(directory) + ) { + createdDirectories.add(directory); + } + await fsp.copyFile( + sourcePath, + destinationPath, + fs.constants.COPYFILE_EXCL + ); + await syncFile(destinationPath); + } + + const stagedManifest = await loadReleaseManifest(stagingPath); + await verifyReleaseArtifacts(stagingPath, stagedManifest); + await verifyReleaseBuildIdentities(stagingPath, stagedManifest); + if (stagedManifest.commitSha !== commitSha) { + throw new Error( + `Staged release identity ${stagedManifest.commitSha} does not match ${commitSha}` + ); + } + const deepestFirst = [...createdDirectories].toSorted( + (left, right) => right.length - left.length + ); + for (const directory of deepestFirst) { + await syncDirectory(directory); + } + try { + await fsp.rename(stagingPath, finalPath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST" && code !== "ENOTEMPTY") { + throw error; + } + // A publisher from an older process may have won the same SHA. + const concurrentlyPublished = await loadManagedReleaseFromLayout( + layout, + commitSha + ); + await fsp.rm(stagingPath, { recursive: true }); + await syncDirectory(layout.releasesPath); + return concurrentlyPublished; + } + await syncDirectory(layout.releasesPath); + } catch (error) { + await fsp.rm(stagingPath, { force: true, recursive: true }); + throw error; + } + return loadManagedReleaseFromLayout(layout, commitSha); + }, + RELEASE_PUBLICATION_LOCK_WAIT_MS + ); +} + async function executeReleaseTransition( layout: DashboardReleaseLayout, journal: ReleaseTransitionJournal, diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index f18d5391c..5c7e44b0d 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -32,6 +32,7 @@ import { loadManagedRelease, managedReleasePath, pruneDashboardReleases, + publishVerifiedDashboardRelease, readDashboardReleaseState, RELEASE_TRANSITION_LOCK_FILE_NAME, RELEASE_TRANSITION_LOCK_PROGRAM, @@ -45,6 +46,7 @@ import { RELEASE_MANIFEST_FILE_NAME, writeReleaseManifest, } from "../src/releaseManifest.ts"; +import { createReleaseFixture } from "./support/releaseFixture.ts"; const temporaryRoots: string[] = []; const FIRST_COMMIT = "a".repeat(40); @@ -278,6 +280,32 @@ describe("Dashboard immutable release manager", () => { ); }); + it("does not expose active staging paths before publication owns the transition lock", async () => { + const releasesRoot = temporaryReleasesRoot(); + const buildRoot = temporaryReleasesRoot(); + await ensureDashboardReleaseLayout(releasesRoot); + await createReleaseFixture(buildRoot, FIRST_COMMIT); + await readDashboardReleaseState(releasesRoot); + const lockFileDescriptor = holdTransitionLock(releasesRoot); + const publication = publishVerifiedDashboardRelease( + buildRoot, + FIRST_COMMIT, + releasesRoot + ); + try { + await Bun.sleep(100); + expect( + readdirSync(path.join(releasesRoot, "releases")).filter((entry) => + entry.startsWith(".staging-") + ) + ).toEqual([]); + } finally { + closeSync(lockFileDescriptor); + } + const release = await publication; + expect(release.commitSha).toBe(FIRST_COMMIT); + }); + it("activates and rolls back verified releases through relative atomic links", async () => { const root = temporaryReleasesRoot(); await createManagedRelease(root, FIRST_COMMIT); From 178686dba2e0480138a2c280a0723050b2b4c210 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 10:04:46 +0200 Subject: [PATCH 09/11] fix: harden atomic cutover recovery --- backend/src/services/pullRequests.ts | 12 ++- backend/src/services/scheduledJobs.ts | 48 +++++++++-- backend/test/jobExecutionQueue.test.ts | 105 ++++++++++++++++++++++++- backend/test/serviceBehavior.test.ts | 13 +++ docs/setup/production-deploy.md | 68 +++++++++++++--- 5 files changed, 226 insertions(+), 20 deletions(-) diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 778c4fbf6..41b873ece 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -1690,6 +1690,12 @@ function didScheduleOrphanedReleaseCutoverRecovery( updatedAt: dateToISOString(new Date()), note: "Interrupted release cutover recovered before candidate activation; current verified release remains ready", }; + const activeCandidateRecoveredJob: DeploymentJob = { + ...job, + status: "isOk", + updatedAt: dateToISOString(new Date()), + note: "Interrupted release cutover recovered; active candidate passed restart and commit-bound readiness", + }; const script = [ "sleep 1", ...releaseCutoverShellFunctions(), @@ -1718,7 +1724,11 @@ function didScheduleOrphanedReleaseCutoverRecovery( " NODE_ENV=production \\", ' "$bun_executable" "$trusted_lifecycle" "$@"', "}", - "resolve_trusted_lifecycle", + "resolve_trusted_lifecycle || exit 1", + 'if [ "$current_commit" = "$candidate_commit" ] && restart_services && ready_for_commit "${candidate_commit:0:8}"; then', + ` ${deploymentJobUpdateCommand(activeCandidateRecoveredJob)}`, + " exit 0", + "fi", 'if activation_output="$(run_lifecycle activate "$candidate_commit")"; then', ' rollback_commit="$(printf "%s" "$activation_output" | /usr/bin/jq --raw-output \'.previous.commitSha // empty\')"', ' [[ "$rollback_commit" =~ ^[0-9a-f]{40}$ ]] || exit 1', diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index f3099c446..a6e131451 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -67,6 +67,7 @@ const scheduledJobRuntimeState: { isSchedulerTickRunning: boolean; isExecutorClaimingPaused: boolean; isExecutorTickRunning: boolean; + missingCutoverRecoveryWarningKey: string | undefined; nextDeploymentCutoverReconcileAt: number; workerId: string; } = { @@ -78,6 +79,7 @@ const scheduledJobRuntimeState: { isSchedulerTickRunning: false, isExecutorClaimingPaused: false, isExecutorTickRunning: false, + missingCutoverRecoveryWarningKey: undefined, nextDeploymentCutoverReconcileAt: 0, workerId: "", }; @@ -1268,6 +1270,7 @@ function pauseExecutorClaims(): () => void { function resetExecutorClaimPause(): void { scheduledJobRuntimeState.executorClaimPauseGeneration += 1; scheduledJobRuntimeState.isExecutorClaimingPaused = false; + scheduledJobRuntimeState.missingCutoverRecoveryWarningKey = undefined; scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt = 0; } @@ -1290,9 +1293,20 @@ function readSystemdUnitState(unit: string): SystemdUnitState { stdout: "pipe", }); const stderr = new TextDecoder().decode(result.stderr).trim(); - if (stderr || result.exitCode !== 0) { + if (result.exitCode !== 0) { + console.warn("[ScheduledJobs] systemctl show failed", { + exitCode: result.exitCode, + stderr, + unit, + }); return "unknown"; } + if (stderr) { + console.warn("[ScheduledJobs] systemctl show reported diagnostics", { + stderr, + unit, + }); + } const properties = new Map( new TextDecoder() .decode(result.stdout) @@ -1356,10 +1370,14 @@ function isDeploymentCutoverReconciliationExpired( export function reconcileOrphanedDeploymentCutovers( timestamp = nowIso(), readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState, - recoverCutover: - | DeploymentCutoverRecoveryHandler - | undefined = scheduledJobRuntimeState.deploymentCutoverRecoveryHandler + ...recoveryHandlerOverride: [ + recoverCutover?: DeploymentCutoverRecoveryHandler | undefined, + ] ): number { + const recoverCutover = + recoveryHandlerOverride.length === 0 + ? scheduledJobRuntimeState.deploymentCutoverRecoveryHandler + : recoveryHandlerOverride[0]; const pendingRows = database .query( `SELECT id, commit_sha AS candidateCommit, updated_at AS updatedAt @@ -1377,6 +1395,7 @@ export function reconcileOrphanedDeploymentCutovers( updatedAt: row.updatedAt, })); let recovered = 0; + const cutoversMissingRecoveryHandler: string[] = []; for (const cutover of pending) { let state: DeploymentGuardianState = "unknown"; try { @@ -1391,7 +1410,11 @@ export function reconcileOrphanedDeploymentCutovers( state === "inactive" || (state === "unknown" && isDeploymentCutoverReconciliationExpired(cutover.updatedAt, timestamp)); - if (!shouldRecover || !recoverCutover) { + if (!shouldRecover) { + continue; + } + if (!recoverCutover) { + cutoversMissingRecoveryHandler.push(cutover.id); continue; } try { @@ -1405,6 +1428,20 @@ export function reconcileOrphanedDeploymentCutovers( ); } } + const sortedMissingHandlerIds = cutoversMissingRecoveryHandler.toSorted( + (left, right) => left.localeCompare(right) + ); + const warningKey = sortedMissingHandlerIds.join(","); + if ( + warningKey && + scheduledJobRuntimeState.missingCutoverRecoveryWarningKey !== warningKey + ) { + console.warn( + "[ScheduledJobs] Cannot recover orphaned deployment cutovers because no recovery handler is registered", + { cutoverIds: sortedMissingHandlerIds } + ); + } + scheduledJobRuntimeState.missingCutoverRecoveryWarningKey = warningKey || undefined; return recovered; } @@ -1413,6 +1450,7 @@ export function registerDeploymentCutoverRecoveryHandler( didScheduleRecovery: DeploymentCutoverRecoveryHandler ): void { scheduledJobRuntimeState.deploymentCutoverRecoveryHandler = didScheduleRecovery; + scheduledJobRuntimeState.missingCutoverRecoveryWarningKey = undefined; } function hasPendingDeploymentCutover(): boolean { diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 40cbd1edc..ba851a4f7 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -1,3 +1,7 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + import { afterEach, describe, expect, it, jest } from "bun:test"; import { database } from "../src/database.ts"; @@ -133,7 +137,63 @@ describe("persistent job execution queue", () => { ).toEqual({ job_id: deploymentId }); }); - it("bounds unknown guardian inspection by scheduling rollback recovery", () => { + it("warns once when an orphaned cutover has no recovery handler", () => { + const startedAt = "2026-07-26T03:00:00.000Z"; + const deploymentId = createRestartScheduledDeployment(startedAt); + const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); + try { + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:11:00.000Z", + () => "inactive", + undefined + ) + ).toBe(0); + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:12:00.000Z", + () => "inactive", + undefined + ) + ).toBe(0); + expect(warning).toHaveBeenCalledTimes(1); + expect(warning).toHaveBeenCalledWith( + "[ScheduledJobs] Cannot recover orphaned deployment cutovers because no recovery handler is registered", + { cutoverIds: [deploymentId] } + ); + } finally { + warning.mockRestore(); + } + }); + + it("bounds an explicit unknown guardian state by scheduling recovery", () => { + const startedAt = "2026-07-26T03:00:00.000Z"; + const deploymentId = createRestartScheduledDeployment(startedAt); + const recovery = jest.fn(() => true); + + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:01:00.000Z", + () => "unknown", + recovery + ) + ).toBe(0); + expect(recovery).not.toHaveBeenCalled(); + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:11:00.000Z", + () => "unknown", + recovery + ) + ).toBe(1); + expect(recovery).toHaveBeenCalledWith({ + candidateCommit: "c".repeat(40), + id: deploymentId, + updatedAt: startedAt, + }); + }); + + it("bounds guardian inspection failures by scheduling rollback recovery", () => { const startedAt = "2026-07-26T03:00:00.000Z"; const deploymentId = createRestartScheduledDeployment(startedAt); const recovery = jest.fn(() => true); @@ -189,6 +249,49 @@ describe("persistent job execution queue", () => { ).toEqual({ job_id: deploymentId }); }); + it("accepts a loaded active unit when systemctl emits benign diagnostics", () => { + const startedAt = "2026-07-26T03:00:00.000Z"; + createRestartScheduledDeployment(startedAt); + const fakeBin = mkdtempSync(path.join(tmpdir(), "mira-systemctl-test-")); + const systemctl = path.join(fakeBin, "systemctl"); + const originalPath = process.env.PATH; + writeFileSync( + systemctl, + String.raw`#!/usr/bin/env bash +printf 'benign diagnostic\n' >&2 +printf 'LoadState=loaded\nActiveState=active\n' +` + ); + chmodSync(systemctl, 0o755); + process.env.PATH = `${fakeBin}${path.delimiter}${originalPath ?? ""}`; + const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); + const recovery = jest.fn(() => true); + try { + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:11:00.000Z", + undefined, + recovery + ) + ).toBe(0); + expect(recovery).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledWith( + "[ScheduledJobs] systemctl show reported diagnostics", + expect.objectContaining({ + stderr: "benign diagnostic", + }) + ); + } finally { + warning.mockRestore(); + if (originalPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = originalPath; + } + rmSync(fakeBin, { force: true, recursive: true }); + } + }); + it("persists worker progress and structured action failures", async () => { const actionKey = `test.worker-${Bun.randomUUIDv7()}`; registerScheduledJobAction(actionKey, async (_job, _signal, context) => { diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 15638078c..64883a757 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -2099,6 +2099,19 @@ printf 'scheduled\n' expect(recoveryCommand).toContain( 'trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous")' ); + expect(recoveryCommand).toContain( + 'if [ "$current_commit" = "$candidate_commit" ] && restart_services && ready_for_commit "${candidate_commit:0:8}"; then' + ); + expect(recoveryCommand).toContain( + "Interrupted release cutover recovered; active candidate passed restart and commit-bound readiness" + ); + expect( + recoveryCommand.indexOf('if [ "$current_commit" = "$candidate_commit" ]') + ).toBeLessThan( + recoveryCommand.indexOf( + 'activation_output="$(run_lifecycle activate "$candidate_commit")"' + ) + ); expect(recoveryCommand).toContain( "run_lifecycle rollback && restart_services" ); diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 93cec5b00..6020bc96e 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -194,24 +194,56 @@ systemctl --user restart mira-dashboard-worker.service mira-dashboard.service Use a commit-bound readiness function; exhausting the loop is a failure: ```bash +worker_identity() { + local properties active substate pid started + properties="$( + systemctl --user show mira-dashboard-worker.service \ + --property=ActiveState \ + --property=SubState \ + --property=MainPID \ + --property=ExecMainStartTimestampMonotonic \ + --no-pager 2>/dev/null + )" || return 1 + active="$(sed -n 's/^ActiveState=//p' <<<"$properties")" + substate="$(sed -n 's/^SubState=//p' <<<"$properties")" + pid="$(sed -n 's/^MainPID=//p' <<<"$properties")" + started="$(sed -n 's/^ExecMainStartTimestampMonotonic=//p' <<<"$properties")" + [[ "$active" == active && "$substate" == running ]] || return 1 + [[ "$pid:$started" =~ ^[1-9][0-9]*:[1-9][0-9]*$ ]] || return 1 + printf '%s:%s' "$pid" "$started" +} + +readiness_matches() { + local expected="$1" + local response + response="$(curl --fail --silent --show-error \ + --connect-timeout 2 --max-time 5 \ + "http://127.0.0.1:${DASHBOARD_PORT}/api/health/ready" || true)" + jq --exit-status --arg expected "$expected" \ + '.status == "isReady" + and .checks.release.ready == true + and .checks.release.backendCommit == $expected + and .checks.release.frontendCommit == $expected + and .checks.worker.ready == true' <<<"$response" >/dev/null +} + ready_for_commit() { local full_sha="$1" local expected="${full_sha:0:8}" - local response + local initial_worker_identity current_worker_identity + initial_worker_identity="" for attempt in {1..30}; do - response="$(curl --fail --silent --show-error \ - --connect-timeout 2 --max-time 5 \ - "http://127.0.0.1:${DASHBOARD_PORT}/api/health/ready" || true)" - if jq --exit-status --arg expected "$expected" \ - '.status == "isReady" - and .checks.release.backendCommit == $expected - and .checks.release.frontendCommit == $expected - and .checks.worker.ready == true' <<<"$response" >/dev/null; then - return 0 + if readiness_matches "$expected"; then + initial_worker_identity="$(worker_identity || true)" + [[ -n "$initial_worker_identity" ]] && break fi sleep 1 done - return 1 + [[ -n "$initial_worker_identity" ]] || return 1 + sleep 31 + current_worker_identity="$(worker_identity || true)" + [[ "$current_worker_identity" == "$initial_worker_identity" ]] || return 1 + readiness_matches "$expected" } recover_legacy_deployment() { @@ -313,16 +345,26 @@ Normal activation automatically rolls back on restart or commit-bound readiness failure. Manual rollback is a failure-only operation: ```bash +set -euo pipefail RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases DATABASE_PATH=/home/ubuntu/projects/mira-dashboard-state/mira-dashboard.db +PREVIOUS_RELEASE="$( + readlink --canonicalize-existing "$RELEASES_ROOT/previous" +)" +PREVIOUS_SHA="$(basename -- "$PREVIOUS_RELEASE")" +[[ "$PREVIOUS_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$PREVIOUS_RELEASE" == "$RELEASES_ROOT/releases/$PREVIOUS_SHA" ]] +PREVIOUS_LIFECYCLE="$PREVIOUS_RELEASE/backend/dist/releaseLifecycle.js" +test -f "$PREVIOUS_LIFECYCLE" +test ! -L "$PREVIOUS_LIFECYCLE" env MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ NODE_ENV=production \ - bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" status + bun "$PREVIOUS_LIFECYCLE" status env MIRA_DASHBOARD_DB_PATH="$DATABASE_PATH" \ MIRA_DASHBOARD_RELEASES_ROOT="$RELEASES_ROOT" \ NODE_ENV=production \ - bun "$RELEASES_ROOT/current/backend/dist/releaseLifecycle.js" rollback + bun "$PREVIOUS_LIFECYCLE" rollback systemctl --user restart mira-dashboard-worker.service mira-dashboard.service curl --fail --silent --show-error \ "http://127.0.0.1:${DASHBOARD_PORT:-3100}/api/health/ready" | jq From c105e62513ab6088830a95240e9a9a8d0d51d201 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 10:48:37 +0200 Subject: [PATCH 10/11] fix: close atomic cutover recovery gaps --- backend/src/releaseManager.ts | 32 +++++++--- backend/src/services/pullRequests.ts | 20 ++++-- backend/src/services/scheduledJobs.ts | 64 ++++++++++++++++--- backend/test/jobExecutionQueue.test.ts | 87 +++++++++++++++++++++----- backend/test/releaseManager.test.ts | 72 ++++++++++++++++++++- backend/test/serviceBehavior.test.ts | 18 +++++- 6 files changed, 254 insertions(+), 39 deletions(-) diff --git a/backend/src/releaseManager.ts b/backend/src/releaseManager.ts index d70f7e424..46da87314 100644 --- a/backend/src/releaseManager.ts +++ b/backend/src/releaseManager.ts @@ -75,6 +75,10 @@ export interface DashboardReleaseManagerOptions { schemaCutoverMode?: "coordinated"; } +export interface DashboardReleasePublicationOptions { + onTransitionLockContention?: () => void; +} + export interface DashboardLiveSchemaState { migrations: DatabaseMigrationIdentity[]; version: number; @@ -683,7 +687,9 @@ export function assertReleaseCanOpenLiveSchema( } } -function assertHostRuntimeCompatible(release: ManagedDashboardRelease): void { +export function assertDashboardReleaseHostRuntimeCompatible( + release: ManagedDashboardRelease +): void { if (release.manifest.bunVersion !== Bun.version) { throw new Error( `Release ${release.commitSha} requires Bun ${release.manifest.bunVersion}; host runs ${Bun.version}` @@ -970,7 +976,8 @@ export function assertReleaseTransitionLockCommandSucceeded( async function acquireReleaseTransitionLock( layout: DashboardReleaseLayout, lockMode: "exclusive" | "shared", - waitTimeoutMs = 0 + waitTimeoutMs = 0, + onContention?: () => void ): Promise { const deadline = Date.now() + waitTimeoutMs; while (true) { @@ -993,6 +1000,7 @@ async function acquireReleaseTransitionLock( } await lockFile.close(); if (result.status === 75 && Date.now() < deadline) { + onContention?.(); await Bun.sleep( Math.min(RELEASE_TRANSITION_LOCK_RETRY_MS, deadline - Date.now()) ); @@ -1010,9 +1018,15 @@ async function withReleaseTransitionLock( layout: DashboardReleaseLayout, lockMode: "exclusive" | "shared", transition: () => Promise, - waitTimeoutMs = 0 + waitTimeoutMs = 0, + onContention?: () => void ): Promise { - const lockFile = await acquireReleaseTransitionLock(layout, lockMode, waitTimeoutMs); + const lockFile = await acquireReleaseTransitionLock( + layout, + lockMode, + waitTimeoutMs, + onContention + ); let result: T | undefined; let transitionError: unknown; try { @@ -1045,7 +1059,8 @@ async function withReleaseTransitionLock( export async function publishVerifiedDashboardRelease( buildRoot: string, commitSha: string, - releasesRoot = resolveDashboardReleasesRoot() + releasesRoot = resolveDashboardReleasesRoot(), + options: DashboardReleasePublicationOptions = {} ): Promise { assertReleaseCommitSha(commitSha); const layout = await ensureDashboardReleaseLayout(releasesRoot); @@ -1143,7 +1158,8 @@ export async function publishVerifiedDashboardRelease( } return loadManagedReleaseFromLayout(layout, commitSha); }, - RELEASE_PUBLICATION_LOCK_WAIT_MS + RELEASE_PUBLICATION_LOCK_WAIT_MS, + options.onTransitionLockContention ); } @@ -1200,7 +1216,7 @@ export async function activateDashboardRelease( return withReleaseTransitionLock(layout, "exclusive", async () => { await recoverInterruptedReleaseTransition(layout); const candidate = await loadManagedReleaseFromLayout(layout, commitSha); - assertHostRuntimeCompatible(candidate); + assertDashboardReleaseHostRuntimeCompatible(candidate); const state = await readActivationReleaseStateFromLayout(layout); if (state.current) { assertReleaseActivationCompatible( @@ -1284,7 +1300,7 @@ export async function rollbackDashboardRelease( const activeRelease = state.current; const rollbackRelease = state.previous; - assertHostRuntimeCompatible(rollbackRelease); + assertDashboardReleaseHostRuntimeCompatible(rollbackRelease); const maximumInspectableSchemaVersion = Math.max( DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum, activeRelease.manifest.schema.maximumCompatible, diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 41b873ece..90237689e 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -16,6 +16,7 @@ import { stageDashboardRelease, } from "../releaseDeployment.ts"; import { + assertDashboardReleaseHostRuntimeCompatible, readDashboardReleaseState, resolveDashboardReleasesRoot, } from "../releaseManager.ts"; @@ -1703,12 +1704,18 @@ function didScheduleOrphanedReleaseCutoverRecovery( `candidate_commit=${shellQuote(candidateCommit)}`, `bun_executable=${shellQuote(resolveBunExecutable())}`, "resolve_trusted_lifecycle() {", + ' candidate_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/releases/$candidate_commit") || return 1', + ' [ "$candidate_release" = "$releases_root/releases/$candidate_commit" ] || return 1', ' current_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/current") || return 1', ' current_commit="$(/usr/bin/basename -- "$current_release")"', ' [[ "$current_commit" =~ ^[0-9a-f]{40}$ ]] || return 1', ' [ "$current_release" = "$releases_root/releases/$current_commit" ] || return 1', ' if [ "$current_commit" = "$candidate_commit" ]; then', - ' trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous") || return 1', + ' if [ -e "$releases_root/previous" ] || [ -L "$releases_root/previous" ]; then', + ' trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous") || return 1', + " else", + ' trusted_release="$candidate_release"', + " fi", " else", ' trusted_release="$current_release"', " fi", @@ -1725,11 +1732,13 @@ function didScheduleOrphanedReleaseCutoverRecovery( ' "$bun_executable" "$trusted_lifecycle" "$@"', "}", "resolve_trusted_lifecycle || exit 1", - 'if [ "$current_commit" = "$candidate_commit" ] && restart_services && ready_for_commit "${candidate_commit:0:8}"; then', - ` ${deploymentJobUpdateCommand(activeCandidateRecoveredJob)}`, - " exit 0", - "fi", 'if activation_output="$(run_lifecycle activate "$candidate_commit")"; then', + ' activation_commit="$(printf "%s" "$activation_output" | /usr/bin/jq --raw-output \'.current.commitSha // empty\')"', + ' [ "$activation_commit" = "$candidate_commit" ] || exit 1', + ' if restart_services && ready_for_commit "${candidate_commit:0:8}"; then', + ` ${deploymentJobUpdateCommand(activeCandidateRecoveredJob)}`, + " exit 0", + " fi", ' rollback_commit="$(printf "%s" "$activation_output" | /usr/bin/jq --raw-output \'.previous.commitSha // empty\')"', ' [[ "$rollback_commit" =~ ^[0-9a-f]{40}$ ]] || exit 1', ' [ "$rollback_commit" != "$candidate_commit" ] || exit 1', @@ -1839,6 +1848,7 @@ async function runDeploymentJob( "Managed deployment requires a distinct verified rollback release" ); } + assertDashboardReleaseHostRuntimeCompatible(rollbackRelease); const restartScheduled: DeploymentJob = { ...currentJob, diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index a6e131451..e5d7eb913 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -37,6 +37,7 @@ const deploymentCutoverReconcileIntervalMs = 5000; const deploymentCutoverMaximumUnknownMs = 10 * 60 * 1000; const interruptedHandlerGraceMs = 30_000; const RELEASE_COMMIT_PATTERN = /^(?:[\da-f]{8,40}|development)$/u; +const FULL_RELEASE_COMMIT_PATTERN = /^[\da-f]{40}$/u; const DEPLOYMENT_GUARDIAN_UNIT_PREFIX = "mira-dashboard-deploy-"; const DEPLOYMENT_RECOVERY_UNIT_PREFIX = "mira-dashboard-deploy-recovery-"; const actionHandlers = new Map(); @@ -1367,6 +1368,46 @@ function isDeploymentCutoverReconciliationExpired( ); } +function didTerminalizeUnrecoverableDeploymentCutover( + cutover: OrphanedDeploymentCutover, + timestamp: string +): boolean { + const terminalize = database.transaction(() => { + const result = database + .prepare( + `UPDATE deployment_jobs + SET status = 'failed', + updated_at = ?, + note = ? + WHERE id = ? + AND status = 'restart-scheduled'` + ) + .run( + timestamp, + "Interrupted legacy deployment cutover cannot be recovered because it lacks a persisted full candidate SHA", + cutover.id + ); + if (result.changes === 0) { + return false; + } + database + .prepare("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") + .run(cutover.id); + return true; + }); + const didTerminalize = terminalize(); + if (didTerminalize) { + console.warn( + "[ScheduledJobs] Terminalized unrecoverable legacy deployment cutover", + { + candidateCommit: cutover.candidateCommit, + cutoverId: cutover.id, + } + ); + } + return didTerminalize; +} + export function reconcileOrphanedDeploymentCutovers( timestamp = nowIso(), readGuardianState: DeploymentGuardianStateReader = readDeploymentGuardianState, @@ -1394,7 +1435,7 @@ export function reconcileOrphanedDeploymentCutovers( id: row.id, updatedAt: row.updatedAt, })); - let recovered = 0; + let reconciled = 0; const cutoversMissingRecoveryHandler: string[] = []; for (const cutover of pending) { let state: DeploymentGuardianState = "unknown"; @@ -1413,13 +1454,22 @@ export function reconcileOrphanedDeploymentCutovers( if (!shouldRecover) { continue; } + if ( + !cutover.candidateCommit || + !FULL_RELEASE_COMMIT_PATTERN.test(cutover.candidateCommit) + ) { + if (didTerminalizeUnrecoverableDeploymentCutover(cutover, timestamp)) { + reconciled += 1; + } + continue; + } if (!recoverCutover) { cutoversMissingRecoveryHandler.push(cutover.id); continue; } try { if (recoverCutover(cutover)) { - recovered += 1; + reconciled += 1; } } catch (error) { console.warn( @@ -1442,7 +1492,7 @@ export function reconcileOrphanedDeploymentCutovers( ); } scheduledJobRuntimeState.missingCutoverRecoveryWarningKey = warningKey || undefined; - return recovered; + return reconciled; } /** Registers the detached rollback scheduler used for orphaned release cutovers. */ @@ -1458,10 +1508,10 @@ function hasPendingDeploymentCutover(): boolean { if (now >= scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt) { scheduledJobRuntimeState.nextDeploymentCutoverReconcileAt = now + deploymentCutoverReconcileIntervalMs; - const recovered = reconcileOrphanedDeploymentCutovers(); - if (recovered > 0) { - console.warn("[ScheduledJobs] Scheduled orphaned deployment rollbacks", { - scheduled: recovered, + const reconciled = reconcileOrphanedDeploymentCutovers(); + if (reconciled > 0) { + console.warn("[ScheduledJobs] Reconciled orphaned deployment cutovers", { + reconciled, }); } } diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index ba851a4f7..b5a2e75ba 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -81,9 +81,11 @@ function createScheduledTestJob( return id; } -function createRestartScheduledDeployment(updatedAt: string): string { +function createRestartScheduledDeployment( + updatedAt: string, + candidateCommit = "c".repeat(40) +): string { const deploymentId = `test-orphaned-cutover-${Bun.randomUUIDv7()}`; - const candidateCommit = "c".repeat(40); testDeploymentIds.add(deploymentId); database .prepare( @@ -166,6 +168,52 @@ describe("persistent job execution queue", () => { } }); + it("terminalizes an inactive legacy cutover without a persisted full SHA", () => { + const startedAt = "2026-07-26T03:00:00.000Z"; + const deploymentId = createRestartScheduledDeployment(startedAt, "c0ffee12"); + const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); + const recovery = jest.fn(() => true); + try { + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:01:00.000Z", + () => "inactive", + recovery + ) + ).toBe(1); + expect(recovery).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledWith( + "[ScheduledJobs] Terminalized unrecoverable legacy deployment cutover", + { + candidateCommit: "c0ffee12", + cutoverId: deploymentId, + } + ); + } finally { + warning.mockRestore(); + } + expect( + database + .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") + .get(deploymentId) + ).toEqual({ + note: "Interrupted legacy deployment cutover cannot be recovered because it lacks a persisted full candidate SHA", + status: "failed", + }); + expect( + database + .prepare("SELECT job_id FROM deployment_lock WHERE job_id = ?") + .get(deploymentId) + ).toBeNull(); + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:02:00.000Z", + () => "inactive", + recovery + ) + ).toBe(0); + }); + it("bounds an explicit unknown guardian state by scheduling recovery", () => { const startedAt = "2026-07-26T03:00:00.000Z"; const deploymentId = createRestartScheduledDeployment(startedAt); @@ -179,6 +227,13 @@ describe("persistent job execution queue", () => { ) ).toBe(0); expect(recovery).not.toHaveBeenCalled(); + expect( + reconcileOrphanedDeploymentCutovers( + "2026-07-26T03:10:00.000Z", + () => "unknown", + recovery + ) + ).toBe(1); expect( reconcileOrphanedDeploymentCutovers( "2026-07-26T03:11:00.000Z", @@ -191,6 +246,7 @@ describe("persistent job execution queue", () => { id: deploymentId, updatedAt: startedAt, }); + expect(recovery).toHaveBeenCalledTimes(2); }); it("bounds guardian inspection failures by scheduling rollback recovery", () => { @@ -252,21 +308,22 @@ describe("persistent job execution queue", () => { it("accepts a loaded active unit when systemctl emits benign diagnostics", () => { const startedAt = "2026-07-26T03:00:00.000Z"; createRestartScheduledDeployment(startedAt); - const fakeBin = mkdtempSync(path.join(tmpdir(), "mira-systemctl-test-")); - const systemctl = path.join(fakeBin, "systemctl"); const originalPath = process.env.PATH; - writeFileSync( - systemctl, - String.raw`#!/usr/bin/env bash + let fakeBin: string | undefined; + const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); + try { + fakeBin = mkdtempSync(path.join(tmpdir(), "mira-systemctl-test-")); + const systemctl = path.join(fakeBin, "systemctl"); + writeFileSync( + systemctl, + String.raw`#!/usr/bin/env bash printf 'benign diagnostic\n' >&2 printf 'LoadState=loaded\nActiveState=active\n' ` - ); - chmodSync(systemctl, 0o755); - process.env.PATH = `${fakeBin}${path.delimiter}${originalPath ?? ""}`; - const warning = jest.spyOn(console, "warn").mockImplementation(() => {}); - const recovery = jest.fn(() => true); - try { + ); + chmodSync(systemctl, 0o755); + process.env.PATH = `${fakeBin}${path.delimiter}${originalPath ?? ""}`; + const recovery = jest.fn(() => true); expect( reconcileOrphanedDeploymentCutovers( "2026-07-26T03:11:00.000Z", @@ -288,7 +345,9 @@ printf 'LoadState=loaded\nActiveState=active\n' } else { process.env.PATH = originalPath; } - rmSync(fakeBin, { force: true, recursive: true }); + if (fakeBin) { + rmSync(fakeBin, { force: true, recursive: true }); + } } }); diff --git a/backend/test/releaseManager.test.ts b/backend/test/releaseManager.test.ts index 5c7e44b0d..9d33eec54 100644 --- a/backend/test/releaseManager.test.ts +++ b/backend/test/releaseManager.test.ts @@ -26,6 +26,7 @@ import { import { runReleaseLifecycleCommand } from "../src/releaseLifecycle.ts"; import { activateDashboardRelease, + assertDashboardReleaseHostRuntimeCompatible, assertReleaseTransitionLockCommandSucceeded, ensureDashboardReleaseLayout, isReleaseTransitionLockAvailable, @@ -108,6 +109,19 @@ function holdTransitionLock(releasesRoot: string): number { return lockFileDescriptor; } +async function throwWhenPromiseSettles( + promise: Promise, + message: string +): Promise { + await promise; + throw new Error(message); +} + +async function throwAfterDelay(milliseconds: number, message: string): Promise { + await Bun.sleep(milliseconds); + throw new Error(message); +} + function temporaryReleasesRoot(): string { const root = mkdtempSync(path.join(tmpdir(), "mira-releases-")); temporaryRoots.push(root); @@ -287,13 +301,30 @@ describe("Dashboard immutable release manager", () => { await createReleaseFixture(buildRoot, FIRST_COMMIT); await readDashboardReleaseState(releasesRoot); const lockFileDescriptor = holdTransitionLock(releasesRoot); + const { promise: lockContention, resolve: didReachLockContention } = + Promise.withResolvers(); const publication = publishVerifiedDashboardRelease( buildRoot, FIRST_COMMIT, - releasesRoot + releasesRoot, + { + onTransitionLockContention: () => { + didReachLockContention(); + }, + } ); try { - await Bun.sleep(100); + await Promise.race([ + lockContention, + throwWhenPromiseSettles( + publication, + "Release publication completed before waiting for the held transition lock" + ), + throwAfterDelay( + 2000, + "Release publication did not reach transition-lock contention" + ), + ]); expect( readdirSync(path.join(releasesRoot, "releases")).filter((entry) => entry.startsWith(".staging-") @@ -552,6 +583,10 @@ describe("Dashboard immutable release manager", () => { const runtimeRoot = temporaryReleasesRoot(); await createManagedRelease(runtimeRoot, FIRST_COMMIT, FIRST_COMMIT, "0.0.0"); + const incompatibleRelease = await loadManagedRelease(runtimeRoot, FIRST_COMMIT); + expect(() => + assertDashboardReleaseHostRuntimeCompatible(incompatibleRelease) + ).toThrow("requires Bun 0.0.0"); await expect( activateDashboardRelease(FIRST_COMMIT, runtimeRoot, SCHEMA_6_OPTIONS) ).rejects.toThrow("requires Bun 0.0.0"); @@ -740,6 +775,39 @@ describe("Dashboard immutable release manager", () => { expect(existsSync(path.join(root, ".release-transition.json"))).toBe(false); }); + it("recovers a journal when current changed before previous was linked", async () => { + const root = temporaryReleasesRoot(); + await createManagedRelease(root, FIRST_COMMIT); + await createManagedRelease(root, SECOND_COMMIT); + await activateDashboardRelease(FIRST_COMMIT, root, SCHEMA_6_OPTIONS); + writeFileSync( + path.join(root, ".release-transition.json"), + `${JSON.stringify({ + after: { + current: SECOND_COMMIT, + previous: FIRST_COMMIT, + }, + before: { + current: FIRST_COMMIT, + previous: false, + }, + formatVersion: 1, + operation: "activate", + })}\n` + ); + rmSync(path.join(root, "current")); + symlinkSync(`releases/${SECOND_COMMIT}`, path.join(root, "current"), "dir"); + + const recovered = await activateDashboardRelease( + SECOND_COMMIT, + root, + SCHEMA_6_OPTIONS + ); + expect(recovered.current?.commitSha).toBe(SECOND_COMMIT); + expect(recovered.previous?.commitSha).toBe(FIRST_COMMIT); + expect(existsSync(path.join(root, ".release-transition.json"))).toBe(false); + }); + it.skipIf(!isReleaseTransitionLockAvailable())( "serializes status and transitions with a kernel-owned lock", async () => { diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 64883a757..adaa2b5fc 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -2100,17 +2100,29 @@ printf 'scheduled\n' 'trusted_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/previous")' ); expect(recoveryCommand).toContain( - 'if [ "$current_commit" = "$candidate_commit" ] && restart_services && ready_for_commit "${candidate_commit:0:8}"; then' + 'candidate_release=$(/usr/bin/readlink --canonicalize-existing "$releases_root/releases/$candidate_commit")' + ); + expect(recoveryCommand).toContain('trusted_release="$candidate_release"'); + expect(recoveryCommand).toContain( + 'activation_output="$(run_lifecycle activate "$candidate_commit")"' + ); + expect(recoveryCommand).toContain( + '[ "$activation_commit" = "$candidate_commit" ]' + ); + expect(recoveryCommand).toContain( + 'if restart_services && ready_for_commit "${candidate_commit:0:8}"; then' ); expect(recoveryCommand).toContain( "Interrupted release cutover recovered; active candidate passed restart and commit-bound readiness" ); expect( - recoveryCommand.indexOf('if [ "$current_commit" = "$candidate_commit" ]') - ).toBeLessThan( recoveryCommand.indexOf( 'activation_output="$(run_lifecycle activate "$candidate_commit")"' ) + ).toBeLessThan( + recoveryCommand.indexOf( + 'if restart_services && ready_for_commit "${candidate_commit:0:8}"; then' + ) ); expect(recoveryCommand).toContain( "run_lifecycle rollback && restart_services" From 23237366ba6041d3df9480555624e0dfe06adea6 Mon Sep 17 00:00:00 2001 From: mira-2026 Date: Sun, 26 Jul 2026 11:11:11 +0200 Subject: [PATCH 11/11] fix: separate OpenClaw roots during cutover --- backend/src/routes/configFileRoutes.ts | 4 +- backend/src/routes/fileRoutes.ts | 4 +- backend/src/routes/mediaRoutes.ts | 4 +- backend/src/routes/openclawConfigRoutes.ts | 4 +- backend/src/services/agents.ts | 4 +- backend/src/services/gitHygiene.ts | 1 - backend/test/gitHygiene.test.ts | 7 +- backend/test/routeAndServiceBehavior.test.ts | 82 ++++++++++++++++++++ docs/setup/production-deploy.md | 2 + docs/setup/secrets-and-env.md | 4 +- 10 files changed, 97 insertions(+), 19 deletions(-) diff --git a/backend/src/routes/configFileRoutes.ts b/backend/src/routes/configFileRoutes.ts index d5f252f7b..f0dab692c 100644 --- a/backend/src/routes/configFileRoutes.ts +++ b/backend/src/routes/configFileRoutes.ts @@ -22,9 +22,7 @@ const CONFIG_WRITE_BODY_LIMIT = MAX_CONFIG_WRITE_SIZE * 2; const ALLOWED_CONFIG_FILES = new Set(["openclaw.json", "hooks/transforms/agentmail.ts"]); function openclawRoot(): string | undefined { - const configured = - process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim(); + const configured = process.env.OPENCLAW_HOME?.trim(); const rawHome = process.env.HOME?.trim(); const home = rawHome && path.isAbsolute(rawHome) ? path.resolve(rawHome) : os.homedir().trim(); diff --git a/backend/src/routes/fileRoutes.ts b/backend/src/routes/fileRoutes.ts index 325bebca0..859c09500 100644 --- a/backend/src/routes/fileRoutes.ts +++ b/backend/src/routes/fileRoutes.ts @@ -29,9 +29,7 @@ interface FileItem { } function defaultWorkspaceRoot(): string { - const openclawHome = - process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim(); + const openclawHome = process.env.OPENCLAW_HOME?.trim(); if ( openclawHome && path.isAbsolute(openclawHome) && diff --git a/backend/src/routes/mediaRoutes.ts b/backend/src/routes/mediaRoutes.ts index 826e70e50..8a724da51 100644 --- a/backend/src/routes/mediaRoutes.ts +++ b/backend/src/routes/mediaRoutes.ts @@ -316,9 +316,7 @@ async function proxyGatewayMedia(request: Request): Promise { } function resolveOpenclawRoot(): string | undefined { - const configuredRoot = - process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim(); + const configuredRoot = process.env.OPENCLAW_HOME?.trim(); const homeDirectory = process.env.HOME?.trim() || os.homedir().trim(); if ( !configuredRoot && diff --git a/backend/src/routes/openclawConfigRoutes.ts b/backend/src/routes/openclawConfigRoutes.ts index 743f90f15..cc985164c 100644 --- a/backend/src/routes/openclawConfigRoutes.ts +++ b/backend/src/routes/openclawConfigRoutes.ts @@ -80,9 +80,7 @@ function resolveSafeAbsolutePath(candidate: string | undefined): string | undefi } function resolveOpenClawHome(): string | undefined { - const configuredRoot = - process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim(); + const configuredRoot = process.env.OPENCLAW_HOME?.trim(); if (configuredRoot) { return resolveSafeAbsolutePath(configuredRoot); } diff --git a/backend/src/services/agents.ts b/backend/src/services/agents.ts index a9d406333..17e909ee9 100644 --- a/backend/src/services/agents.ts +++ b/backend/src/services/agents.ts @@ -19,9 +19,7 @@ function defaultOpenclawRoot(): string { } function resolveOpenclawRoot(): string { - const configuredRoot = - process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim(); + const configuredRoot = process.env.OPENCLAW_HOME?.trim(); if (configuredRoot) { const resolved = Path.resolve(configuredRoot); return Path.isAbsolute(configuredRoot) && Path.parse(resolved).root !== resolved diff --git a/backend/src/services/gitHygiene.ts b/backend/src/services/gitHygiene.ts index 23ffe222e..5a8b6e785 100644 --- a/backend/src/services/gitHygiene.ts +++ b/backend/src/services/gitHygiene.ts @@ -56,7 +56,6 @@ function getOpenClawRoot(): string { return ( process.env.MIRA_OPENCLAW_ROOT?.trim() || process.env.OPENCLAW_HOME?.trim() || - process.env.MIRA_DASHBOARD_OPENCLAW_HOME?.trim() || path.join(homeDirectory, ".openclaw") ); } diff --git a/backend/test/gitHygiene.test.ts b/backend/test/gitHygiene.test.ts index ec3ef7a5e..2af40c796 100644 --- a/backend/test/gitHygiene.test.ts +++ b/backend/test/gitHygiene.test.ts @@ -179,16 +179,19 @@ describe("git hygiene automation", () => { ); }); - it("uses the process home OpenClaw default when no OpenClaw home is configured", async () => { + it("uses the process home OpenClaw default instead of the Dashboard client identity", async () => { rememberEnvironment("HOME"); rememberEnvironment("MIRA_OPENCLAW_ROOT"); rememberEnvironment("OPENCLAW_HOME"); rememberEnvironment("MIRA_DASHBOARD_OPENCLAW_HOME"); const homeRoot = createTemporaryRoot("mira-openclaw-home-default-"); + const dashboardClientRoot = createTemporaryRoot( + "mira-dashboard-openclaw-client-" + ); process.env.HOME = homeRoot; delete process.env.MIRA_OPENCLAW_ROOT; delete process.env.OPENCLAW_HOME; - delete process.env.MIRA_DASHBOARD_OPENCLAW_HOME; + process.env.MIRA_DASHBOARD_OPENCLAW_HOME = dashboardClientRoot; const calls: Array<{ arguments_: readonly string[]; cwd: string }> = []; const runProcessSpy = jest .spyOn(processModule, "runProcess") diff --git a/backend/test/routeAndServiceBehavior.test.ts b/backend/test/routeAndServiceBehavior.test.ts index 699449ff0..821461e69 100644 --- a/backend/test/routeAndServiceBehavior.test.ts +++ b/backend/test/routeAndServiceBehavior.test.ts @@ -1638,6 +1638,88 @@ describe("backend route and service behavior", () => { expect(stored.disable_intent_json).toBeNull(); }); + it("keeps primary OpenClaw data separate from the Dashboard client identity", async () => { + rememberEnvironment("HOME"); + rememberEnvironment("OPENCLAW_HOME"); + rememberEnvironment("MIRA_DASHBOARD_OPENCLAW_HOME"); + rememberEnvironment("WORKSPACE_ROOT"); + const homeRoot = createTemporaryRoot("mira-primary-openclaw-home-"); + const primaryRoot = path.join(homeRoot, ".openclaw"); + const dashboardClientRoot = createTemporaryRoot("mira-dashboard-client-home-"); + mkdirSync(path.join(primaryRoot, "media", "images"), { recursive: true }); + mkdirSync(path.join(primaryRoot, "workspace"), { recursive: true }); + mkdirSync(path.join(dashboardClientRoot, "media", "images"), { + recursive: true, + }); + mkdirSync(path.join(dashboardClientRoot, "workspace"), { recursive: true }); + writeFileSync(path.join(primaryRoot, "openclaw.json"), '{"primary":true}\n'); + writeFileSync( + path.join(dashboardClientRoot, "openclaw.json"), + '{"clientIdentity":true}\n' + ); + writeFileSync(path.join(primaryRoot, "workspace", "primary.txt"), "primary"); + writeFileSync( + path.join(dashboardClientRoot, "workspace", "client.txt"), + "client" + ); + writeFileSync( + path.join(primaryRoot, "media", "images", "primary.txt"), + "primary media" + ); + writeFileSync( + path.join(dashboardClientRoot, "media", "images", "client.txt"), + "client media" + ); + process.env.HOME = homeRoot; + delete process.env.OPENCLAW_HOME; + process.env.MIRA_DASHBOARD_OPENCLAW_HOME = dashboardClientRoot; + delete process.env.WORKSPACE_ROOT; + + const { configFileRoutes } = await import("../src/routes/configFileRoutes.ts"); + const configList = await responseJson( + await configFileRoutes["/api/config-files"].GET() + ); + expect(configList.root).toBe(primaryRoot); + + const { fileRoutes } = await import("../src/routes/fileRoutes.ts"); + const workspaceList = await responseJson( + await fileRoutes["/api/files"].GET( + new Request("https://test.local/api/files") + ) + ); + expect(workspaceList).toMatchObject({ + files: [expect.objectContaining({ name: "primary.txt" })], + root: path.join(primaryRoot, "workspace"), + }); + + const { mediaRoutes } = await import("../src/routes/mediaRoutes.ts"); + const media = await mediaRoutes["/api/media"].GET( + new Request( + "https://test.local/api/media?path=images/primary.txt&preview=text" + ) + ); + expect(media.status).toBe(200); + await expect(media.text()).resolves.toBe("primary media"); + + const agentId = `separation-${Bun.randomUUIDv7()}`; + try { + const { updateAgentCurrentTask } = await import("../src/services/agents.ts"); + await updateAgentCurrentTask(agentId, "Primary OpenClaw root"); + expect( + existsSync( + path.join(primaryRoot, "agents", agentId, "sessions", "metadata.json") + ) + ).toBe(true); + expect(existsSync(path.join(dashboardClientRoot, "agents", agentId))).toBe( + false + ); + } finally { + database + .prepare("DELETE FROM agent_task_history WHERE agent_id = ?") + .run(agentId); + } + }); + it("config file route allowlist, reads, writes, and backups", async () => { isolateOpenClawEnvironment("mira-config-file-route-"); const root = process.env.OPENCLAW_HOME!; diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index 6020bc96e..4491e1a62 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -107,6 +107,7 @@ git switch main git pull --ff-only origin main RELEASES_ROOT=/home/ubuntu/projects/mira-dashboard-releases +WORKTREE_ROOT=/home/ubuntu/projects/mira-dashboard-worktrees OLD_STATE_ROOT=/home/ubuntu/projects/mira-dashboard/backend/data STATE_ROOT=/home/ubuntu/projects/mira-dashboard-state OLD_DATABASE_PATH="$OLD_STATE_ROOT/mira-dashboard.db" @@ -129,6 +130,7 @@ if ! [[ "$DASHBOARD_PORT" =~ ^[0-9]+$ ]] || (( DASHBOARD_PORT < 1 || DASHBOARD_PORT > 65535 )); then DASHBOARD_PORT=3100 fi +install --directory --mode=0700 "$WORKTREE_ROOT" env \ MIRA_DASHBOARD_DB_PATH="$OLD_DATABASE_PATH" \ diff --git a/docs/setup/secrets-and-env.md b/docs/setup/secrets-and-env.md index b409f976f..6b1f897c1 100644 --- a/docs/setup/secrets-and-env.md +++ b/docs/setup/secrets-and-env.md @@ -46,7 +46,9 @@ and immutable releases. SQLite backups are derived as `/home/ubuntu/projects/mira-dashboard-state/backups/`. Versioned `backend/config/` files are release artifacts, not external state. `OPENCLAW_HOME` remains the primary OpenClaw installation/configuration root; -`MIRA_DASHBOARD_OPENCLAW_HOME` is the separate Dashboard client identity root. +`MIRA_DASHBOARD_OPENCLAW_HOME` is the separate Dashboard client identity root +and is never used as a fallback for primary OpenClaw files, agents, config, +workspace, or media. ## Network, Auth, And Browser Access