diff --git a/.github/ISSUE_TEMPLATE/ops_deploy.yml b/.github/ISSUE_TEMPLATE/ops_deploy.yml index ae0cb83ff..3bb132952 100644 --- a/.github/ISSUE_TEMPLATE/ops_deploy.yml +++ b/.github/ISSUE_TEMPLATE/ops_deploy.yml @@ -47,7 +47,7 @@ body: 1. Pull latest main 2. Build frontend/backend 3. Restart service - 4. Verify /api/health + 4. Verify /api/health/ready validations: required: true - type: checkboxes diff --git a/.gitignore b/.gitignore index e3e373ad1..3a01ebc38 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ lerna-debug.log* node_modules dist +release-manifest.json backend/data/ data/ .test-openclaw/ diff --git a/README.md b/README.md index a1f63bff2..cb3812fc8 100644 --- a/README.md +++ b/README.md @@ -103,22 +103,25 @@ Production preparation is deliberately separate from ordinary builds: bun run deploy:prepare ``` -This builds both applications and runs the restore-verified SQLite preflight. -Use it before a production restart; plain `build` remains safe for CI and local -verification. +This builds both applications, runs the restore-verified SQLite preflight, and +writes the checksummed release manifest used by production readiness and +activation. Use it before a production restart; plain `build` remains safe for +CI and local verification. ## Runtime notes - Backend default port: `3100`. - Frontend dev port: `5173`. -- Health endpoints: `/health` and `/api/health`. +- Health endpoints: public `/api/health/live`, public `/api/health/ready`, and + authenticated `/api/health/diagnostics`. - Dashboard SQLite uses WAL, numbered checksum-validated migrations, restrictive storage modes, deploy/maintenance snapshots, and automated restore checks. - Frontend builds and the local frontend dev server use Bun's HTML bundler with Babel React Compiler and Bun Tailwind plugins. - Dev server listens on all addresses so the dashboard can be reached over Tailscale when needed. - Auth is enforced by the backend request policy for every API route except - `GET|HEAD /api/health`, `GET|HEAD /api/auth/bootstrap`, + `GET|HEAD /api/health/live`, `GET|HEAD /api/health/ready`, + `GET|HEAD /api/auth/bootstrap`, `POST /api/auth/register-first-user`, `POST /api/auth/login`, `POST /api/auth/login/totp`, `POST /api/auth/login/recovery`, `POST /api/auth/login/webauthn/options`, diff --git a/backend/scripts/build.ts b/backend/scripts/build.ts index 50e5b66cc..b16371585 100644 --- a/backend/scripts/build.ts +++ b/backend/scripts/build.ts @@ -1,13 +1,22 @@ -import { mkdir, rm } from "node:fs/promises"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; +import { resolveBuildSourceIdentity } from "./buildSourceIdentity.ts"; + const backendDirectory = path.resolve(import.meta.dirname, ".."); const outdir = path.join(backendDirectory, "dist"); +const commitSha = resolveBuildSourceIdentity(backendDirectory); +if (commitSha === "unknown") { + throw new Error("Backend build requires a full Git commit identity"); +} await rm(outdir, { force: true, recursive: true }); await mkdir(outdir, { recursive: true }); const result = await Bun.build({ + define: { + __BACKEND_BUILD_COMMIT__: JSON.stringify(commitSha), + }, entrypoints: [ path.join(backendDirectory, "src/serverStart.ts"), path.join(backendDirectory, "src/workerStart.ts"), @@ -16,7 +25,7 @@ const result = await Bun.build({ ], format: "esm", outdir, - packages: "external", + packages: "bundle", splitting: false, sourcemap: "external", target: "bun", @@ -25,3 +34,17 @@ const result = await Bun.build({ if (!result.success) { throw new AggregateError(result.logs, "Backend build failed"); } + +await writeFile( + path.join(outdir, "build-identity.json"), + `${JSON.stringify( + { + bunVersion: Bun.version, + commitSha, + component: "backend", + formatVersion: 1, + }, + undefined, + 2 + )}\n` +); diff --git a/backend/scripts/buildSourceIdentity.ts b/backend/scripts/buildSourceIdentity.ts new file mode 100644 index 000000000..985ee9c96 --- /dev/null +++ b/backend/scripts/buildSourceIdentity.ts @@ -0,0 +1,42 @@ +const FULL_GIT_COMMIT_PATTERN = /^[\da-f]{40}$/u; + +function gitOutput(repoDirectory: string, arguments_: string[]): string | undefined { + try { + const result = Bun.spawnSync({ + cmd: ["git", "-C", repoDirectory, ...arguments_], + stderr: "ignore", + stdin: "ignore", + stdout: "pipe", + }); + if (result.exitCode !== 0) { + return undefined; + } + return new TextDecoder().decode(result.stdout).trim(); + } catch { + return undefined; + } +} + +export function isReleaseBuildCommit(value: string): boolean { + return FULL_GIT_COMMIT_PATTERN.test(value); +} + +/** + * Returns a full release commit only for a clean source tree. Dirty builds stay + * usable for local verification but cannot be accepted by release:manifest. + */ +export function resolveBuildSourceIdentity(repoDirectory = process.cwd()): string { + const commit = gitOutput(repoDirectory, ["rev-parse", "HEAD"]); + if (!commit || !isReleaseBuildCommit(commit)) { + return "unknown"; + } + const status = gitOutput(repoDirectory, [ + "status", + "--porcelain=v1", + "--untracked-files=all", + ]); + if (status === undefined) { + return "unknown"; + } + return status ? `${commit}-dirty` : commit; +} diff --git a/backend/src/buildIdentity.ts b/backend/src/buildIdentity.ts new file mode 100644 index 000000000..8a7f8a610 --- /dev/null +++ b/backend/src/buildIdentity.ts @@ -0,0 +1,19 @@ +declare const __BACKEND_BUILD_COMMIT__: string | undefined; + +const FULL_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; + +/** + * Returns the commit embedded by the backend bundler. + * + * Source-mode development and tests intentionally use a non-release identity. + * Production readiness requires a bundled full SHA that matches the manifest. + */ +export function getBackendBuildCommit(): string { + if ( + typeof __BACKEND_BUILD_COMMIT__ === "string" && + FULL_COMMIT_SHA_PATTERN.test(__BACKEND_BUILD_COMMIT__) + ) { + return __BACKEND_BUILD_COMMIT__; + } + return "development"; +} diff --git a/backend/src/databaseMigrationRunner.ts b/backend/src/databaseMigrationRunner.ts index e04945abe..396ae432b 100644 --- a/backend/src/databaseMigrationRunner.ts +++ b/backend/src/databaseMigrationRunner.ts @@ -4,6 +4,7 @@ import { type DatabaseMigration, databaseMigrations, } from "./databaseMigrations/index.ts"; +import { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY } from "./databaseSchemaCompatibility.ts"; import { createVerifiedSqliteBackup, pruneSqliteBackups, @@ -76,20 +77,37 @@ function appliedMigrationRows(database: Database): AppliedMigrationRow[] { .all() as AppliedMigrationRow[]; } -export function validateDatabaseMigrationHistory(database: Database): number { +export function validateDatabaseMigrationHistory( + database: Database, + maximumCompatibleVersion: number = DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum +): number { assertMigrationRegistry(); + if ( + !Number.isSafeInteger(maximumCompatibleVersion) || + maximumCompatibleVersion < databaseMigrations.length + ) { + throw new TypeError("Invalid maximum compatible SQLite schema version"); + } const appliedRows = appliedMigrationRows(database); for (const [index, row] of appliedRows.entries()) { - const expected = databaseMigrations[index]; - if (!expected) { + const expectedVersion = index + 1; + if (row.version !== expectedVersion) { throw new Error( - `Database contains unknown SQLite migration version ${row.version}` + `SQLite migration history is not contiguous at version ${expectedVersion}` ); } - if (row.version !== expected.version) { - throw new Error( - `SQLite migration history is not contiguous at version ${expected.version}` - ); + const expected = databaseMigrations[index]; + if (!expected) { + if ( + row.version > maximumCompatibleVersion || + !/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(row.name) || + !/^[\da-f]{64}$/u.test(row.checksum) + ) { + throw new Error( + `Database contains incompatible SQLite migration version ${row.version}` + ); + } + continue; } if (row.name !== expected.name) { throw new Error( @@ -153,7 +171,7 @@ function applyPendingDatabaseMigrations( createBackup?: () => SqliteBackupResult | undefined ): DatabaseMigrationResult { const appliedCountBeforeLock = validateDatabaseMigrationHistory(database); - if (appliedCountBeforeLock === databaseMigrations.length) { + if (appliedCountBeforeLock >= databaseMigrations.length) { return { applied: [] }; } @@ -162,7 +180,7 @@ function applyPendingDatabaseMigrations( database.run("BEGIN IMMEDIATE"); try { const appliedCount = validateDatabaseMigrationHistory(database); - if (appliedCount === databaseMigrations.length) { + if (appliedCount >= databaseMigrations.length) { database.run("COMMIT"); return { applied }; } diff --git a/backend/src/databaseSchemaCompatibility.ts b/backend/src/databaseSchemaCompatibility.ts new file mode 100644 index 000000000..597da3ae8 --- /dev/null +++ b/backend/src/databaseSchemaCompatibility.ts @@ -0,0 +1,41 @@ +import { databaseMigrations } from "./databaseMigrations/index.ts"; + +const CURRENT_DATABASE_SCHEMA_VERSION = databaseMigrations.at(-1)?.version ?? 0; + +/** + * Keep this range explicit. An expand migration may widen the maximum before + * the migration ships; a contract migration must narrow it only after the + * previous release has left the rollback window. + */ +export const DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY = Object.freeze({ + maximum: 6, + minimum: 6, + target: CURRENT_DATABASE_SCHEMA_VERSION, +}); + +interface DatabaseSchemaCompatibility { + maximum: number; + minimum: number; +} + +if ( + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target < + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum || + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target > + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum +) { + throw new Error( + "Dashboard database schema target is outside the declared release compatibility range" + ); +} + +export function isDatabaseSchemaCompatible( + version: number, + compatibility: DatabaseSchemaCompatibility = DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY +): boolean { + return ( + Number.isSafeInteger(version) && + version >= compatibility.minimum && + version <= compatibility.maximum + ); +} diff --git a/backend/src/frontendAssets.ts b/backend/src/frontendAssets.ts new file mode 100644 index 000000000..d4c5c7ec2 --- /dev/null +++ b/backend/src/frontendAssets.ts @@ -0,0 +1,33 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { getProcessReleaseRoot } from "./releaseManifest.ts"; + +export function resolveFrontendPath( + environment: Record = process.env, + releaseRoot = getProcessReleaseRoot() +): string { + const releaseFrontendPath = path.join(releaseRoot, "dist"); + const configuredPath = environment.MIRA_DASHBOARD_FRONTEND_PATH?.trim(); + if (!configuredPath) { + return releaseFrontendPath; + } + if ( + environment.NODE_ENV === "production" && + path.resolve(configuredPath) !== path.resolve(releaseFrontendPath) + ) { + throw new Error( + "MIRA_DASHBOARD_FRONTEND_PATH cannot override the checksummed release frontend in production" + ); + } + return configuredPath; +} + +export function isFrontendIndexReady(): boolean { + try { + const indexStat = fs.statSync(path.join(resolveFrontendPath(), "index.html")); + return indexStat.isFile(); + } catch { + return false; + } +} diff --git a/backend/src/health.ts b/backend/src/health.ts new file mode 100644 index 000000000..f232314f5 --- /dev/null +++ b/backend/src/health.ts @@ -0,0 +1,154 @@ +import { database } from "./database.ts"; +import { validateDatabaseMigrationHistory } from "./databaseMigrationRunner.ts"; +import { + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY, + isDatabaseSchemaCompatible, +} from "./databaseSchemaCompatibility.ts"; +import { isFrontendIndexReady } from "./frontendAssets.ts"; +import gateway from "./gateway.ts"; +import { + getRuntimeReleaseIdentity, + type RuntimeReleaseIdentity, +} from "./releaseManifest.ts"; +import { + getJobExecutionSummary, + isJobWorkerReleaseReady, +} from "./services/jobExecutionQueue.ts"; + +interface DatabaseReadiness { + currentSchemaVersion?: number; + maximumCompatibleSchemaVersion: number; + minimumCompatibleSchemaVersion: number; + ready: boolean; + targetSchemaVersion: number; +} + +export interface ReadinessSignals { + database: DatabaseReadiness; + frontendReady: boolean; + gatewayConnected: boolean; + release: RuntimeReleaseIdentity; + sessionCount: number; + workerReady: boolean; +} + +export interface DashboardReadinessSnapshot { + checks: { + database: DatabaseReadiness; + frontend: { ready: boolean }; + release: { + backendCommit: string; + frontendCommit: string; + issue?: RuntimeReleaseIdentity["issue"]; + manifestFormatVersion?: number; + ready: boolean; + source: RuntimeReleaseIdentity["source"]; + }; + worker: { ready: boolean }; + }; + dependencies: { + gatewayConnected: boolean; + }; + status: "isReady" | "notReady"; +} + +function databaseReadiness(): DatabaseReadiness { + try { + database.query("SELECT 1").get(); + const currentSchemaVersion = validateDatabaseMigrationHistory(database); + return { + currentSchemaVersion, + maximumCompatibleSchemaVersion: + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum, + minimumCompatibleSchemaVersion: + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum, + ready: isDatabaseSchemaCompatible(currentSchemaVersion), + targetSchemaVersion: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target, + }; + } catch (error) { + console.warn("[Health] Database readiness failed:", error); + return { + maximumCompatibleSchemaVersion: + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum, + minimumCompatibleSchemaVersion: + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum, + ready: false, + targetSchemaVersion: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target, + }; + } +} + +function isWorkerReady(release: RuntimeReleaseIdentity): boolean { + try { + if (release.source === "manifest" && release.commitSha) { + return isJobWorkerReleaseReady(release.commitSha); + } + return getJobExecutionSummary().workerOnline; + } catch (error) { + console.warn("[Health] Failed to read job worker telemetry:", error); + return false; + } +} + +export async function collectReadinessSignals(): Promise { + const release = await getRuntimeReleaseIdentity(); + return { + database: databaseReadiness(), + frontendReady: isFrontendIndexReady(), + gatewayConnected: gateway.isConnected(), + release, + sessionCount: gateway.getSessions().length, + workerReady: isWorkerReady(release), + }; +} + +export function evaluateReadiness(signals: ReadinessSignals): DashboardReadinessSnapshot { + const ready = + signals.database.ready && + signals.frontendReady && + signals.release.ready && + signals.workerReady; + return { + checks: { + database: signals.database, + frontend: { ready: signals.frontendReady }, + release: { + backendCommit: signals.release.backendCommit, + frontendCommit: signals.release.frontendCommit, + ...(signals.release.issue && { issue: signals.release.issue }), + ...(signals.release.manifestFormatVersion !== undefined && { + manifestFormatVersion: signals.release.manifestFormatVersion, + }), + ready: signals.release.ready, + source: signals.release.source, + }, + worker: { ready: signals.workerReady }, + }, + dependencies: { + // Gateway availability is diagnostic. A Gateway outage cannot be + // repaired by rolling Dashboard code back. + gatewayConnected: signals.gatewayConnected, + }, + status: ready ? "isReady" : "notReady", + }; +} + +export function livenessSnapshot() { + return { + status: "isOk" as const, + uptimeSeconds: Math.floor(process.uptime()), + }; +} + +export async function readinessSnapshot(): Promise { + return evaluateReadiness(await collectReadinessSignals()); +} + +export async function diagnosticsSnapshot() { + const signals = await collectReadinessSignals(); + return { + ...evaluateReadiness(signals), + releaseDetails: signals.release, + sessionCount: signals.sessionCount, + }; +} diff --git a/backend/src/releaseManifest.ts b/backend/src/releaseManifest.ts new file mode 100644 index 000000000..1813f3d1e --- /dev/null +++ b/backend/src/releaseManifest.ts @@ -0,0 +1,706 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; + +import { getBackendBuildCommit } from "./buildIdentity.ts"; +import { databaseMigrations } from "./databaseMigrations/index.ts"; +import { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY } from "./databaseSchemaCompatibility.ts"; +import { guardedPath, writeTextNoFollowGuarded } from "./lib/guardedOps.ts"; + +export { DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY } from "./databaseSchemaCompatibility.ts"; + +export const RELEASE_MANIFEST_FILE_NAME = "release-manifest.json"; +export const RELEASE_MANIFEST_FORMAT_VERSION = 1; + +const MAX_RELEASE_MANIFEST_BYTES = 256 * 1024; +const RELEASE_ARTIFACT_DIRECTORIES = ["dist", "backend/dist"] as const; +const RELEASE_STATIC_ARTIFACTS = [ + "backend/config/log-rotation.json", + "backend/bun.lock", + "backend/package.json", + "bun.lock", + "package.json", +] as const; +const REQUIRED_RELEASE_ARTIFACTS = [ + ...RELEASE_STATIC_ARTIFACTS, + "backend/dist/build-identity.json", + "backend/dist/databasePreflight.js", + "backend/dist/resetDashboardPassword.js", + "backend/dist/serverStart.js", + "backend/dist/workerStart.js", + "dist/build-identity.json", + "dist/index.html", +] as const; +const MAX_BUILD_IDENTITY_BYTES = 4096; +const RUNTIME_RELEASE_VERIFICATION_CACHE_MS = 15_000; +const SHA_256_PATTERN = /^[\da-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u; +const RUNTIME_COMMIT_PATTERN = /^[\da-f]{8,40}$/u; + +export interface ReleaseManifestArtifact { + path: string; + sha256: string; + sizeBytes: number; +} + +export interface DashboardReleaseManifest { + artifacts: ReleaseManifestArtifact[]; + builtAt: string; + bunVersion: string; + commitSha: string; + commitShort: string; + commitTitle: string; + components: { + backendCommit: string; + frontendCommit: string; + }; + formatVersion: 1; + schema: { + maximumCompatible: number; + migrationRegistrySha256: string; + minimumCompatible: number; + target: number; + }; +} + +export interface RuntimeReleaseIdentity { + artifactCount?: number; + backendCommit: string; + commitSha?: string; + frontendCommit: string; + issue?: "manifest-code-mismatch" | "manifest-invalid" | "manifest-missing"; + manifestFormatVersion?: number; + ready: boolean; + schema?: DashboardReleaseManifest["schema"]; + source: "git" | "manifest" | "unknown"; +} + +export function requireRunnableReleaseCommit( + release: RuntimeReleaseIdentity, + runtimeLabel: string, + environment = process.env.NODE_ENV +): string { + if (environment === "production" && !release.ready) { + throw new Error( + `${runtimeLabel} release identity is not ready (${release.issue ?? release.source})` + ); + } + const releaseCommit = release.commitSha ?? release.backendCommit; + if (!RUNTIME_COMMIT_PATTERN.test(releaseCommit)) { + throw new Error( + `${runtimeLabel} release identity does not contain a valid commit` + ); + } + return releaseCommit; +} + +interface CreateReleaseManifestOptions { + builtAt?: Date; + bunVersion?: string; + commitSha?: string; + commitTitle?: string; + releaseRoot: string; +} + +interface ComponentBuildIdentity { + bunVersion: string; + commitSha: string; + component: "backend" | "frontend"; + formatVersion: 1; +} + +interface RuntimeReleaseIdentityCache { + expiresAt: number; + key: string; + promise: Promise; +} + +const runtimeReleaseIdentityCacheState: { + entry?: RuntimeReleaseIdentityCache; +} = {}; + +function isPlainRecord(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function compareStrings(left: string, right: string): number { + return left.localeCompare(right); +} + +function hasExactKeys(record: Record, expected: string[]): boolean { + const actual = Object.keys(record).toSorted(compareStrings); + const sortedExpected = expected.toSorted(compareStrings); + return ( + actual.length === sortedExpected.length && + actual.every((key, index) => key === sortedExpected[index]) + ); +} + +function sha256(value: Uint8Array | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +export function databaseMigrationRegistrySha256(): string { + const serialized = databaseMigrations + .map((migration) => `${migration.version}\0${migration.name}\0${migration.sql}`) + .join("\0"); + return sha256(serialized); +} + +function isSafeArtifactPath(value: string): boolean { + if ( + !value || + value.includes("\\") || + value.includes("\0") || + path.posix.isAbsolute(value) || + path.posix.normalize(value) !== value + ) { + return false; + } + const parts = value.split("/"); + if (parts.some((part) => !part || part === "." || part === "..")) { + return false; + } + return ( + RELEASE_STATIC_ARTIFACTS.includes( + value as (typeof RELEASE_STATIC_ARTIFACTS)[number] + ) || + RELEASE_ARTIFACT_DIRECTORIES.some((directory) => + value.startsWith(`${directory}/`) + ) + ); +} + +function artifactPath(releaseRoot: string, relativePath: string): string { + if (!isSafeArtifactPath(relativePath)) { + throw new TypeError(`Invalid release artifact path: ${relativePath}`); + } + return path.join(releaseRoot, ...relativePath.split("/")); +} + +async function readRegularFileNoFollow(filePath: string): Promise { + const file = await fsp.open( + filePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW + ); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.nlink !== 1) { + throw new TypeError("Release artifacts must be single-link regular files"); + } + return await file.readFile(); + } finally { + await file.close(); + } +} + +async function collectArtifactDirectory( + releaseRoot: string, + relativeDirectory: string +): Promise { + const absoluteDirectory = artifactPath( + releaseRoot, + `${relativeDirectory}/placeholder` + ); + const directoryPath = path.dirname(absoluteDirectory); + const directoryStat = await fsp.lstat(directoryPath); + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) { + throw new TypeError( + `Release artifact directory must be a real directory: ${relativeDirectory}` + ); + } + + const artifacts: string[] = []; + const visit = async (absolute: string, relative: string): Promise => { + const entries = await fsp.readdir(absolute, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isSymbolicLink()) { + throw new TypeError( + `Release artifact tree must not contain symlinks: ${relative}/${entry.name}` + ); + } + const childAbsolute = path.join(absolute, entry.name); + const childRelative = `${relative}/${entry.name}`; + if (entry.isDirectory()) { + await visit(childAbsolute, childRelative); + } else if (entry.isFile()) { + artifacts.push(childRelative); + } else { + throw new TypeError( + `Release artifact tree must contain only files and directories: ${childRelative}` + ); + } + } + }; + + await visit(directoryPath, relativeDirectory); + return artifacts; +} + +export async function listReleaseArtifactPaths(releaseRoot: string): Promise { + const realReleaseRoot = await fsp.realpath(releaseRoot); + const rootStat = await fsp.lstat(realReleaseRoot); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { + throw new TypeError("Release root must resolve to a real directory"); + } + + const paths: string[] = [...RELEASE_STATIC_ARTIFACTS]; + for (const directory of RELEASE_ARTIFACT_DIRECTORIES) { + paths.push(...(await collectArtifactDirectory(realReleaseRoot, directory))); + } + return paths.toSorted(compareStrings); +} + +async function releaseArtifact( + releaseRoot: string, + relativePath: string +): Promise { + const content = await readRegularFileNoFollow( + artifactPath(releaseRoot, relativePath) + ); + return { + path: relativePath, + sha256: sha256(content), + sizeBytes: content.byteLength, + }; +} + +function gitOutput(releaseRoot: string, arguments_: string[]): string { + const result = Bun.spawnSync({ + cmd: ["git", "-C", releaseRoot, ...arguments_], + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + if (result.exitCode !== 0) { + throw new Error(`Git release identity command failed: ${arguments_.join(" ")}`); + } + return new TextDecoder().decode(result.stdout).trim(); +} + +function assertGitReleaseSourceClean(releaseRoot: string): void { + const status = gitOutput(releaseRoot, [ + "status", + "--porcelain=v1", + "--untracked-files=all", + ]); + if (status) { + throw new Error("Release source contains uncommitted changes"); + } +} + +function assertCommitIdentity(commitSha: string, commitTitle: string): void { + if (!COMMIT_SHA_PATTERN.test(commitSha)) { + throw new TypeError("Release commit must be a full lowercase Git SHA"); + } + if (!commitTitle || commitTitle.length > 500 || commitTitle.includes("\0")) { + throw new TypeError("Release commit title is invalid"); + } +} + +async function loadComponentBuildIdentity( + releaseRoot: string, + component: ComponentBuildIdentity["component"] +): Promise { + const relativePath = + component === "backend" + ? "backend/dist/build-identity.json" + : "dist/build-identity.json"; + const content = await readRegularFileNoFollow( + artifactPath(releaseRoot, relativePath) + ); + if (content.byteLength === 0 || content.byteLength > MAX_BUILD_IDENTITY_BYTES) { + throw new TypeError(`${component} build identity must be a bounded file`); + } + const value = JSON.parse(content.toString("utf8")) as unknown; + if ( + !isPlainRecord(value) || + !hasExactKeys(value, ["bunVersion", "commitSha", "component", "formatVersion"]) || + value.component !== component || + value.formatVersion !== 1 || + typeof value.commitSha !== "string" || + !COMMIT_SHA_PATTERN.test(value.commitSha) || + typeof value.bunVersion !== "string" || + !value.bunVersion || + value.bunVersion.length > 64 + ) { + throw new TypeError(`${component} build identity is invalid`); + } + return { + bunVersion: value.bunVersion, + commitSha: value.commitSha, + component, + formatVersion: 1, + }; +} + +export async function createReleaseManifest( + options: CreateReleaseManifestOptions +): Promise { + const releaseRoot = await fsp.realpath(options.releaseRoot); + if (options.commitSha === undefined || options.commitTitle === undefined) { + assertGitReleaseSourceClean(releaseRoot); + } + const commitSha = options.commitSha ?? gitOutput(releaseRoot, ["rev-parse", "HEAD"]); + const commitTitle = + options.commitTitle ?? gitOutput(releaseRoot, ["log", "-1", "--pretty=%s"]); + assertCommitIdentity(commitSha, commitTitle); + const bunVersion = options.bunVersion ?? Bun.version; + const [backendBuild, frontendBuild] = await Promise.all([ + loadComponentBuildIdentity(releaseRoot, "backend"), + loadComponentBuildIdentity(releaseRoot, "frontend"), + ]); + for (const build of [backendBuild, frontendBuild]) { + if (build.commitSha !== commitSha || build.bunVersion !== bunVersion) { + throw new Error( + `${build.component} build identity does not match the release source` + ); + } + } + + const artifactPaths = await listReleaseArtifactPaths(releaseRoot); + const artifacts: ReleaseManifestArtifact[] = []; + for (const relativePath of artifactPaths) { + artifacts.push(await releaseArtifact(releaseRoot, relativePath)); + } + const commitShort = commitSha.slice(0, 8); + const manifest: DashboardReleaseManifest = { + artifacts, + builtAt: (options.builtAt ?? new Date()).toISOString(), + bunVersion, + commitSha, + commitShort, + commitTitle, + components: { + backendCommit: commitShort, + frontendCommit: commitShort, + }, + formatVersion: RELEASE_MANIFEST_FORMAT_VERSION, + schema: { + maximumCompatible: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum, + migrationRegistrySha256: databaseMigrationRegistrySha256(), + minimumCompatible: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum, + target: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target, + }, + }; + return parseReleaseManifest(manifest); +} + +function parseArtifact(value: unknown): ReleaseManifestArtifact { + if ( + !isPlainRecord(value) || + !hasExactKeys(value, ["path", "sha256", "sizeBytes"]) || + typeof value.path !== "string" || + !isSafeArtifactPath(value.path) || + typeof value.sha256 !== "string" || + !SHA_256_PATTERN.test(value.sha256) || + !Number.isSafeInteger(value.sizeBytes) || + (value.sizeBytes as number) < 0 + ) { + throw new TypeError("Release manifest contains an invalid artifact"); + } + return { + path: value.path, + sha256: value.sha256, + sizeBytes: value.sizeBytes as number, + }; +} + +function parseSchema(value: unknown): DashboardReleaseManifest["schema"] { + if ( + !isPlainRecord(value) || + !hasExactKeys(value, [ + "maximumCompatible", + "migrationRegistrySha256", + "minimumCompatible", + "target", + ]) + ) { + throw new TypeError("Release manifest schema declaration is invalid"); + } + const { maximumCompatible, minimumCompatible, target } = value; + if ( + !Number.isSafeInteger(maximumCompatible) || + !Number.isSafeInteger(minimumCompatible) || + !Number.isSafeInteger(target) || + (minimumCompatible as number) < 0 || + (minimumCompatible as number) > (target as number) || + (target as number) > (maximumCompatible as number) || + typeof value.migrationRegistrySha256 !== "string" || + !SHA_256_PATTERN.test(value.migrationRegistrySha256) + ) { + throw new TypeError("Release manifest schema range is invalid"); + } + return { + maximumCompatible: maximumCompatible as number, + migrationRegistrySha256: value.migrationRegistrySha256, + minimumCompatible: minimumCompatible as number, + target: target as number, + }; +} + +export function parseReleaseManifest(value: unknown): DashboardReleaseManifest { + if ( + !isPlainRecord(value) || + !hasExactKeys(value, [ + "artifacts", + "builtAt", + "bunVersion", + "commitSha", + "commitShort", + "commitTitle", + "components", + "formatVersion", + "schema", + ]) || + value.formatVersion !== RELEASE_MANIFEST_FORMAT_VERSION || + typeof value.commitSha !== "string" || + typeof value.commitShort !== "string" || + typeof value.commitTitle !== "string" || + typeof value.builtAt !== "string" || + typeof value.bunVersion !== "string" || + !Array.isArray(value.artifacts) || + value.artifacts.length === 0 || + value.artifacts.length > 10_000 || + !isPlainRecord(value.components) || + !hasExactKeys(value.components, ["backendCommit", "frontendCommit"]) + ) { + throw new TypeError("Release manifest shape is invalid"); + } + assertCommitIdentity(value.commitSha, value.commitTitle); + const expectedShortCommit = value.commitSha.slice(0, 8); + if ( + value.commitShort !== expectedShortCommit || + value.components.backendCommit !== expectedShortCommit || + value.components.frontendCommit !== expectedShortCommit || + Number.isNaN(Date.parse(value.builtAt)) || + new Date(value.builtAt).toISOString() !== value.builtAt || + !value.bunVersion || + value.bunVersion.length > 64 + ) { + throw new TypeError("Release manifest identity is invalid"); + } + + const artifacts = value.artifacts.map((artifact) => parseArtifact(artifact)); + const artifactPaths = artifacts.map((artifact) => artifact.path); + const sortedArtifactPaths = artifactPaths.toSorted(compareStrings); + if ( + new Set(artifactPaths).size !== artifactPaths.length || + artifactPaths.some( + (artifactPath_, index) => artifactPath_ !== sortedArtifactPaths[index] + ) || + REQUIRED_RELEASE_ARTIFACTS.some( + (requiredPath) => !artifactPaths.includes(requiredPath) + ) + ) { + throw new TypeError("Release manifest artifact inventory is invalid"); + } + + return { + artifacts, + builtAt: value.builtAt, + bunVersion: value.bunVersion, + commitSha: value.commitSha, + commitShort: value.commitShort, + commitTitle: value.commitTitle, + components: { + backendCommit: value.components.backendCommit as string, + frontendCommit: value.components.frontendCommit as string, + }, + formatVersion: RELEASE_MANIFEST_FORMAT_VERSION, + schema: parseSchema(value.schema), + }; +} + +export async function writeReleaseManifest( + options: CreateReleaseManifestOptions +): Promise { + const releaseRoot = await fsp.realpath(options.releaseRoot); + const manifest = await createReleaseManifest({ ...options, releaseRoot }); + const serialized = `${JSON.stringify(manifest, undefined, 2)}\n`; + if (Buffer.byteLength(serialized) > MAX_RELEASE_MANIFEST_BYTES) { + throw new TypeError("Release manifest must be a bounded regular file"); + } + await writeTextNoFollowGuarded( + guardedPath(path.join(releaseRoot, RELEASE_MANIFEST_FILE_NAME)), + serialized, + 0o644 + ); + return manifest; +} + +export async function loadReleaseManifest( + releaseRoot: string +): Promise { + const realReleaseRoot = await fsp.realpath(releaseRoot); + const manifestPath = path.join(realReleaseRoot, RELEASE_MANIFEST_FILE_NAME); + const file = await fsp.open( + manifestPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW + ); + try { + const stat = await file.stat(); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size === 0 || + stat.size > MAX_RELEASE_MANIFEST_BYTES + ) { + throw new TypeError("Release manifest must be a bounded regular file"); + } + // eslint-disable-next-line unicorn/consistent-json-file-read -- The pinned no-follow descriptor must remain the read target. + const serialized = await file.readFile("utf8"); + return parseReleaseManifest(JSON.parse(serialized) as unknown); + } finally { + await file.close(); + } +} + +export async function verifyReleaseArtifacts( + releaseRoot: string, + manifest: DashboardReleaseManifest +): Promise { + const realReleaseRoot = await fsp.realpath(releaseRoot); + const inventory = await listReleaseArtifactPaths(realReleaseRoot); + const declared = manifest.artifacts.map((artifact) => artifact.path); + if ( + inventory.length !== declared.length || + inventory.some((artifactPath_, index) => artifactPath_ !== declared[index]) + ) { + throw new Error("Release artifact inventory does not match its manifest"); + } + for (const artifact of manifest.artifacts) { + const actual = await releaseArtifact(realReleaseRoot, artifact.path); + if ( + actual.sha256 !== artifact.sha256 || + actual.sizeBytes !== artifact.sizeBytes + ) { + throw new Error(`Release artifact verification failed: ${artifact.path}`); + } + } +} + +function inferProcessReleaseRoot(): string { + const configured = process.env.MIRA_DASHBOARD_RELEASE_ROOT?.trim(); + const candidate = configured + ? path.resolve(configured) + : path.basename(process.cwd()) === "backend" + ? path.dirname(process.cwd()) + : path.resolve(import.meta.dirname, "..", ".."); + try { + return fs.realpathSync(candidate); + } catch { + return candidate; + } +} + +const PROCESS_RELEASE_ROOT = inferProcessReleaseRoot(); + +export function getProcessReleaseRoot(): string { + return PROCESS_RELEASE_ROOT; +} + +function fallbackGitCommit(releaseRoot: string): string { + try { + const value = gitOutput(releaseRoot, ["rev-parse", "--short=8", "HEAD"]); + return /^[\da-f]{8}$/u.test(value) ? value : "unknown"; + } catch { + return "unknown"; + } +} + +export async function loadRuntimeReleaseIdentity( + releaseRoot = PROCESS_RELEASE_ROOT, + environment = process.env.NODE_ENV, + backendBuildCommit = getBackendBuildCommit() +): Promise { + try { + const manifest = await loadReleaseManifest(releaseRoot); + await verifyReleaseArtifacts(releaseRoot, manifest); + const isManifestMatchesCode = + manifest.commitSha === backendBuildCommit && + manifest.bunVersion === Bun.version && + manifest.schema.target === DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target && + manifest.schema.minimumCompatible === + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum && + manifest.schema.maximumCompatible === + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum && + manifest.schema.migrationRegistrySha256 === databaseMigrationRegistrySha256(); + return { + artifactCount: manifest.artifacts.length, + backendCommit: manifest.components.backendCommit, + commitSha: manifest.commitSha, + frontendCommit: manifest.components.frontendCommit, + ...(!isManifestMatchesCode && { + issue: "manifest-code-mismatch" as const, + }), + manifestFormatVersion: manifest.formatVersion, + ready: isManifestMatchesCode, + schema: manifest.schema, + source: "manifest", + }; + } catch (error) { + const commit = fallbackGitCommit(releaseRoot); + const isMissing = (error as NodeJS.ErrnoException).code === "ENOENT"; + const isDevelopmentFallback = environment !== "production" && isMissing; + return { + backendCommit: commit, + frontendCommit: commit, + ...(!isDevelopmentFallback && { + issue: isMissing + ? ("manifest-missing" as const) + : ("manifest-invalid" as const), + }), + ready: isDevelopmentFallback, + source: commit === "unknown" ? "unknown" : "git", + }; + } +} + +export function getRuntimeReleaseIdentity( + releaseRoot = PROCESS_RELEASE_ROOT, + environment = process.env.NODE_ENV, + backendBuildCommit = getBackendBuildCommit() +): Promise { + const key = JSON.stringify([releaseRoot, environment, backendBuildCommit]); + const now = Date.now(); + if ( + runtimeReleaseIdentityCacheState.entry?.key === key && + runtimeReleaseIdentityCacheState.entry.expiresAt > now + ) { + return runtimeReleaseIdentityCacheState.entry.promise; + } + + const promise = loadRuntimeReleaseIdentity( + releaseRoot, + environment, + backendBuildCommit + ); + const cacheEntry: RuntimeReleaseIdentityCache = { + // Concurrent probes share the in-flight verification. The bounded TTL + // begins only after the complete artifact scan settles. + expiresAt: Infinity, + key, + promise, + }; + runtimeReleaseIdentityCacheState.entry = cacheEntry; + void promise + .then(() => { + if (runtimeReleaseIdentityCacheState.entry === cacheEntry) { + cacheEntry.expiresAt = Date.now() + RUNTIME_RELEASE_VERIFICATION_CACHE_MS; + } + }) + .catch(() => { + if (runtimeReleaseIdentityCacheState.entry === cacheEntry) { + runtimeReleaseIdentityCacheState.entry = undefined; + } + }); + return promise; +} + +export function invalidateRuntimeReleaseIdentityCache(): void { + runtimeReleaseIdentityCacheState.entry = undefined; +} diff --git a/backend/src/requestPolicy.ts b/backend/src/requestPolicy.ts index b38224f98..2ceb93734 100644 --- a/backend/src/requestPolicy.ts +++ b/backend/src/requestPolicy.ts @@ -84,6 +84,8 @@ const BUCKET_STALE_MS = Math.max(apiRule.windowMs, authRule.windowMs) * 2; const SAFE_REQUEST_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); const PUBLIC_API_METHODS = new Map>([ ["/api/health", new Set(["GET", "HEAD"])], + ["/api/health/live", new Set(["GET", "HEAD"])], + ["/api/health/ready", new Set(["GET", "HEAD"])], ["/api/auth/bootstrap", new Set(["GET", "HEAD"])], ["/api/auth/login", new Set(["POST"])], ["/api/auth/login/recovery", new Set(["POST"])], diff --git a/backend/src/routes.ts b/backend/src/routes.ts index c1c31e28a..78584326c 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -1,6 +1,5 @@ -import path from "node:path"; - import gateway from "./gateway.ts"; +import { diagnosticsSnapshot, livenessSnapshot, readinessSnapshot } from "./health.ts"; import { json } from "./http.ts"; import { withRequestPolicy } from "./requestPolicy.ts"; import { accountSecurityRoutes } from "./routes/accountSecurityRoutes.ts"; @@ -32,44 +31,39 @@ import { sttRoutes } from "./routes/sttRoutes.ts"; import { taskRoutes } from "./routes/taskRoutes.ts"; import { terminalRoutes } from "./routes/terminalRoutes.ts"; import { ttsRoutes } from "./routes/ttsRoutes.ts"; -import { getJobExecutionSummary } from "./services/jobExecutionQueue.ts"; +function live() { + return json(livenessSnapshot()); +} -const BACKEND_COMMIT = (() => { - try { - return ( - Bun.spawnSync(["git", "rev-parse", "--short", "HEAD"], { - cwd: path.join(import.meta.dirname, ".."), - stderr: "ignore", - }) - .stdout?.toString() - ?.trim() || "unknown" - ); - } catch { - return "unknown"; - } -})(); +async function ready() { + const snapshot = await readinessSnapshot(); + return json(snapshot, { status: snapshot.status === "isReady" ? 200 : 503 }); +} -function backendCommit(): string { - return BACKEND_COMMIT; +async function legacyReady() { + const snapshot = await readinessSnapshot(); + const isReady = snapshot.status === "isReady"; + return json( + { + status: isReady ? "isOk" : "notReady", + workerOnline: snapshot.checks.worker.ready, + }, + { status: isReady ? 200 : 503 } + ); } -function isWorkerOnline(): boolean { - try { - return getJobExecutionSummary().workerOnline; - } catch (error) { - console.warn("[Health] Failed to read job worker telemetry:", error); - return false; - } +async function diagnostics() { + return json(await diagnosticsSnapshot()); } -function health() { - return json({ - status: "isOk", - gatewayConnected: gateway.isConnected(), - sessionCount: gateway.getSessions().length, - backendCommit: backendCommit() || "unknown", - workerOnline: isWorkerOnline(), - }); +function retiredHealth() { + return json( + { + error: "Gone", + replacements: ["/api/health/live", "/api/health/ready"], + }, + { status: 410 } + ); } function sessions() { @@ -78,10 +72,25 @@ function sessions() { const routeTable = { "/health": { - GET: health, + GET: retiredHealth, + HEAD: retiredHealth, }, + "/api/health/diagnostics": { + GET: diagnostics, + }, + // Transitional compatibility for the in-flight pre-readiness deploy + // executor. Remove after the atomic release executor has completed cutover. "/api/health": { - GET: health, + GET: legacyReady, + HEAD: legacyReady, + }, + "/api/health/live": { + GET: live, + HEAD: live, + }, + "/api/health/ready": { + GET: ready, + HEAD: ready, }, "/api/sessions": { GET: sessions, diff --git a/backend/src/server.ts b/backend/src/server.ts index 0d3486237..965d6f8e2 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -12,6 +12,7 @@ import { } from "./auth.ts"; import { validateAutomationCredentials } from "./automationAuth.ts"; import type { DashboardSocket } from "./dashboardSocket.ts"; +import { resolveFrontendPath } from "./frontendAssets.ts"; import gateway from "./gateway.ts"; import { isAllowedDashboardOrigin, sessionIdFromCookie } from "./http.ts"; import { requiresRecentMfaForGatewayMethod } from "./requestPolicy.ts"; @@ -75,13 +76,6 @@ function hasHiddenStaticSegment(relativePath: string): boolean { return relativePath.split(path.sep).some((segment) => segment.startsWith(".")); } -function resolveFrontendPath(): string { - return ( - process.env.MIRA_DASHBOARD_FRONTEND_PATH || - path.join(import.meta.dirname, "..", "..", "dist") - ); -} - export function resolveListenPort(value = process.env.PORT): number { const trimmed = value?.trim() ?? ""; if (!/^\d+$/u.test(trimmed)) { @@ -118,6 +112,7 @@ export function createServer(port = resolveListenPort()): Server) { for (const handler of ws.data.closeHandlers) { diff --git a/backend/src/serverStart.ts b/backend/src/serverStart.ts index feb7bfd9b..fc07bd2e8 100644 --- a/backend/src/serverStart.ts +++ b/backend/src/serverStart.ts @@ -1,5 +1,9 @@ import { getPersistedGatewayToken } from "./auth.ts"; import gateway from "./gateway.ts"; +import { + getRuntimeReleaseIdentity, + requireRunnableReleaseCommit, +} from "./releaseManifest.ts"; import { createServer, resolveListenPort } from "./server.ts"; import { shouldStartScheduledJobs } from "./serverStartPolicy.ts"; import { startDashboardJobWorker, stopDashboardJobWorker } from "./services/jobWorker.ts"; @@ -7,10 +11,10 @@ import { registerPullRequestJobLifecycleHandlers } from "./services/pullRequests const serverStartState: { activeServer: ReturnType | undefined; - isStarting: boolean; + startupPromise: Promise | undefined; } = { activeServer: undefined, - isStarting: false, + startupPromise: undefined, }; export { runLogRotationCli } from "./services/logRotation.ts"; @@ -42,7 +46,7 @@ export function resolveGatewayToken( } /** Starts Gateway and notification monitors after the HTTP server is listening. */ -export function handleServerListening(): void { +export function handleServerListening(releaseCommit: string): void { let isGatewayStarted = false; try { registerPullRequestJobLifecycleHandlers(); @@ -57,7 +61,7 @@ export function handleServerListening(): void { } if (shouldStartScheduledJobs()) { - startDashboardJobWorker(); + startDashboardJobWorker(releaseCommit); } } catch (error) { console.error("[Backend] Failed to start background services:", error); @@ -79,22 +83,34 @@ export function handleServerListening(): void { } /** Binds the HTTP server and starts runtime-only background services. */ -export function startBackendServer(port = resolveListenPort()): void { - if (serverStartState.activeServer || serverStartState.isStarting) { - return; +export function startBackendServer(port = resolveListenPort()): Promise { + if (serverStartState.activeServer) { + return Promise.resolve(); } - serverStartState.isStarting = true; - try { - serverStartState.activeServer = createServer(port); - handleServerListening(); - serverStartState.isStarting = false; - } catch (error) { - serverStartState.isStarting = false; - serverStartState.activeServer = undefined; - console.error("[Backend] Failed to start server:", error); - process.exitCode = 1; - throw error; + if (serverStartState.startupPromise) { + return serverStartState.startupPromise; } + const startup = Promise.withResolvers(); + serverStartState.startupPromise = startup.promise; + void (async () => { + try { + const release = await getRuntimeReleaseIdentity(); + const releaseCommit = requireRunnableReleaseCommit(release, "Backend"); + serverStartState.activeServer = createServer(port); + handleServerListening(releaseCommit); + startup.resolve(); + } catch (error) { + serverStartState.activeServer = undefined; + console.error("[Backend] Failed to start server:", error); + process.exitCode = 1; + startup.reject(error); + } finally { + if (serverStartState.startupPromise === startup.promise) { + serverStartState.startupPromise = undefined; + } + } + })(); + return startup.promise; } export async function stopBackendServer(): Promise { @@ -124,7 +140,7 @@ interface BackendServerEntrypointOptions { isDirect?: boolean; reportFailure?: (error: unknown) => void; runServer?: () => Promise; - startServer?: () => void; + startServer?: () => Promise | void; startOnImport?: string; } @@ -140,7 +156,7 @@ export async function runBackendServer(port = resolveListenPort()): Promise; const oldestQueuedAt = fromSqlNullable(counts.oldest_queued_at); const parsedOldestQueuedAt = oldestQueuedAt ? Date.parse(oldestQueuedAt) : NaN; - const workerFreshAfter = new Date(timestamp - 30_000).toISOString(); + const workerFreshAfter = new Date( + timestamp - WORKER_HEARTBEAT_MAX_AGE_MS + ).toISOString(); const worker = database .prepare( `SELECT COUNT(*) AS count, @@ -460,6 +464,26 @@ export function getJobExecutionSummary(timestamp = Date.now()): JobExecutionSumm }; } +export function isJobWorkerReleaseReady( + releaseCommit: string, + timestamp = Date.now() +): boolean { + if (!RELEASE_COMMIT_PATTERN.test(releaseCommit)) { + return false; + } + const freshAfter = new Date(timestamp - WORKER_HEARTBEAT_MAX_AGE_MS).toISOString(); + const row = database + .prepare( + `SELECT 1 + FROM job_workers + WHERE heartbeat_at >= ? + AND id LIKE ? + LIMIT 1` + ) + .get(freshAfter, `dashboard-worker:${releaseCommit}:%`); + return row !== null && row !== undefined; +} + export function registerJobWorker( id: string, capacity: number, diff --git a/backend/src/services/jobWorker.ts b/backend/src/services/jobWorker.ts index 1ee6de436..447c6425d 100644 --- a/backend/src/services/jobWorker.ts +++ b/backend/src/services/jobWorker.ts @@ -61,12 +61,12 @@ function registerScheduledActions(): void { } /** Starts the persistent queue scheduler and its single-concurrency executor. */ -export function startDashboardJobWorker(): void { +export function startDashboardJobWorker(releaseCommit = "development"): void { if (workerState.isStarted || workerState.pendingStop) return; workerState.isStarted = true; try { registerScheduledActions(); - startScheduledJobExecutor(); + startScheduledJobExecutor(releaseCommit); startScheduledJobScheduler(); } catch (error) { stopScheduledJobScheduler(); diff --git a/backend/src/services/logRotation.ts b/backend/src/services/logRotation.ts index a55ee1e09..0a365dee9 100644 --- a/backend/src/services/logRotation.ts +++ b/backend/src/services/logRotation.ts @@ -108,10 +108,6 @@ function caughtMessage(error: unknown): string { } function defaultConfigPath(): string { - const configured = process.env.MIRA_LOG_ROTATION_CONFIG; - if (configured?.trim()) { - return configured; - } if (fsSyncExists(CWD_CONFIG_PATH)) { return CWD_CONFIG_PATH; } @@ -2035,17 +2031,9 @@ function buildElevatedLogRotationCliArguments( } function elevatedLogRotationEnvironment(): NodeJS.ProcessEnv { - const allowed = [ - "PATH", - "HOME", - "LANG", - "NODE_ENV", - "TZ", - "MIRA_DASHBOARD_DB_PATH", - "MIRA_LOG_ROTATION_CONFIG", - ]; + const allowed = ["PATH", "HOME", "LANG", "NODE_ENV", "TZ", "MIRA_DASHBOARD_DB_PATH"]; const environment: NodeJS.ProcessEnv = {}; - // Keep sudo -E narrow: only runtime lookup, home/locale, mode, and config path. + // Keep sudo -E narrow: only runtime lookup, home/locale, mode, and database path. for (const key of allowed) { if (process.env[key] !== undefined) { environment[key] = process.env[key]; @@ -2057,7 +2045,6 @@ function elevatedLogRotationEnvironment(): NodeJS.ProcessEnv { export async function runLogRotationCli(): Promise { try { const summary = await runLogRotationService({ - config: process.env.MIRA_LOG_ROTATION_CONFIG, isDryRun: process.argv.includes("--dry-run"), }); if (process.argv.includes("--json")) { diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index 7a8ed0f86..17c91dc66 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -1472,8 +1472,17 @@ async function scheduleRestartHealthCheck( "sleep 2", "restart_status=0", `systemctl --user restart ${DASHBOARD_SERVICES.join(" ")} || restart_status=$?`, - "sleep 4", - `if [ "$restart_status" -eq 0 ] && curl -fsS http://127.0.0.1:3100/api/health | grep -Fq '"workerOnline":true'; then`, + "health_status=1", + 'if [ "$restart_status" -eq 0 ]; then', + " for attempt in {1..20}; do", + " if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 http://127.0.0.1:3100/api/health/ready >/dev/null; then", + " health_status=0", + " break", + " fi", + " sleep 1", + " done", + "fi", + `if [ "$restart_status" -eq 0 ] && [ "$health_status" -eq 0 ]; then`, ` ${deploymentJobUpdateCommand(okJob)}`, "else", ` ${deploymentJobUpdateCommand(failedJob)}`, diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index ab90ca1cf..fe0c83a5a 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -34,6 +34,7 @@ const executorTickMs = 1000; const executorHeartbeatMs = 1000; const executorCapacity = 1; const interruptedHandlerGraceMs = 30_000; +const RELEASE_COMMIT_PATTERN = /^(?:[\da-f]{8,40}|development)$/u; const actionHandlers = new Map(); const interruptedHandlerSettled = new WeakMap< ScheduledJobInterruptionError, @@ -59,7 +60,7 @@ const scheduledJobRuntimeState: { isSchedulerTickRunning: false, isExecutorClaimingPaused: false, isExecutorTickRunning: false, - workerId: `dashboard-worker:${process.pid}:${Bun.randomUUIDv7()}`, + workerId: "", }; export type ScheduledJobScheduleType = "interval" | "daily" | "cron"; @@ -1302,8 +1303,16 @@ export function startScheduledJobScheduler(): void { scheduleTick(); } -export function startScheduledJobExecutor(): void { +function workerIdForRelease(releaseCommit: string): string { + if (!RELEASE_COMMIT_PATTERN.test(releaseCommit)) { + throw new Error("Job worker release commit must be an 8-40 character SHA"); + } + return `dashboard-worker:${releaseCommit}:${process.pid}:${Bun.randomUUIDv7()}`; +} + +export function startScheduledJobExecutor(releaseCommit = "development"): void { if (scheduledJobRuntimeState.executor) return; + scheduledJobRuntimeState.workerId = workerIdForRelease(releaseCommit); resetExecutorClaimPause(); const timestamp = nowIso(); const recoveredLegacyRuns = recoverOrphanedScheduledJobRuns(timestamp); diff --git a/backend/src/workerStart.ts b/backend/src/workerStart.ts index b0dc27690..4df325e95 100644 --- a/backend/src/workerStart.ts +++ b/backend/src/workerStart.ts @@ -1,4 +1,8 @@ import { validateAuthenticationConfig, validateStoredSecretConfig } from "./auth.ts"; +import { + getRuntimeReleaseIdentity, + requireRunnableReleaseCommit, +} from "./releaseManifest.ts"; import { startDashboardJobWorker, stopDashboardJobWorker } from "./services/jobWorker.ts"; const WORKER_KEEP_ALIVE_INTERVAL_MS = 60_000; @@ -15,6 +19,8 @@ export function createWorkerKeepAliveHandle(): NodeJS.Timeout { } export async function runDashboardWorker(): Promise { + const release = await getRuntimeReleaseIdentity(); + const releaseCommit = requireRunnableReleaseCommit(release, "Worker"); validateAuthenticationConfig(); validateStoredSecretConfig(); const shutdown = Promise.withResolvers(); @@ -23,7 +29,7 @@ export async function runDashboardWorker(): Promise { process.once("SIGINT", stop); process.once("SIGTERM", stop); try { - startDashboardJobWorker(); + startDashboardJobWorker(releaseCommit); await shutdown.promise; } finally { process.removeListener("SIGINT", stop); diff --git a/backend/test/buildSourceIdentity.test.ts b/backend/test/buildSourceIdentity.test.ts new file mode 100644 index 000000000..cffc6541f --- /dev/null +++ b/backend/test/buildSourceIdentity.test.ts @@ -0,0 +1,74 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "bun:test"; + +import { + isReleaseBuildCommit, + resolveBuildSourceIdentity, +} from "../scripts/buildSourceIdentity.ts"; + +const temporaryRoots: string[] = []; + +function runGit(repoDirectory: string, arguments_: string[]): string { + const result = Bun.spawnSync({ + cmd: [ + "git", + "-C", + repoDirectory, + "-c", + "user.name=Mira build test", + "-c", + "user.email=mira-build-test@example.invalid", + ...arguments_, + ], + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + if (result.exitCode !== 0) { + throw new Error(new TextDecoder().decode(result.stderr)); + } + return new TextDecoder().decode(result.stdout).trim(); +} + +afterEach(() => { + const rootsToRemove = [...temporaryRoots]; + temporaryRoots.length = 0; + for (const root of rootsToRemove) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("release build source identity", () => { + it("marks tracked and untracked source changes as non-release builds", () => { + const root = mkdtempSync(path.join(tmpdir(), "mira-build-identity-")); + temporaryRoots.push(root); + mkdirSync(path.join(root, "src")); + writeFileSync(path.join(root, "src", "entry.ts"), "export const value = 1;\n"); + runGit(root, ["init", "--initial-branch=main"]); + runGit(root, ["add", "."]); + runGit(root, ["commit", "-m", "Initial source"]); + const commit = runGit(root, ["rev-parse", "HEAD"]); + + expect(resolveBuildSourceIdentity(root)).toBe(commit); + expect(isReleaseBuildCommit(commit)).toBe(true); + + writeFileSync(path.join(root, "src", "entry.ts"), "export const value = 2;\n"); + expect(resolveBuildSourceIdentity(root)).toBe(`${commit}-dirty`); + expect(isReleaseBuildCommit(`${commit}-dirty`)).toBe(false); + + runGit(root, ["restore", "src/entry.ts"]); + writeFileSync(path.join(root, "src", "new.ts"), "export {};\n"); + expect(resolveBuildSourceIdentity(root)).toBe(`${commit}-dirty`); + }); + + it("returns unknown outside a readable Git source tree", () => { + const root = mkdtempSync(path.join(tmpdir(), "mira-build-identity-")); + temporaryRoots.push(root); + + expect(resolveBuildSourceIdentity(root)).toBe("unknown"); + expect(isReleaseBuildCommit("unknown")).toBe(false); + }); +}); diff --git a/backend/test/bunNativeServerBehavior.test.ts b/backend/test/bunNativeServerBehavior.test.ts index aaaacf97e..c9594976b 100644 --- a/backend/test/bunNativeServerBehavior.test.ts +++ b/backend/test/bunNativeServerBehavior.test.ts @@ -324,10 +324,16 @@ describe("Bun-native dashboard backend", () => { }); it("reports health and auth bootstrap state", async () => { - const health = await api<{ status: string; sessionCount: number }>("/api/health"); - expect(health.status).toBe(200); - expect(health.body.status).toBe("isOk"); - expect(health.body.sessionCount).toBe(0); + const live = await api<{ status: string; uptimeSeconds: number }>( + "/api/health/live" + ); + expect(live.status).toBe(200); + expect(live.body.status).toBe("isOk"); + expect(live.body.uptimeSeconds).toBeGreaterThanOrEqual(0); + const liveHead = await api("/api/health/live", { + method: "HEAD", + }); + expect(liveHead).toEqual({ body: undefined, status: 200 }); const bootstrap = await api<{ hasGatewayToken: boolean; @@ -507,6 +513,17 @@ describe("Bun-native dashboard backend", () => { }); it("serves the app shell and hashed static assets", async () => { + const retiredHealth = await fetch(`${state.baseUrl}/health`); + expect(retiredHealth.status).toBe(410); + await expect(retiredHealth.json()).resolves.toEqual({ + error: "Gone", + replacements: ["/api/health/live", "/api/health/ready"], + }); + const retiredHealthHead = await fetch(`${state.baseUrl}/health`, { + method: "HEAD", + }); + expect(retiredHealthHead.status).toBe(410); + const appRoute = await fetch(`${state.baseUrl}/tasks`); expect(appRoute.status).toBe(200); expect(appRoute.headers.get("content-type")).toContain("text/html"); diff --git a/backend/test/databaseLifecycle.test.ts b/backend/test/databaseLifecycle.test.ts index c943c41b9..5180fd432 100644 --- a/backend/test/databaseLifecycle.test.ts +++ b/backend/test/databaseLifecycle.test.ts @@ -508,7 +508,7 @@ describe("Dashboard SQLite lifecycle", () => { } }); - it("fails closed on checksum drift and unknown migration versions", () => { + it("fails closed on drift and permits only explicitly compatible future migrations", () => { const root = temporaryRoot("mira-db-migrations-drift-"); const databasePath = path.join(root, "dashboard.db"); const database = openWalDatabase(databasePath); @@ -536,7 +536,16 @@ describe("Dashboard SQLite lifecycle", () => { ) .run("2026-07-23T00:00:00.000Z"); expect(() => validateDatabaseMigrationHistory(database)).toThrow( - "unknown SQLite migration version 7" + "incompatible SQLite migration version 7" + ); + database + .prepare( + "UPDATE schema_migrations SET name = ?, checksum = ? WHERE version = 7" + ) + .run("future-additive", "a".repeat(64)); + expect(validateDatabaseMigrationHistory(database, 7)).toBe(7); + expect(() => validateDatabaseMigrationHistory(database, 6)).toThrow( + "incompatible SQLite migration version 7" ); database.prepare("DELETE FROM schema_migrations WHERE version = 7").run(); diff --git a/backend/test/healthReadiness.test.ts b/backend/test/healthReadiness.test.ts new file mode 100644 index 000000000..d8c1d0631 --- /dev/null +++ b/backend/test/healthReadiness.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "bun:test"; + +import { isDatabaseSchemaCompatible } from "../src/databaseSchemaCompatibility.ts"; +import { evaluateReadiness, type ReadinessSignals } from "../src/health.ts"; + +function readySignals(): ReadinessSignals { + return { + database: { + currentSchemaVersion: 6, + maximumCompatibleSchemaVersion: 6, + minimumCompatibleSchemaVersion: 6, + ready: true, + targetSchemaVersion: 6, + }, + frontendReady: true, + gatewayConnected: true, + release: { + artifactCount: 8, + backendCommit: "aaaaaaaa", + commitSha: "a".repeat(40), + frontendCommit: "aaaaaaaa", + manifestFormatVersion: 1, + ready: true, + source: "manifest", + }, + sessionCount: 4, + workerReady: true, + }; +} + +describe("Dashboard readiness contract", () => { + it("requires release, database, frontend, and worker readiness", () => { + const ready = evaluateReadiness(readySignals()); + expect(ready).toMatchObject({ + checks: { + database: { ready: true }, + frontend: { ready: true }, + release: { ready: true }, + worker: { ready: true }, + }, + status: "isReady", + }); + expect(ready.checks.release).toEqual({ + backendCommit: "aaaaaaaa", + frontendCommit: "aaaaaaaa", + manifestFormatVersion: 1, + ready: true, + source: "manifest", + }); + expect("commitSha" in ready.checks.release).toBe(false); + expect("schema" in ready.checks.release).toBe(false); + + const baseline = readySignals(); + const blockedSignals: ReadinessSignals[] = [ + { + ...baseline, + database: { ...baseline.database, ready: false }, + }, + { ...baseline, frontendReady: false }, + { + ...baseline, + release: { ...baseline.release, ready: false }, + }, + { ...baseline, workerReady: false }, + ]; + for (const signals of blockedSignals) { + expect(evaluateReadiness(signals).status).toBe("notReady"); + } + }); + + it("reports Gateway availability without using it as a rollback signal", () => { + const signals = readySignals(); + signals.gatewayConnected = false; + + expect(evaluateReadiness(signals)).toMatchObject({ + dependencies: { + gatewayConnected: false, + }, + status: "isReady", + }); + }); + + it("accepts only database schemas inside the explicit rollback window", () => { + expect(isDatabaseSchemaCompatible(7, { maximum: 7, minimum: 6 })).toBe(true); + expect(isDatabaseSchemaCompatible(5, { maximum: 7, minimum: 6 })).toBe(false); + expect(isDatabaseSchemaCompatible(8, { maximum: 7, minimum: 6 })).toBe(false); + }); +}); diff --git a/backend/test/httpApiBehavior.test.ts b/backend/test/httpApiBehavior.test.ts index 1b4127244..1c0744dab 100644 --- a/backend/test/httpApiBehavior.test.ts +++ b/backend/test/httpApiBehavior.test.ts @@ -310,15 +310,45 @@ describe("Mira Dashboard backend integration", () => { }); it("reports health and auth bootstrap state without production data", async () => { - const health = await api<{ + const live = await api<{ status: string; uptimeSeconds: number }>( + "/api/health/live" + ); + expect(live.status).toBe(200); + expect(live.body.status).toBe("isOk"); + expect(live.body.uptimeSeconds).toBeGreaterThanOrEqual(0); + + const liveHead = await api("/api/health/live", { + method: "HEAD", + }); + expect(liveHead).toEqual({ body: undefined, status: 200 }); + + const ready = await api<{ status: string }>("/api/health/ready"); + expect(ready.status).toBe(503); + expect(ready.body.status).toBe("notReady"); + const readyHead = await api("/api/health/ready", { + method: "HEAD", + }); + expect(readyHead).toEqual({ body: undefined, status: 503 }); + + const legacyReady = await api<{ status: string; - sessionCount: number; workerOnline: boolean; }>("/api/health"); - expect(health.status).toBe(200); - expect(health.body.status).toBe("isOk"); - expect(health.body.sessionCount).toBe(0); - expect(health.body.workerOnline).toBe(false); + expect(legacyReady).toEqual({ + body: { + status: "notReady", + workerOnline: false, + }, + status: 503, + }); + const legacyReadyHead = await api("/api/health", { + method: "HEAD", + }); + expect(legacyReadyHead).toEqual({ body: undefined, status: 503 }); + + const diagnostics = await api<{ error: string }>("/api/health/diagnostics"); + expect(diagnostics.status).toBe(401); + expect(diagnostics.body).toEqual({ error: "Unauthorized" }); const bootstrap = await api<{ isBootstrapRequired: boolean; diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts index 66c61c27a..689b1b4a3 100644 --- a/backend/test/jobExecutionQueue.test.ts +++ b/backend/test/jobExecutionQueue.test.ts @@ -15,6 +15,7 @@ import { getJobExecution, getJobExecutionSummary, insertJobExecution, + isJobWorkerReleaseReady, protectRunningJobExecutionFromCancellation, recoverExpiredJobExecutions, registerJobWorker, @@ -162,6 +163,35 @@ describe("persistent job execution queue", () => { unregisterJobWorker(workerId); }); + it("requires a fresh heartbeat from the requested worker release", () => { + const releaseCommit = "a".repeat(40); + const workerId = `dashboard-worker:${releaseCommit}:123:${Bun.randomUUIDv7()}`; + registerJobWorker(workerId, 1, "2026-07-22T10:00:00.000Z"); + try { + expect( + isJobWorkerReleaseReady( + releaseCommit, + Date.parse("2026-07-22T10:00:29.000Z") + ) + ).toBe(true); + expect( + isJobWorkerReleaseReady( + releaseCommit, + Date.parse("2026-07-22T10:00:31.000Z") + ) + ).toBe(false); + expect( + isJobWorkerReleaseReady( + "b".repeat(40), + Date.parse("2026-07-22T10:00:29.000Z") + ) + ).toBe(false); + expect(isJobWorkerReleaseReady("not-a-commit")).toBe(false); + } finally { + unregisterJobWorker(workerId); + } + }); + it("wraps worker children in class-specific systemd scopes", () => { const command = withJobResourceClass("host-heavy", () => scopedJobProcessCommand( diff --git a/backend/test/openClawChatBridge.test.ts b/backend/test/openClawChatBridge.test.ts index 29cd61d89..e61f800a3 100644 --- a/backend/test/openClawChatBridge.test.ts +++ b/backend/test/openClawChatBridge.test.ts @@ -3328,22 +3328,29 @@ describe("OpenClaw chat bridge", () => { const provisionalRunId = "dashboard-chat-quiet-before-restart"; const providerRunId = "provider-after-quiet-restart"; const disconnectedAt = 1_785_000_000_000; + const hydratedAt = disconnectedAt - 60 * 60_000 - 1; const providerStartedAt = disconnectedAt + 1000; - store.snapshots.set( - MAIN, - persistedSnapshot(MAIN, provisionalRunId, disconnectedAt - 7 * 60 * 60_000) - ); - const bridge = new OpenClawChatBridge(store); - - bridge.snapshot(MAIN); - bridge.markGatewayDisconnected(disconnectedAt); - expect(bridge.flush()).toBe(true); - expect(store.snapshots.get(MAIN)?.interruptedAtByRun).toEqual({ - [provisionalRunId]: disconnectedAt, - }); - const restarted = new OpenClawChatBridge(store); - const dateNow = jest.spyOn(Date, "now").mockReturnValue(providerStartedAt); + const dateNow = jest.spyOn(Date, "now"); try { + dateNow.mockReturnValue(hydratedAt); + store.snapshots.set( + MAIN, + persistedSnapshot( + MAIN, + provisionalRunId, + disconnectedAt - 7 * 60 * 60_000 + ) + ); + const bridge = new OpenClawChatBridge(store); + + bridge.snapshot(MAIN); + bridge.markGatewayDisconnected(disconnectedAt); + expect(bridge.flush()).toBe(true); + expect(store.snapshots.get(MAIN)?.interruptedAtByRun).toEqual({ + [provisionalRunId]: disconnectedAt, + }); + const restarted = new OpenClawChatBridge(store); + dateNow.mockReturnValue(providerStartedAt); restarted.recordEvent( "agent", { @@ -3354,15 +3361,15 @@ describe("OpenClaw chat bridge", () => { }, [] ); + + expect( + restarted + .snapshot(MAIN) + .events.map((event) => (event.payload as { runId?: string }).runId) + ).toEqual([providerRunId, providerRunId]); } finally { dateNow.mockRestore(); } - - expect( - restarted - .snapshot(MAIN) - .events.map((event) => (event.payload as { runId?: string }).runId) - ).toEqual([providerRunId, providerRunId]); }); it("does not join an interrupted run across a newer chat send", () => { diff --git a/backend/test/releaseManifest.test.ts b/backend/test/releaseManifest.test.ts new file mode 100644 index 000000000..7a5a871bd --- /dev/null +++ b/backend/test/releaseManifest.test.ts @@ -0,0 +1,512 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "bun:test"; + +import { + createReleaseManifest, + DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY, + getRuntimeReleaseIdentity, + invalidateRuntimeReleaseIdentityCache, + loadReleaseManifest, + loadRuntimeReleaseIdentity, + parseReleaseManifest, + RELEASE_MANIFEST_FILE_NAME, + requireRunnableReleaseCommit, + verifyReleaseArtifacts, + writeReleaseManifest, +} from "../src/releaseManifest.ts"; + +const temporaryRoots: string[] = []; +const TEST_COMMIT = "a".repeat(40); +const TEST_BUILT_AT = new Date("2026-07-25T15:00:00.000Z"); +const TEST_BUN_VERSION = "1.3.14"; + +function writeTestBuildIdentities(root: string, commitSha = TEST_COMMIT): void { + writeFileSync( + path.join(root, "dist", "build-identity.json"), + `${JSON.stringify({ + bunVersion: TEST_BUN_VERSION, + commitSha, + component: "frontend", + formatVersion: 1, + })}\n` + ); + writeFileSync( + path.join(root, "backend", "dist", "build-identity.json"), + `${JSON.stringify({ + bunVersion: TEST_BUN_VERSION, + commitSha, + component: "backend", + formatVersion: 1, + })}\n` + ); +} + +function temporaryReleaseRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), "mira-release-manifest-")); + temporaryRoots.push(root); + mkdirSync(path.join(root, "backend", "dist"), { recursive: true }); + mkdirSync(path.join(root, "backend", "config"), { recursive: true }); + mkdirSync(path.join(root, "dist", "assets"), { recursive: true }); + writeFileSync(path.join(root, "package.json"), "{}\n"); + writeFileSync(path.join(root, "bun.lock"), "root-lock\n"); + writeFileSync(path.join(root, "backend", "package.json"), "{}\n"); + writeFileSync(path.join(root, "backend", "bun.lock"), "backend-lock\n"); + writeFileSync( + path.join(root, "backend", "config", "log-rotation.json"), + '{"jobs":[]}\n' + ); + writeFileSync(path.join(root, "dist", "index.html"), "
release
\n"); + writeFileSync(path.join(root, "dist", "assets", "app.js"), "export {};\n"); + writeTestBuildIdentities(root); + writeFileSync( + path.join(root, "backend", "dist", "databasePreflight.js"), + "export {};\n" + ); + writeFileSync( + path.join(root, "backend", "dist", "resetDashboardPassword.js"), + "export {};\n" + ); + writeFileSync(path.join(root, "backend", "dist", "serverStart.js"), "export {};\n"); + writeFileSync(path.join(root, "backend", "dist", "workerStart.js"), "export {};\n"); + return root; +} + +function manifestOptions(releaseRoot: string) { + return { + builtAt: TEST_BUILT_AT, + bunVersion: TEST_BUN_VERSION, + commitSha: TEST_COMMIT, + commitTitle: "Test atomic release", + releaseRoot, + }; +} + +function runGit(releaseRoot: string, arguments_: string[]): string { + const result = Bun.spawnSync({ + cmd: [ + "git", + "-C", + releaseRoot, + "-c", + "user.name=Mira release test", + "-c", + "user.email=mira-release-test@example.invalid", + ...arguments_, + ], + stderr: "pipe", + stdin: "ignore", + stdout: "pipe", + }); + if (result.exitCode !== 0) { + throw new Error(new TextDecoder().decode(result.stderr)); + } + return new TextDecoder().decode(result.stdout).trim(); +} + +afterEach(() => { + invalidateRuntimeReleaseIdentityCache(); + const rootsToRemove = [...temporaryRoots]; + temporaryRoots.length = 0; + for (const root of rootsToRemove) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("Dashboard release manifest", () => { + it("accepts only runtime identities that can safely start services", () => { + expect( + requireRunnableReleaseCommit( + { + backendCommit: TEST_COMMIT, + commitSha: TEST_COMMIT, + frontendCommit: TEST_COMMIT, + ready: true, + source: "manifest", + }, + "Backend", + "production" + ) + ).toBe(TEST_COMMIT); + expect(() => + requireRunnableReleaseCommit( + { + backendCommit: TEST_COMMIT, + frontendCommit: TEST_COMMIT, + issue: "manifest-invalid", + ready: false, + source: "manifest", + }, + "Worker", + "production" + ) + ).toThrow("Worker release identity is not ready (manifest-invalid)"); + expect(() => + requireRunnableReleaseCommit( + { + backendCommit: "unknown", + frontendCommit: "unknown", + ready: true, + source: "unknown", + }, + "Backend", + "development" + ) + ).toThrow("Backend release identity does not contain a valid commit"); + }); + + it("builds a deterministic manifest for every runtime artifact", async () => { + const root = temporaryReleaseRoot(); + const manifest = await createReleaseManifest(manifestOptions(root)); + + expect(manifest).toMatchObject({ + builtAt: TEST_BUILT_AT.toISOString(), + bunVersion: TEST_BUN_VERSION, + commitSha: TEST_COMMIT, + commitShort: "aaaaaaaa", + commitTitle: "Test atomic release", + components: { + backendCommit: "aaaaaaaa", + frontendCommit: "aaaaaaaa", + }, + formatVersion: 1, + schema: { + maximumCompatible: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum, + minimumCompatible: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum, + target: DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.target, + }, + }); + expect(manifest.artifacts.map((artifact) => artifact.path)).toEqual([ + "backend/bun.lock", + "backend/config/log-rotation.json", + "backend/dist/build-identity.json", + "backend/dist/databasePreflight.js", + "backend/dist/resetDashboardPassword.js", + "backend/dist/serverStart.js", + "backend/dist/workerStart.js", + "backend/package.json", + "bun.lock", + "dist/assets/app.js", + "dist/build-identity.json", + "dist/index.html", + "package.json", + ]); + expect( + manifest.artifacts.every( + (artifact) => + /^[\da-f]{64}$/u.test(artifact.sha256) && artifact.sizeBytes > 0 + ) + ).toBe(true); + }); + + it("writes, reloads, and verifies a complete release", async () => { + const root = temporaryReleaseRoot(); + const written = await writeReleaseManifest(manifestOptions(root)); + const loaded = await loadReleaseManifest(root); + + expect(loaded).toEqual(written); + await expect(verifyReleaseArtifacts(root, loaded)).resolves.toBeUndefined(); + expect( + readFileSync(path.join(root, RELEASE_MANIFEST_FILE_NAME), "utf8") + ).toEndWith("\n"); + }); + + it("refuses to write a manifest larger than the loader accepts", async () => { + const root = temporaryReleaseRoot(); + const longName = "x".repeat(120); + for (let index = 0; index < 1200; index += 1) { + writeFileSync( + path.join( + root, + "dist", + "assets", + `${index.toString().padStart(4, "0")}-${longName}.js` + ), + "x" + ); + } + + await expect(writeReleaseManifest(manifestOptions(root))).rejects.toThrow( + "bounded regular file" + ); + expect(existsSync(path.join(root, RELEASE_MANIFEST_FILE_NAME))).toBe(false); + }); + + it("fails artifact verification after content or inventory drift", async () => { + const root = temporaryReleaseRoot(); + const manifest = await writeReleaseManifest(manifestOptions(root)); + + writeFileSync(path.join(root, "backend", "dist", "serverStart.js"), "tampered\n"); + await expect(verifyReleaseArtifacts(root, manifest)).rejects.toThrow( + "Release artifact verification failed: backend/dist/serverStart.js" + ); + + writeFileSync(path.join(root, "dist", "unexpected.js"), "unexpected\n"); + await expect(verifyReleaseArtifacts(root, manifest)).rejects.toThrow( + "Release artifact inventory does not match its manifest" + ); + }); + + it("rejects symlinked and malformed artifact declarations", async () => { + const root = temporaryReleaseRoot(); + symlinkSync( + path.join(root, "dist", "index.html"), + path.join(root, "dist", "assets", "linked.js") + ); + + await expect(createReleaseManifest(manifestOptions(root))).rejects.toThrow( + "Release artifact tree must not contain symlinks" + ); + + const validRoot = temporaryReleaseRoot(); + const manifest = await createReleaseManifest(manifestOptions(validRoot)); + expect(() => + parseReleaseManifest({ + ...manifest, + artifacts: [ + ...manifest.artifacts.slice(0, -1), + { + path: "../outside", + sha256: "b".repeat(64), + sizeBytes: 1, + }, + ], + }) + ).toThrow("Release manifest contains an invalid artifact"); + expect(() => + parseReleaseManifest({ + ...manifest, + schema: { + ...manifest.schema, + minimumCompatible: manifest.schema.target + 1, + }, + }) + ).toThrow("Release manifest schema range is invalid"); + }); + + it("requires the default runtime log-rotation configuration", async () => { + const root = temporaryReleaseRoot(); + rmSync(path.join(root, "backend", "config", "log-rotation.json")); + + await expect(createReleaseManifest(manifestOptions(root))).rejects.toThrow( + "ENOENT" + ); + }); + + it("requires every runtime and recovery entrypoint", async () => { + for (const relativePath of [ + "backend/dist/resetDashboardPassword.js", + "backend/dist/workerStart.js", + ]) { + const root = temporaryReleaseRoot(); + rmSync(path.join(root, relativePath)); + + await expect(createReleaseManifest(manifestOptions(root))).rejects.toThrow( + "Release manifest artifact inventory is invalid" + ); + } + }); + + it("derives identity only from a clean Git release source", async () => { + const root = temporaryReleaseRoot(); + writeFileSync( + path.join(root, ".gitignore"), + "/backend/dist/\n/dist/\n/release-manifest.json\n" + ); + runGit(root, ["init", "--initial-branch=main"]); + runGit(root, ["add", "."]); + runGit(root, ["commit", "-m", "Test release source"]); + const commitSha = runGit(root, ["rev-parse", "HEAD"]); + writeTestBuildIdentities(root, commitSha); + + await expect(createReleaseManifest({ releaseRoot: root })).resolves.toMatchObject( + { + commitTitle: "Test release source", + } + ); + + writeFileSync(path.join(root, "untracked-runtime.ts"), "export {};\n"); + await expect(createReleaseManifest({ releaseRoot: root })).rejects.toThrow( + "Release source contains uncommitted changes" + ); + }); + + it("refuses to stamp stale frontend or backend build outputs", async () => { + for (const component of ["frontend", "backend"] as const) { + const root = temporaryReleaseRoot(); + const identityPath = + component === "frontend" + ? path.join(root, "dist", "build-identity.json") + : path.join(root, "backend", "dist", "build-identity.json"); + writeFileSync( + identityPath, + `${JSON.stringify({ + bunVersion: TEST_BUN_VERSION, + commitSha: "b".repeat(40), + component, + formatVersion: 1, + })}\n` + ); + + await expect(createReleaseManifest(manifestOptions(root))).rejects.toThrow( + `${component} build identity does not match the release source` + ); + } + }); + + it("refuses to stamp build identities captured from dirty source", async () => { + for (const component of ["frontend", "backend"] as const) { + const root = temporaryReleaseRoot(); + const identityPath = + component === "frontend" + ? path.join(root, "dist", "build-identity.json") + : path.join(root, "backend", "dist", "build-identity.json"); + writeFileSync( + identityPath, + `${JSON.stringify({ + bunVersion: TEST_BUN_VERSION, + commitSha: `${TEST_COMMIT}-dirty`, + component, + formatVersion: 1, + })}\n` + ); + + await expect(createReleaseManifest(manifestOptions(root))).rejects.toThrow( + `${component} build identity is invalid` + ); + } + }); + + it("requires the deployed manifest to match the running code in production", async () => { + const root = temporaryReleaseRoot(); + const manifest = await writeReleaseManifest(manifestOptions(root)); + + await expect( + loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + backendCommit: "aaaaaaaa", + frontendCommit: "aaaaaaaa", + ready: true, + source: "manifest", + }); + await expect( + loadRuntimeReleaseIdentity(root, "production", "b".repeat(40)) + ).resolves.toMatchObject({ + issue: "manifest-code-mismatch", + ready: false, + source: "manifest", + }); + + writeFileSync( + path.join(root, RELEASE_MANIFEST_FILE_NAME), + `${JSON.stringify( + { + ...manifest, + schema: { + ...manifest.schema, + migrationRegistrySha256: "b".repeat(64), + }, + }, + undefined, + 2 + )}\n` + ); + await expect( + loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + issue: "manifest-code-mismatch", + ready: false, + source: "manifest", + }); + }); + + it("rejects a manifest built by another Bun runtime", async () => { + const root = temporaryReleaseRoot(); + const manifest = await writeReleaseManifest(manifestOptions(root)); + writeFileSync( + path.join(root, RELEASE_MANIFEST_FILE_NAME), + `${JSON.stringify( + { + ...manifest, + bunVersion: "0.0.0", + }, + undefined, + 2 + )}\n` + ); + + await expect( + loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + issue: "manifest-code-mismatch", + ready: false, + source: "manifest", + }); + }); + + it("fails runtime readiness when a declared artifact changes", async () => { + const root = temporaryReleaseRoot(); + await writeReleaseManifest(manifestOptions(root)); + writeFileSync(path.join(root, "backend", "dist", "serverStart.js"), "drift\n"); + + await expect( + loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + issue: "manifest-invalid", + ready: false, + }); + }); + + it("coalesces readiness verification and revalidates after invalidation", async () => { + const root = temporaryReleaseRoot(); + await writeReleaseManifest(manifestOptions(root)); + + const first = getRuntimeReleaseIdentity(root, "production", TEST_COMMIT); + const concurrent = getRuntimeReleaseIdentity(root, "production", TEST_COMMIT); + expect(concurrent).toBe(first); + await expect(first).resolves.toMatchObject({ + ready: true, + source: "manifest", + }); + + writeFileSync(path.join(root, "dist", "assets", "app.js"), "drift\n"); + await expect( + getRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + ready: true, + source: "manifest", + }); + + invalidateRuntimeReleaseIdentityCache(); + await expect( + getRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + issue: "manifest-invalid", + ready: false, + }); + }); + + it("allows only non-production Git fallback when a manifest is absent", async () => { + const root = temporaryReleaseRoot(); + + await expect( + loadRuntimeReleaseIdentity(root, "production", TEST_COMMIT) + ).resolves.toMatchObject({ + issue: "manifest-missing", + ready: false, + }); + await expect(loadRuntimeReleaseIdentity(root, "test")).resolves.toMatchObject({ + ready: true, + }); + }); +}); diff --git a/backend/test/serverStartupPolicy.test.ts b/backend/test/serverStartupPolicy.test.ts index 75096ed6d..7ac4e60a5 100644 --- a/backend/test/serverStartupPolicy.test.ts +++ b/backend/test/serverStartupPolicy.test.ts @@ -1,10 +1,14 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Server } from "bun"; import { describe, expect, it, jest } from "bun:test"; +import * as releaseManifestModule from "../src/releaseManifest.ts"; + +const TEST_RELEASE_COMMIT = "a".repeat(40); + describe("server start scheduler policy", () => { it("starts scheduled jobs unless explicitly disabled", async () => { const { shouldStartScheduledJobs } = await import("../src/serverStartPolicy.ts"); @@ -32,6 +36,43 @@ describe("server start scheduler policy", () => { ).toBe(true); }); + it("keeps production frontend assets inside the checksummed release", async () => { + const { resolveFrontendPath } = await import("../src/frontendAssets.ts"); + const releaseRoot = "/opt/mira-dashboard/releases/test-release"; + const releaseFrontend = path.join(releaseRoot, "dist"); + + expect(resolveFrontendPath({ NODE_ENV: "production" }, releaseRoot)).toBe( + releaseFrontend + ); + expect( + resolveFrontendPath( + { + MIRA_DASHBOARD_FRONTEND_PATH: releaseFrontend, + NODE_ENV: "production", + }, + releaseRoot + ) + ).toBe(releaseFrontend); + expect(() => + resolveFrontendPath( + { + MIRA_DASHBOARD_FRONTEND_PATH: "/tmp/unverified-frontend", + NODE_ENV: "production", + }, + releaseRoot + ) + ).toThrow("cannot override the checksummed release frontend"); + expect( + resolveFrontendPath( + { + MIRA_DASHBOARD_FRONTEND_PATH: "/tmp/development-frontend", + NODE_ENV: "development", + }, + releaseRoot + ) + ).toBe("/tmp/development-frontend"); + }); + it("resolves backend startup entrypoint and gateway token decisions without starting services", async () => { const { isDirectEntrypoint, @@ -187,6 +228,9 @@ describe("server start scheduler policy", () => { await expect(workerStart.runDashboardWorker()).rejects.toThrow( "worker startup failed" ); + expect(startSpy).toHaveBeenCalledWith( + expect.stringMatching(/^[\da-f]{8,40}$/u) + ); expect(stopSpy).toHaveBeenCalledTimes(1); expect(process.listenerCount("SIGINT")).toBe(sigintListeners); expect(process.listenerCount("SIGTERM")).toBe(sigtermListeners); @@ -196,6 +240,81 @@ describe("server start scheduler policy", () => { } }); + it("verifies the production worker release before opening SQLite", async () => { + const temporaryRoot = mkdtempSync(path.join(tmpdir(), "mira-worker-release-")); + const databasePath = path.join(temporaryRoot, "dashboard.db"); + const child = Bun.spawn({ + cmd: [ + process.execPath, + path.resolve(import.meta.dirname, "../src/workerStart.ts"), + ], + cwd: path.resolve(import.meta.dirname, ".."), + env: { + ...process.env, + MIRA_DASHBOARD_DB_PATH: databasePath, + MIRA_DASHBOARD_RELEASE_ROOT: temporaryRoot, + NODE_ENV: "production", + }, + stderr: "pipe", + stdin: "ignore", + stdout: "ignore", + }); + + try { + const [exitCode, stderr] = await Promise.all([ + child.exited, + new Response(child.stderr).text(), + ]); + + expect(exitCode).toBe(1); + expect(stderr).toContain( + "Worker release identity is not ready (manifest-missing)" + ); + expect(existsSync(databasePath)).toBe(false); + } finally { + child.kill(); + rmSync(temporaryRoot, { force: true, recursive: true }); + } + }); + + it("verifies the production backend release before opening SQLite", async () => { + const temporaryRoot = mkdtempSync(path.join(tmpdir(), "mira-backend-release-")); + const databasePath = path.join(temporaryRoot, "dashboard.db"); + const child = Bun.spawn({ + cmd: [ + process.execPath, + path.resolve(import.meta.dirname, "../src/serverStart.ts"), + ], + cwd: path.resolve(import.meta.dirname, ".."), + env: { + ...process.env, + MIRA_DASHBOARD_DB_PATH: databasePath, + MIRA_DASHBOARD_RELEASE_ROOT: temporaryRoot, + NODE_ENV: "production", + PORT: "0", + }, + stderr: "pipe", + stdin: "ignore", + stdout: "ignore", + }); + + try { + const [exitCode, stderr] = await Promise.all([ + child.exited, + new Response(child.stderr).text(), + ]); + + expect(exitCode).toBe(1); + expect(stderr).toContain( + "Backend release identity is not ready (manifest-missing)" + ); + expect(existsSync(databasePath)).toBe(false); + } finally { + child.kill(); + rmSync(temporaryRoot, { force: true, recursive: true }); + } + }); + it("keeps worker startup blocked until failed executor cleanup is retried", async () => { const backups = await import("../src/services/backups.ts"); const cacheRefresh = await import("../src/services/cacheRefresh.ts"); @@ -331,7 +450,7 @@ describe("server start scheduler policy", () => { const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); try { process.env.OPENCLAW_GATEWAY_TOKEN = " test-token "; - serverStartModule.handleServerListening(); + serverStartModule.handleServerListening(TEST_RELEASE_COMMIT); expect(initSpy).toHaveBeenCalledWith("test-token"); await new Promise((resolve) => setTimeout(resolve, 20)); } finally { @@ -354,6 +473,46 @@ describe("server start scheduler policy", () => { } }); + it("starts the combined worker with the verified release identity", async () => { + const originalGatewayToken = process.env.OPENCLAW_GATEWAY_TOKEN; + const originalSchedulerDisabled = process.env.MIRA_DASHBOARD_DISABLE_SCHEDULER; + const originalExecutionRole = process.env.MIRA_DASHBOARD_EXECUTION_ROLE; + process.env.OPENCLAW_GATEWAY_TOKEN = "test-token"; + delete process.env.MIRA_DASHBOARD_DISABLE_SCHEDULER; + process.env.MIRA_DASHBOARD_EXECUTION_ROLE = "combined"; + const gatewayModule = await import("../src/gateway.ts"); + const jobWorker = await import("../src/services/jobWorker.ts"); + const serverStartModule = await import("../src/serverStart.ts"); + const initSpy = jest + .spyOn(gatewayModule.default, "init") + .mockImplementation(() => {}); + const startWorkerSpy = jest + .spyOn(jobWorker, "startDashboardJobWorker") + .mockImplementation(() => {}); + try { + serverStartModule.handleServerListening(TEST_RELEASE_COMMIT); + expect(startWorkerSpy).toHaveBeenCalledWith(TEST_RELEASE_COMMIT); + } finally { + initSpy.mockRestore(); + startWorkerSpy.mockRestore(); + if (originalGatewayToken === undefined) { + delete process.env.OPENCLAW_GATEWAY_TOKEN; + } else { + process.env.OPENCLAW_GATEWAY_TOKEN = originalGatewayToken; + } + if (originalSchedulerDisabled === undefined) { + delete process.env.MIRA_DASHBOARD_DISABLE_SCHEDULER; + } else { + process.env.MIRA_DASHBOARD_DISABLE_SCHEDULER = originalSchedulerDisabled; + } + if (originalExecutionRole === undefined) { + delete process.env.MIRA_DASHBOARD_EXECUTION_ROLE; + } else { + process.env.MIRA_DASHBOARD_EXECUTION_ROLE = originalExecutionRole; + } + } + }); + it("warns but keeps startup alive when no gateway token is configured", async () => { const originalGatewayToken = process.env.OPENCLAW_GATEWAY_TOKEN; const originalLegacyToken = process.env.OPENCLAW_TOKEN; @@ -379,7 +538,7 @@ describe("server start scheduler policy", () => { const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); try { - serverStartModule.handleServerListening(); + serverStartModule.handleServerListening(TEST_RELEASE_COMMIT); expect(initSpy).not.toHaveBeenCalled(); expect(warnSpy).toHaveBeenCalledWith( "[Backend] No gateway token configured yet; waiting for bootstrap registration" @@ -439,9 +598,9 @@ describe("server start scheduler policy", () => { const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); try { - expect(() => serverStartModule.handleServerListening()).toThrow( - "gateway boot failed" - ); + expect(() => + serverStartModule.handleServerListening(TEST_RELEASE_COMMIT) + ).toThrow("gateway boot failed"); expect(shutdownSpy).not.toHaveBeenCalled(); expect(errorSpy).toHaveBeenCalledWith( "[Backend] Failed to start background services:", @@ -464,6 +623,44 @@ describe("server start scheduler policy", () => { } }); + it("shares concurrent startup failures and clears the completed attempt", async () => { + const startupFailure = new Error("release verification failed"); + const release = + Promise.withResolvers< + Awaited< + ReturnType + > + >(); + const releaseSpy = jest + .spyOn(releaseManifestModule, "getRuntimeReleaseIdentity") + .mockReturnValue(release.promise); + const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + const originalExitCode = process.exitCode; + const { startBackendServer, stopBackendServer } = + await import("../src/serverStart.ts"); + try { + await stopBackendServer(); + process.exitCode = 0; + + const firstStartup = startBackendServer(0); + const concurrentStartup = startBackendServer(0); + expect(concurrentStartup).toBe(firstStartup); + + release.reject(startupFailure); + await expect(firstStartup).rejects.toBe(startupFailure); + await expect(concurrentStartup).rejects.toBe(startupFailure); + + const retry = startBackendServer(0); + expect(retry).not.toBe(firstStartup); + await expect(retry).rejects.toBe(startupFailure); + } finally { + await stopBackendServer(); + releaseSpy.mockRestore(); + errorSpy.mockRestore(); + process.exitCode = originalExitCode; + } + }); + it("starts, stops, and handles web shutdown signals with isolated runtime state", async () => { const environmentKeys = [ "MIRA_DASHBOARD_DB_PATH", @@ -492,8 +689,10 @@ describe("server start scheduler policy", () => { const { runBackendServer, startBackendServer, stopBackendServer } = await import("../src/serverStart.ts"); - startBackendServer(0); - startBackendServer(0); + const firstStartup = startBackendServer(0); + const concurrentStartup = startBackendServer(0); + expect(concurrentStartup).toBe(firstStartup); + await concurrentStartup; await stopBackendServer(); await stopBackendServer(); @@ -606,9 +805,12 @@ describe("server start scheduler policy", () => { let isReady = false; for (let attempt = 0; attempt < 100; attempt += 1) { try { - const response = await fetch(`http://127.0.0.1:${port}/api/health`, { - signal: AbortSignal.timeout(100), - }); + const response = await fetch( + `http://127.0.0.1:${port}/api/health/live`, + { + signal: AbortSignal.timeout(100), + } + ); if (response.ok) { isReady = true; break; diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index 7e8c05056..18074d704 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -1971,8 +1971,10 @@ printf 'scheduled\n' `mira-dashboard-deploy-${job.id}` ); const restartCommand = await Bun.file(systemdLog).text(); - expect(restartCommand).toContain("/api/health"); - expect(restartCommand).toContain('"workerOnline":true'); + expect(restartCommand).toContain("/api/health/ready"); + expect(restartCommand).toContain("--connect-timeout 2 --max-time 5"); + expect(restartCommand).toContain("for attempt in {1..20}"); + expect(restartCommand).not.toContain('"workerOnline":true'); expect(restartCommand).not.toContain("/api/job-executions"); expect(existsSync(path.join(fakeRoot, "node_modules"))).toBe(false); expect(existsSync(path.join(fakeRoot, "backend", "node_modules"))).toBe( diff --git a/backend/test/utilityBehavior.test.ts b/backend/test/utilityBehavior.test.ts index 7557b0f84..fa8090428 100644 --- a/backend/test/utilityBehavior.test.ts +++ b/backend/test/utilityBehavior.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { Server } from "bun"; import { describe, expect, it, jest } from "bun:test"; +import * as databaseMigrationRunnerModule from "../src/databaseMigrationRunner.ts"; import { isAllowedDashboardOrigin, readJson, @@ -714,24 +715,31 @@ describe("backend service utilities", () => { ); }); - it("keeps health available when worker telemetry cannot be read", async () => { + it("fails readiness when worker telemetry cannot be read", async () => { const summarySpy = jest .spyOn(jobExecutionQueueModule, "getJobExecutionSummary") .mockImplementation(() => { throw new Error("queue telemetry unavailable"); }); + const releaseSummarySpy = jest + .spyOn(jobExecutionQueueModule, "isJobWorkerReleaseReady") + .mockImplementation(() => { + throw new Error("queue telemetry unavailable"); + }); const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); try { const response = await callTestRoute( appRoutes, - "/api/health", + "/api/health/ready", serverWithAddress("127.0.0.1") ); - expect(response.status).toBe(200); + expect(response.status).toBe(503); await expect(response.json()).resolves.toMatchObject({ - status: "isOk", - workerOnline: false, + checks: { + worker: { ready: false }, + }, + status: "notReady", }); expect(warnSpy).toHaveBeenCalledWith( "[Health] Failed to read job worker telemetry:", @@ -739,6 +747,44 @@ describe("backend service utilities", () => { ); } finally { summarySpy.mockRestore(); + releaseSummarySpy.mockRestore(); + warnSpy.mockRestore(); + } + }); + + it("logs database readiness failures without exposing them in the response", async () => { + const databaseError = new Error("database unavailable"); + const migrationSpy = jest + .spyOn(databaseMigrationRunnerModule, "validateDatabaseMigrationHistory") + .mockImplementation(() => { + throw databaseError; + }); + const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); + + try { + const response = await callTestRoute( + appRoutes, + "/api/health/ready", + serverWithAddress("127.0.0.1") + ); + + expect(response.status).toBe(503); + const payload = await response.json(); + expect(payload).toMatchObject({ + checks: { + database: { + ready: false, + }, + }, + status: "notReady", + }); + expect(JSON.stringify(payload)).not.toContain(databaseError.message); + expect(warnSpy).toHaveBeenCalledWith( + "[Health] Database readiness failed:", + databaseError + ); + } finally { + migrationSpy.mockRestore(); warnSpy.mockRestore(); } }); @@ -788,7 +834,7 @@ describe("backend service utilities", () => { string, (request: Request, server: Server) => Response > = { - "/api/health": () => new Response("ok"), + "/api/health/live": () => new Response("ok"), "/api/private": () => new Response("private"), "/api/auth/login": () => new Response("login"), "/syntax": () => { @@ -806,7 +852,7 @@ describe("backend service utilities", () => { const routes = withRequestPolicy(routeEntries); const server = serverWithAddress("203.0.113.10"); - const health = await callTestRoute(routes, "/api/health", server); + const health = await callTestRoute(routes, "/api/health/live", server); expect(health.status).toBe(200); expect(health.headers.get("ratelimit-policy")).toBe("600;w=60"); expect(health.headers.get("x-request-id")).toMatch( @@ -827,7 +873,7 @@ describe("backend service utilities", () => { const secureOrigin = withRequestSecurity( // eslint-disable-next-line unicorn/prefer-https -- Simulates TLS termination at a trusted proxy. - new Request("http://dashboard.example/api/health", { + new Request("http://dashboard.example/api/health/live", { headers: { "x-forwarded-proto": "https" }, }), new Response(), @@ -837,7 +883,7 @@ describe("backend service utilities", () => { "connect-src 'self' wss://dashboard.example" ); const directSecureOrigin = withRequestSecurity( - new Request("https://dashboard.example/api/health"), + new Request("https://dashboard.example/api/health/live"), new Response(), serverWithAddress("203.0.113.10") ); @@ -847,7 +893,7 @@ describe("backend service utilities", () => { const sameOriginMutation = await callTestRoute( routes, - "/api/health", + "/api/health/live", server, { headers: { @@ -889,7 +935,7 @@ describe("backend service utilities", () => { const crossOriginMutation = await callTestRoute( routes, - "/api/health", + "/api/health/live", server, { headers: { @@ -917,7 +963,7 @@ describe("backend service utilities", () => { const missingOriginCrossSiteMutation = await callTestRoute( routes, - "/api/health", + "/api/health/live", server, { headers: { "sec-fetch-site": "same-site" }, diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 240b56c3f..d7fcc3bca 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -5,11 +5,12 @@ route files under `backend/src/routes/` for exact validation details. ## Health -| Method | Path | Purpose | -| ------ | --------------- | ------------------------------------- | -| `GET` | `/health` | Public health and worker check. | -| `GET` | `/api/health` | Public API health and worker check. | -| `GET` | `/api/sessions` | Legacy/session snapshot from Gateway. | +| Method | Path | Purpose | +| ------ | ------------------------- | ------------------------------------------------------ | +| `GET` | `/api/health/live` | Public web-process liveness. | +| `GET` | `/api/health/ready` | Public activation readiness; `503` when not ready. | +| `GET` | `/api/health/diagnostics` | Authenticated readiness details and dependency status. | +| `GET` | `/api/sessions` | Legacy/session snapshot from Gateway. | ## Auth diff --git a/docs/api/overview.md b/docs/api/overview.md index eae5ae539..08a22c4c9 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -49,7 +49,9 @@ or another authentication mechanism. Public routes: -- `GET /api/health` +- `GET|HEAD /api/health/live` +- `GET|HEAD /api/health/ready` — returns `503` whenever any required + activation-readiness check fails - `GET /api/auth/bootstrap` - `GET /api/auth/session` - `POST /api/auth/register-first-user` @@ -60,6 +62,9 @@ Public routes: - `POST /api/auth/login/webauthn/verify` - `POST /api/auth/logout` +The retired top-level `GET|HEAD /health` path returns `410 Gone` instead of +falling through to the SPA shell. This makes stale monitors fail visibly. + The WebSocket endpoint `/ws` is also authenticated and origin-checked. Account-security endpoints under `/api/account/security/*` are protected diff --git a/docs/architecture/gateway-and-chat.md b/docs/architecture/gateway-and-chat.md index c1b2534f8..6c42f48e1 100644 --- a/docs/architecture/gateway-and-chat.md +++ b/docs/architecture/gateway-and-chat.md @@ -47,21 +47,18 @@ Dashboard user: Invalid Gateway auth returns `401`. Rollback failures return `500` because the server may need manual inspection. -## Gateway Health +## Gateway And Runtime Health -`GET /api/health` reports: +`GET /api/health/live` is a public process-liveness probe. +`GET /api/health/ready` is the public deployment gate for the release manifest, +SQLite schema, built frontend, and persistent worker. The authenticated +`GET /api/health/diagnostics` response adds the Gateway dependency, session +count, and release details used by the Dashboard header. -| Field | Meaning | -| ------------------ | ------------------------------------------------------------------ | -| `status` | Backend health state. | -| `gatewayConnected` | Whether the backend Gateway client is authenticated and connected. | -| `sessionCount` | Gateway session count known to Dashboard. | -| `backendCommit` | Git commit served by the backend when available. | -| `workerOnline` | Whether the persistent job worker has a fresh heartbeat. | - -If queue telemetry cannot be read, health remains available and reports -`workerOnline:false`. Treat that degraded value as either an offline worker or -unavailable queue telemetry, and inspect both backend and worker logs. +Gateway connectivity is deliberately diagnostic rather than a release-readiness +gate: rolling Dashboard code back cannot repair an OpenClaw outage. If worker +telemetry cannot be read, readiness returns `503` and the header reports +`WK ○`; inspect both backend and worker logs. If `gatewayConnected:false`, check: @@ -609,7 +606,8 @@ When changing chat event handling, test these cases: ## Local Debug Commands ```bash -curl http://127.0.0.1:3100/api/health +curl --fail http://127.0.0.1:3100/api/health/live +curl --fail http://127.0.0.1:3100/api/health/ready systemctl --user status mira-dashboard.service --no-pager systemctl --user status openclaw-gateway.service --no-pager journalctl --user -u mira-dashboard.service -n 160 --no-pager diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index fbf2d0c90..7092fa2ed 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -96,7 +96,8 @@ Key files: All `/api/*` routes are authenticated except: -- `/api/health` +- `/api/health/live` +- `/api/health/ready` - the exact bootstrap, session-state, login-factor, and logout endpoints under `/api/auth/*`. diff --git a/docs/operations/runbooks.md b/docs/operations/runbooks.md index 5cf9156b5..07a0af9a5 100644 --- a/docs/operations/runbooks.md +++ b/docs/operations/runbooks.md @@ -6,7 +6,7 @@ Docker image update behavior, see [Docker updater](docker-updater.md). ## Check Dashboard Health ```bash -curl http://127.0.0.1:3100/api/health +curl --fail http://127.0.0.1:3100/api/health/ready systemctl --user status mira-dashboard.service --no-pager systemctl --user status mira-dashboard-worker.service --no-pager journalctl --user -u mira-dashboard.service -n 120 --no-pager @@ -16,12 +16,7 @@ journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager Expected health: ```json -{ - "status": "isOk", - "gatewayConnected": true, - "sessionCount": 9, - "backendCommit": "abc1234" -} +{ "status": "isReady", "checks": { "worker": { "ready": true } } } ``` ## Restart Dashboard @@ -29,12 +24,22 @@ Expected health: ```bash systemctl --user restart mira-dashboard.service systemctl --user status mira-dashboard.service --no-pager -curl http://127.0.0.1:3100/api/health +wait_for_dashboard_ready() { + for attempt in {1..20}; do + if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ + http://127.0.0.1:3100/api/health/ready >/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} +wait_for_dashboard_ready ``` ## Dashboard Shows WebSocket Disconnected -1. Check `/api/health`. +1. Check `/api/health/live` and `/api/health/ready`. 2. Check OpenClaw Gateway: ```bash @@ -236,7 +241,17 @@ install -m 0600 "$backup_path" "$db_path" test "$(sqlite3 -readonly "$db_path" "PRAGMA quick_check;")" = "ok" systemctl --user start mira-dashboard.service systemctl --user start mira-dashboard-worker.service -curl --fail --show-error --silent http://127.0.0.1:3100/api/health +wait_for_dashboard_ready() { + for attempt in {1..20}; do + if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ + http://127.0.0.1:3100/api/health/ready >/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} +wait_for_dashboard_ready printf '\nRecovery files retained at %s\n' "$recovery_dir" ``` diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 7c753eab0..304abcd49 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -5,7 +5,8 @@ Use this page when the symptom is unclear. Prefer narrow checks before restarts. ## Quick Triage ```bash -curl http://127.0.0.1:3100/api/health +curl --fail http://127.0.0.1:3100/api/health/live +curl --fail http://127.0.0.1:3100/api/health/ready systemctl --user status mira-dashboard.service --no-pager journalctl --user -u mira-dashboard.service -n 160 --no-pager git -C /home/ubuntu/projects/mira-dashboard status --short --branch @@ -102,9 +103,10 @@ layout unless a deliberate migration requires otherwise. ```bash cd /home/ubuntu/projects/mira-dashboard -bun run build +/usr/local/bin/doppler run --config prd --project rajohan -- \ + bun run deploy:prepare systemctl --user restart mira-dashboard.service -curl http://127.0.0.1:3100/api/health +curl --fail http://127.0.0.1:3100/api/health/ready ``` If the browser still shows old UI, hard refresh or clear the tab cache. diff --git a/docs/security/auth-and-trust-boundaries.md b/docs/security/auth-and-trust-boundaries.md index bf2125e00..04e2ecacf 100644 --- a/docs/security/auth-and-trust-boundaries.md +++ b/docs/security/auth-and-trust-boundaries.md @@ -14,7 +14,8 @@ important trust boundaries: All `/api/*` routes require a Dashboard session except the exact public bootstrap/login surface: -- `/api/health` +- `GET|HEAD /api/health/live` +- `GET|HEAD /api/health/ready` - `GET /api/auth/bootstrap` - `GET /api/auth/session` - `POST /api/auth/register-first-user` diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index 94373704a..c8d49c155 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -45,20 +45,32 @@ bun install --frozen-lockfile ## Build Frontend And Backend -From the repo root: +Build the frontend from the repository root: ```bash +cd /home/ubuntu/projects/mira-dashboard bun run build ``` -From `backend/`: +Build the backend from its own package directory: ```bash +cd /home/ubuntu/projects/mira-dashboard/backend bun run build ``` -The frontend build writes to `dist/`. The backend build writes to -`backend/dist/`. +Return to the repository root and create the checksummed runtime manifest: + +```bash +cd /home/ubuntu/projects/mira-dashboard +bun run release:manifest +``` + +The frontend build writes to `dist/`, the backend build writes to +`backend/dist/`, and `release:manifest` binds both outputs to the checked-out +commit. A fresh host may not have a Dashboard database yet, so this first build +uses `release:manifest` directly instead of the normal database-aware +`deploy:prepare`; first startup creates and migrates the database. ## Configure Secrets @@ -184,7 +196,7 @@ Expected after setup: ## Verify Runtime ```bash -curl http://127.0.0.1:3100/api/health +curl --fail http://127.0.0.1:3100/api/health/ready systemctl --user status mira-dashboard.service --no-pager journalctl --user -u mira-dashboard.service -n 100 --no-pager journalctl --user -u mira-dashboard-worker.service -n 100 --no-pager @@ -194,16 +206,32 @@ Healthy response shape: ```json { - "status": "isOk", - "gatewayConnected": true, - "sessionCount": 9, - "backendCommit": "abc1234", - "workerOnline": true + "checks": { + "database": { + "currentSchemaVersion": 6, + "maximumCompatibleSchemaVersion": 6, + "minimumCompatibleSchemaVersion": 6, + "ready": true, + "targetSchemaVersion": 6 + }, + "frontend": { "ready": true }, + "release": { + "backendCommit": "12345678", + "frontendCommit": "12345678", + "manifestFormatVersion": 1, + "ready": true, + "source": "manifest" + }, + "worker": { "ready": true } + }, + "dependencies": { "gatewayConnected": true }, + "status": "isReady" } ``` -If `gatewayConnected` is false, check the Gateway token, OpenClaw Gateway -service, and `/api/auth/bootstrap` state before debugging the frontend. -If `workerOnline` is false, inspect both Dashboard and -`mira-dashboard-worker.service`; the worker heartbeat may be stale or queue -telemetry may be unavailable. +The authenticated Dashboard header shows `WS`, `BE`, and `WK` separately. If +`WS` is offline, check the Gateway token, OpenClaw Gateway service, and +`/api/auth/bootstrap` state before debugging the frontend. If `WK` is offline, +inspect both Dashboard and `mira-dashboard-worker.service`; the worker heartbeat +may be stale, belong to another release commit, or queue telemetry may be +unavailable. diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index fe6eedf26..8485d5ccd 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -52,10 +52,11 @@ cd .. bun run deploy:prepare ``` -`deploy:prepare` builds the frontend and backend, then runs `db:preflight` -before service restart. Keep ordinary `build` commands side-effect free; use -this combined command for every supported manual or Dashboard-driven deploy so -the database safety gate cannot be skipped accidentally. +`deploy:prepare` builds the frontend and backend, runs `db:preflight`, and +writes the checksummed release manifest before service restart. Keep ordinary +`build` commands side-effect free; use this combined command for every +supported manual or Dashboard-driven deploy so the database and release gates +cannot be skipped accidentally. ## Install Or Refresh Units @@ -103,7 +104,17 @@ journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager ## Smoke Test ```bash -curl http://127.0.0.1:3100/api/health +wait_for_dashboard_ready() { + for attempt in {1..20}; do + if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ + http://127.0.0.1:3100/api/health/ready >/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} +wait_for_dashboard_ready curl http://127.0.0.1:3100/api/auth/bootstrap ``` @@ -133,10 +144,10 @@ callers before restarting into this version: `/home/ubuntu/projects/mira-dashboard/scripts/miraDashboardApi.ts`. 3. Migrate and smoke-test every caller against the currently running scoped-credential-compatible release: - - heartbeat: `cache:read`, `reports:write`; - - task tracking: `agents:write`, `tasks:read`, and `tasks:write`; - - daily summary: `cache:read`, `reports:write`; - - daily brief: `cache:read`, `reports:write`, `tasks:read`. + - heartbeat: `cache:read`, `reports:write`; + - task tracking: `agents:write`, `tasks:read`, and `tasks:write`; + - daily summary: `cache:read`, `reports:write`; + - daily brief: `cache:read`, `reports:write`, `tasks:read`. 4. Confirm allowed and intentionally denied calls have the expected automation actor and scope in `/api/audit-events`. 5. Deploy this release, restart the web unit, verify every scoped caller again, @@ -172,28 +183,53 @@ cd /home/ubuntu/projects/mira-dashboard git log --oneline -n 10 git switch main git reset --hard -bun run build -(cd backend && bun run build) -install -m 0644 systemd/mira-dashboard.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard.service -if test -f backend/dist/workerStart.js; then - install -m 0644 systemd/mira-dashboard-worker.service \ - /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service - systemctl --user daemon-reload - systemctl --user restart mira-dashboard.service - systemctl --user restart mira-dashboard-worker.service +if ! bun -e 'const packageJson = await Bun.file("package.json").json(); process.exit(typeof packageJson.scripts?.["deploy:prepare"] === "string" ? 0 : 1)'; then + echo "Rollback target predates the supported deploy contract" >&2 + exit 1 +fi +bun install --frozen-lockfile +(cd backend && bun install --frozen-lockfile) +if test -f scripts/writeReleaseManifest.ts; then + release_health_path=/api/health/ready else - systemctl --user disable --now mira-dashboard-worker.service - systemctl --user daemon-reload - systemctl --user restart mira-dashboard.service + # One-time bootstrap rollback to the pre-manifest release. + release_health_path=/api/health fi -curl http://127.0.0.1:3100/api/health +/usr/local/bin/doppler run --config prd --project rajohan -- \ + bun run deploy:prepare +install -m 0644 systemd/mira-dashboard.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service +install -m 0644 systemd/mira-dashboard-worker.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service +systemctl --user daemon-reload +systemctl --user restart mira-dashboard.service +systemctl --user restart mira-dashboard-worker.service +wait_for_dashboard_ready() { + for attempt in {1..20}; do + if test "$release_health_path" = "/api/health"; then + if curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ + "http://127.0.0.1:3100${release_health_path}" | + grep -Fq '"workerOnline":true'; then + return 0 + fi + elif curl --fail --silent --show-error --connect-timeout 2 --max-time 5 \ + "http://127.0.0.1:3100${release_health_path}" >/dev/null; then + return 0 + fi + sleep 1 + done + return 1 +} +wait_for_dashboard_ready ``` -If the known-good target predates `workerStart.js`, the branch above stops the -worker before starting that version and reinstalls the checked-out legacy -combined web unit. Do not repeatedly restart a worker unit whose target -entrypoint does not exist. +The conditional health target exists only for the first rollback across the +manifest-contract cutover. Manifest-aware releases always regenerate their +ignored manifest through `deploy:prepare` and verify `/api/health/ready`. + +Rollback targets older than the split-worker/database-preflight contract are +deliberately unsupported. This private single-operator service keeps a supported +known-good release instead of retaining an untested legacy activation path. Do not use `git reset --hard` casually in normal work. It is a rollback procedure for production incidents after an explicit decision. @@ -265,24 +301,68 @@ manager must therefore read the release/schema compatibility declaration before offering or automatically performing rollback; it must never start an incompatible older release against a newer live database. -## Health Signals +## Release Manifest Contract -Healthy `/api/health`: +`bun run deploy:prepare` builds frontend and backend, completes the verified +SQLite preflight, and writes an ignored `release-manifest.json` in the release +root. The manifest is the release identity source when `NODE_ENV=production`; +Git is only a development/test fallback. -```json -{ - "status": "isOk", - "gatewayConnected": true, - "sessionCount": 9, - "backendCommit": "abc1234", - "workerOnline": true -} -``` +Manifest format version 1 records: + +- the full and eight-character Git commit plus commit title and build time; +- the Bun version used for the build; +- matching frontend/backend commit identities emitted inside both build trees; +- the target, minimum-compatible, and maximum-compatible SQLite schema; +- a checksum of the immutable migration registry; +- the SHA-256 and byte length of every frontend/backend build artifact plus + both package manifests, Bun lockfiles, and the default runtime log-rotation + configuration. + +The backend bundle also embeds its full build commit. Runtime readiness requires +that embedded commit, both build-identity files, and the release manifest to +agree. Running `release:manifest` against ignored output left behind by another +checkout therefore fails instead of relabeling stale code. + +Manifest creation and verification reject absolute/traversal paths, symlinks, +hard-linked files, special files, unsorted/duplicate inventories, checksum +drift, and undeclared runtime artifacts. The schema compatibility range is an +explicit code constant. Adding a future migration without reviewing that range +fails the release contract. + +The current in-place executor generates this manifest as a transition step. The +immutable release manager must verify it before activation and compare the live +schema with the previous release's declared range before offering code-only +rollback. + +## Health Signals + +Deployment health is split by purpose: + +- `GET /api/health/live` proves that the web process can answer requests. +- `GET /api/health/ready` requires a valid release identity, + current/accessible SQLite schema, built frontend, and a fresh worker heartbeat + from the exact manifest commit. + Concurrent probes share one artifact scan, and a completed result is reused + for at most 15 seconds before the checksummed inventory is verified again. + This readiness route returns HTTP 503 with `status: "notReady"` when an + internal activation check fails. +- `GET /api/health/diagnostics` returns the readiness breakdown plus session + count and requires an authenticated Dashboard session. +- `GET /api/health` is a temporary compatibility adapter for the pre-readiness + deploy executor. It returns 503 unless the full readiness contract passes and + retains `workerOnline` only until the atomic executor cutover is complete. + +Gateway connectivity is reported as an external dependency but deliberately +does not fail release readiness: rolling Dashboard code back cannot repair an +OpenClaw Gateway outage. Production activation and automatic rollback must use +`/api/health/ready`. Important failures: -- `gatewayConnected: false`: check OpenClaw Gateway service and Gateway token. -- `workerOnline: false`: the worker heartbeat is stale or queue telemetry is +- `dependencies.gatewayConnected: false` in authenticated diagnostics: check + OpenClaw Gateway service and Gateway token. +- `checks.worker.ready: false`: the worker heartbeat is stale or queue telemetry is unavailable; check both Dashboard and worker service logs. - HTTP `503 Frontend Not Built`: build root frontend with `bun run build`. - `Unauthorized` on API routes: auth/session or cookie issue. diff --git a/docs/setup/secrets-and-env.md b/docs/setup/secrets-and-env.md index 843b3cdb2..7175a131a 100644 --- a/docs/setup/secrets-and-env.md +++ b/docs/setup/secrets-and-env.md @@ -31,7 +31,10 @@ Environment token precedence is: | `MIRA_DASHBOARD_OPENCLAW_HOME` | Optional | `~/.openclaw` | Dashboard-specific fallback OpenClaw home. Most file/config/media routes use this only when `OPENCLAW_HOME` is absent. | | `WORKSPACE_ROOT` | Optional | OpenClaw workspace | Root exposed by `/api/files`. Must be absolute and normalized if set. | | `MIRA_DASHBOARD_LOGS_ROOT` | Optional | system log root default | Root used by log stream services. | -| `MIRA_LOG_ROTATION_CONFIG` | Optional | `backend/config/log-rotation.json` | Log rotation config path. | + +`MIRA_DASHBOARD_FRONTEND_PATH` is a development/test escape hatch. Production +serves the active release's checksummed `dist/`; any configured value that does +not resolve exactly to that directory is rejected. ## Network, Auth, And Browser Access diff --git a/package.json b/package.json index 6120e299c..029fd0653 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "scripts": { "dev": "bun --watch scripts/developmentFrontend.ts", "build": "bun node_modules/@typescript/native/bin/tsc -b && bun scripts/buildFrontend.ts", - "deploy:prepare": "bun run build && cd backend && bun run deploy:prepare", + "deploy:prepare": "bun run build && cd backend && bun run deploy:prepare && cd .. && bun run release:manifest", + "release:manifest": "bun scripts/writeReleaseManifest.ts", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx,json,css}\" \"backend/**/*.{ts,js}\"", diff --git a/scripts/frontendBuild.ts b/scripts/frontendBuild.ts index 2817a19ed..5fa247ab5 100644 --- a/scripts/frontendBuild.ts +++ b/scripts/frontendBuild.ts @@ -1,8 +1,12 @@ -import { mkdir, rm } from "node:fs/promises"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import tailwindPlugin from "bun-plugin-tailwind"; +import { + isReleaseBuildCommit, + resolveBuildSourceIdentity, +} from "../backend/scripts/buildSourceIdentity.ts"; import reactCompilerPlugin from "./reactCompilerPlugin"; type FrontendBuildMode = "development" | "production"; @@ -26,38 +30,25 @@ const productionDevtoolsPlugin: Bun.BunPlugin = { }, }; -function getAppCommit(): string { - try { - const result = Bun.spawnSync({ - cmd: ["git", "rev-parse", "--short", "HEAD"], - stderr: "ignore", - stdin: "ignore", - stdout: "pipe", - }); - - if (result.exitCode !== 0) { - return "unknown"; - } - - return new TextDecoder().decode(result.stdout).trim() || "unknown"; - } catch { - return "unknown"; - } -} - export async function buildFrontend({ mode, outdir = "dist", }: FrontendBuildOptions): Promise { const resolvedOutdir = path.resolve(outdir); const isProduction = mode === "production"; + const commitSha = resolveBuildSourceIdentity(); + if (isProduction && commitSha === "unknown") { + throw new Error("Production frontend build requires a full Git commit identity"); + } await rm(resolvedOutdir, { force: true, recursive: true }); await mkdir(resolvedOutdir, { recursive: true }); const result = await Bun.build({ define: { - __APP_COMMIT__: JSON.stringify(getAppCommit()), + __APP_COMMIT__: JSON.stringify( + isReleaseBuildCommit(commitSha) ? commitSha.slice(0, 8) : commitSha + ), "process.env.PUBLIC_DASHBOARD_WS_PORT": "undefined", "process.env.NODE_ENV": JSON.stringify(mode), }, @@ -83,4 +74,18 @@ export async function buildFrontend({ if (!result.success) { throw new AggregateError(result.logs, "Frontend build failed"); } + + await writeFile( + path.join(resolvedOutdir, "build-identity.json"), + `${JSON.stringify( + { + bunVersion: Bun.version, + commitSha, + component: "frontend", + formatVersion: 1, + }, + undefined, + 2 + )}\n` + ); } diff --git a/scripts/writeReleaseManifest.ts b/scripts/writeReleaseManifest.ts new file mode 100644 index 000000000..fb69c2f45 --- /dev/null +++ b/scripts/writeReleaseManifest.ts @@ -0,0 +1,15 @@ +import path from "node:path"; + +import { writeReleaseManifest } from "../backend/src/releaseManifest.ts"; + +const releaseRoot = path.resolve(import.meta.dirname, ".."); +const manifest = await writeReleaseManifest({ releaseRoot }); + +console.log( + JSON.stringify({ + artifactCount: manifest.artifacts.length, + commit: manifest.commitShort, + manifestVersion: manifest.formatVersion, + schema: manifest.schema.target, + }) +); diff --git a/src/components/layout/AppHeader.tsx b/src/components/layout/AppHeader.tsx index 2ae812492..70f3a8afd 100644 --- a/src/components/layout/AppHeader.tsx +++ b/src/components/layout/AppHeader.tsx @@ -26,8 +26,30 @@ export function AppHeader({ const { isConnected } = useOpenClawSocket(); const { data: health, isError: isBackendError } = useHealth(); - const isBackendConnected = !isBackendError && health?.status === "isOk"; - const backendCommit = health?.backendCommit || "unknown"; + const isBackendConnected = !isBackendError && health !== undefined; + const workerState = isBackendConnected + ? health.checks.worker.ready + ? "ready" + : "offline" + : "unknown"; + const workerStatus = { + offline: { + className: "border-red-500/40 bg-red-500/10 text-red-300", + label: "Worker offline", + symbol: "○", + }, + ready: { + className: "border-green-500/40 bg-green-500/10 text-green-300", + label: "Worker online", + symbol: "●", + }, + unknown: { + className: "border-primary-600 bg-primary-800 text-primary-300", + label: "Worker status unavailable", + symbol: "?", + }, + }[workerState]; + const backendCommit = health?.releaseDetails.backendCommit || "unknown"; const frontendCommit = typeof __APP_COMMIT__ === "string" ? __APP_COMMIT__ : "dev"; const hasVersionMismatch = backendCommit !== "unknown" && @@ -95,6 +117,16 @@ export function AppHeader({ BE {isBackendConnected ? "●" : "○"} + + WK + {workerStatus.symbol} +