Skip to content
Merged
551 changes: 551 additions & 0 deletions backend/src/releaseDeployment.ts

Large diffs are not rendered by default.

7 changes: 6 additions & 1 deletion backend/src/releaseLifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
} from "./releaseManager.ts";
import {
activateDashboardRelease,
pruneDashboardReleases,
readDashboardReleaseState,
resolveDashboardReleasesRoot,
rollbackDashboardRelease,
Expand Down Expand Up @@ -70,9 +71,13 @@ export async function runReleaseLifecycleCommand(
state = await readDashboardReleaseState(releasesRoot);
break;
}
case "prune": {
const retainCount = commitSha === undefined ? 3 : Number(commitSha);
return pruneDashboardReleases(retainCount, releasesRoot);
}
default: {
throw new TypeError(
"Usage: releaseLifecycle.js <status|activate COMMIT_SHA [--coordinated-schema-cutover]|rollback>"
"Usage: releaseLifecycle.js <status|activate COMMIT_SHA [--coordinated-schema-cutover]|rollback|prune [RETAIN_COUNT]>"
);
}
}
Expand Down
132 changes: 126 additions & 6 deletions backend/src/releaseManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const RELEASE_COMMIT_SHA_PATTERN = /^[\da-f]{40}$/u;
const RELEASE_TRANSITION_FORMAT_VERSION = 1;
const RELEASE_TRANSITION_JOURNAL_FILE_NAME = ".release-transition.json";
export const RELEASE_TRANSITION_LOCK_FILE_NAME = ".release-transition.lock";
const RETIRED_RELEASE_DIRECTORY_PATTERN =
/^\.retired-[\da-f]{40}-[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u;
const MAX_RELEASE_TRANSITION_FILE_BYTES = 4096;
export const RELEASE_TRANSITION_LOCK_PROGRAM = "/usr/bin/flock";

Expand All @@ -53,6 +55,11 @@ export interface DashboardReleaseState {
root: string;
}

export interface DashboardReleaseRetentionResult {
removed: string[];
retained: string[];
}

export interface DashboardReleaseManagerOptions {
readLiveSchemaState?: (
maximumCompatibleVersion: number
Expand Down Expand Up @@ -397,6 +404,17 @@ function releaseDirectoryIdentity(stat: fs.BigIntStats): string {
return [stat.dev, stat.ino, stat.ctimeNs, stat.birthtimeNs].join(":");
}

function isSameReleaseDirectoryInode(
left: fs.BigIntStats,
right: fs.BigIntStats
): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.birthtimeNs === right.birthtimeNs
);
}

function releaseManifestIdentity(manifest: DashboardReleaseManifest): string {
const canonicalManifest = {
artifacts: manifest.artifacts
Expand Down Expand Up @@ -630,12 +648,6 @@ function assertReleaseMigrationHistoryCompatible(
action: "Activation" | "Rollback"
): void {
const expectedMigrations = release.schema.migrations;
if (!expectedMigrations) {
// Only temporary format-v1 manifests omit migration identities. They stay
// readable for the first managed cutover rollback window, but cannot bind
// activation or rollback to the exact applied migration history.
return;
}
for (const actual of liveState.migrations.slice(0, release.schema.target)) {
const expected = expectedMigrations[actual.version - 1];
if (
Expand Down Expand Up @@ -1195,3 +1207,111 @@ export async function rollbackDashboardRelease(
});
});
}

export async function pruneDashboardReleases(
retainCount = 3,
releasesRoot = resolveDashboardReleasesRoot()
): Promise<DashboardReleaseRetentionResult> {
if (!Number.isSafeInteger(retainCount) || retainCount < 2 || retainCount > 20) {
throw new TypeError("Managed release retention must be between 2 and 20");
}

const layout = await ensureDashboardReleaseLayout(releasesRoot);
return withReleaseTransitionLock(layout, "exclusive", async () => {
await recoverInterruptedReleaseTransition(layout);
const state = await readDashboardReleaseStateFromLayout(layout);
const protectedCommits = new Set(
[state.current?.commitSha, state.previous?.commitSha].filter(
(commitSha): commitSha is string => commitSha !== undefined
)
);
const entries = await fsp.readdir(layout.releasesPath, {
withFileTypes: true,
});
let hasFilesystemChanges = false;
const releases: ManagedDashboardRelease[] = [];
for (const entry of entries) {
if (RETIRED_RELEASE_DIRECTORY_PATTERN.test(entry.name)) {
if (!entry.isDirectory() || entry.isSymbolicLink()) {
throw new TypeError(
`Retired release entry must be a real directory: ${entry.name}`
);
}
await fsp.rm(path.join(layout.releasesPath, entry.name), {
recursive: true,
});
hasFilesystemChanges = true;
continue;
}
if (!RELEASE_COMMIT_SHA_PATTERN.test(entry.name)) {
continue;
}
Comment thread
mira-2026 marked this conversation as resolved.
if (!entry.isDirectory() || entry.isSymbolicLink()) {
throw new TypeError(
`Managed release entry must be a real directory: ${entry.name}`
);
}
releases.push(await loadManagedReleaseFromLayout(layout, entry.name));
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

const newestFirst = releases.toSorted((left, right) => {
const builtAtComparison = right.manifest.builtAt.localeCompare(
left.manifest.builtAt
);
return builtAtComparison || right.commitSha.localeCompare(left.commitSha);
});
const retained = new Set(protectedCommits);
for (const release of newestFirst) {
if (retained.size >= retainCount) {
break;
}
retained.add(release.commitSha);
}

const removed: string[] = [];
for (const release of newestFirst.toReversed()) {
if (retained.has(release.commitSha)) {
continue;
}
const currentStat = await fsp.lstat(release.path, { bigint: true });
if (
!currentStat.isDirectory() ||
currentStat.isSymbolicLink() ||
releaseDirectoryIdentity(currentStat) !== release.directoryIdentity
) {
throw new Error(
`Managed release changed before retention cleanup: ${release.commitSha}`
);
}
const retiredPath = path.join(
layout.releasesPath,
`.retired-${release.commitSha}-${randomUUID()}`
);
await fsp.rename(release.path, retiredPath);
await syncDirectory(layout.releasesPath);
const retiredStat = await fsp.lstat(retiredPath, { bigint: true });
if (
!retiredStat.isDirectory() ||
retiredStat.isSymbolicLink() ||
!isSameReleaseDirectoryInode(currentStat, retiredStat)
) {
throw new Error(
`Managed release changed during retention cleanup: ${release.commitSha}`
);
}
await fsp.rm(retiredPath, { recursive: true });
hasFilesystemChanges = true;
removed.push(release.commitSha);
}
if (hasFilesystemChanges) {
await syncDirectory(layout.releasesPath);
}

return {
removed,
retained: newestFirst
.filter((release) => retained.has(release.commitSha))
.map((release) => release.commitSha),
};
});
}
68 changes: 25 additions & 43 deletions backend/src/releaseManifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const RELEASE_STATIC_ARTIFACTS = [
"bun.lock",
"package.json",
] as const;
const FORMAT_2_REQUIRED_RELEASE_ARTIFACTS = [
const REQUIRED_RELEASE_ARTIFACTS = [
...RELEASE_STATIC_ARTIFACTS,
"backend/dist/build-identity.json",
"backend/dist/databasePreflight.js",
Expand All @@ -37,11 +37,6 @@ const FORMAT_2_REQUIRED_RELEASE_ARTIFACTS = [
"dist/build-identity.json",
"dist/index.html",
] as const;
// Format 1 remains readable only for the first managed cutover and its rollback
// window. Remove this compatibility list once current/previous cannot reference v1.
const FORMAT_1_REQUIRED_RELEASE_ARTIFACTS = FORMAT_2_REQUIRED_RELEASE_ARTIFACTS.filter(
(artifactPath) => artifactPath !== "backend/dist/releaseLifecycle.js"
);
const MAX_BUILD_IDENTITY_BYTES = 4096;
const RUNTIME_RELEASE_VERIFICATION_CACHE_MS = 15_000;
const SHA_256_PATTERN = /^[\da-f]{64}$/u;
Expand All @@ -65,11 +60,11 @@ export interface DashboardReleaseManifest {
backendCommit: string;
frontendCommit: string;
};
formatVersion: 1 | 2;
formatVersion: 2;
schema: {
maximumCompatible: number;
migrations?: DatabaseMigrationIdentity[];
migrationInventorySha256?: string;
migrations: DatabaseMigrationIdentity[];
migrationInventorySha256: string;
migrationRegistrySha256: string;
minimumCompatible: number;
target: number;
Expand Down Expand Up @@ -512,16 +507,14 @@ function parseMigrationIdentity(value: unknown): DatabaseMigrationIdentity {
};
}

function parseSchema(
value: unknown,
formatVersion: DashboardReleaseManifest["formatVersion"]
): DashboardReleaseManifest["schema"] {
function parseSchema(value: unknown): DashboardReleaseManifest["schema"] {
const expectedKeys = [
"maximumCompatible",
"migrations",
"migrationInventorySha256",
"migrationRegistrySha256",
"minimumCompatible",
"target",
...(formatVersion === 2 ? ["migrations", "migrationInventorySha256"] : []),
];
if (!isPlainRecord(value) || !hasExactKeys(value, expectedKeys)) {
throw new TypeError("Release manifest schema declaration is invalid");
Expand All @@ -539,30 +532,25 @@ function parseSchema(
) {
throw new TypeError("Release manifest schema range is invalid");
}
const migrations =
formatVersion === 2 && Array.isArray(value.migrations)
? value.migrations.map((migration) => parseMigrationIdentity(migration))
: undefined;
const migrations = Array.isArray(value.migrations)
? value.migrations.map((migration) => parseMigrationIdentity(migration))
: undefined;
// This digest proves only that a foreign manifest is internally consistent.
// Runtime and release-manager validation bind it to local code and live history.
if (
formatVersion === 2 &&
(!migrations ||
migrations.length !== (target as number) ||
migrations.some((migration, index) => migration.version !== index + 1) ||
typeof value.migrationInventorySha256 !== "string" ||
!SHA_256_PATTERN.test(value.migrationInventorySha256) ||
value.migrationInventorySha256 !==
databaseMigrationInventorySha256(migrations))
!migrations ||
migrations.length !== (target as number) ||
migrations.some((migration, index) => migration.version !== index + 1) ||
typeof value.migrationInventorySha256 !== "string" ||
!SHA_256_PATTERN.test(value.migrationInventorySha256) ||
value.migrationInventorySha256 !== databaseMigrationInventorySha256(migrations)
) {
throw new TypeError("Release manifest migration inventory is invalid");
}
return {
maximumCompatible: maximumCompatible as number,
...(migrations && { migrations }),
...(formatVersion === 2 && {
migrationInventorySha256: value.migrationInventorySha256 as string,
}),
migrations,
migrationInventorySha256: value.migrationInventorySha256 as string,
migrationRegistrySha256: value.migrationRegistrySha256,
minimumCompatible: minimumCompatible as number,
target: target as number,
Expand All @@ -583,8 +571,7 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest {
"formatVersion",
"schema",
]) ||
(value.formatVersion !== 1 &&
value.formatVersion !== RELEASE_MANIFEST_FORMAT_VERSION) ||
value.formatVersion !== RELEASE_MANIFEST_FORMAT_VERSION ||
typeof value.commitSha !== "string" ||
typeof value.commitShort !== "string" ||
typeof value.commitTitle !== "string" ||
Expand Down Expand Up @@ -620,10 +607,9 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest {
artifactPaths.some(
(artifactPath_, index) => artifactPath_ !== sortedArtifactPaths[index]
) ||
(value.formatVersion === 1
? FORMAT_1_REQUIRED_RELEASE_ARTIFACTS
: FORMAT_2_REQUIRED_RELEASE_ARTIFACTS
).some((requiredPath) => !artifactPaths.includes(requiredPath))
REQUIRED_RELEASE_ARTIFACTS.some(
(requiredPath) => !artifactPaths.includes(requiredPath)
)
) {
throw new TypeError("Release manifest artifact inventory is invalid");
}
Expand All @@ -640,7 +626,7 @@ export function parseReleaseManifest(value: unknown): DashboardReleaseManifest {
frontendCommit: value.components.frontendCommit as string,
},
formatVersion: value.formatVersion,
schema: parseSchema(value.schema, value.formatVersion),
schema: parseSchema(value.schema),
};
}

Expand Down Expand Up @@ -774,12 +760,8 @@ export async function loadRuntimeReleaseIdentity(
DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.minimum &&
manifest.schema.maximumCompatible ===
DASHBOARD_DATABASE_SCHEMA_COMPATIBILITY.maximum &&
// Format v1 has no migration inventory, so readiness cannot bind it to
// local migration identities. Remove this branch after the first
// managed-cutover rollback window no longer contains a v1 release.
(manifest.formatVersion === 1 ||
manifest.schema.migrationInventorySha256 ===
databaseMigrationInventorySha256()) &&
manifest.schema.migrationInventorySha256 ===
databaseMigrationInventorySha256() &&
manifest.schema.migrationRegistrySha256 === databaseMigrationRegistrySha256();
return {
artifactCount: manifest.artifacts.length,
Expand Down
1 change: 0 additions & 1 deletion backend/src/requestPolicy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ const BUCKET_CLEANUP_INTERVAL_MS = 60_000;
const BUCKET_STALE_MS = Math.max(apiRule.windowMs, authRule.windowMs) * 2;
const SAFE_REQUEST_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
const PUBLIC_API_METHODS = new Map<string, ReadonlySet<string>>([
["/api/health", new Set(["GET", "HEAD"])],
["/api/health/live", new Set(["GET", "HEAD"])],
["/api/health/ready", new Set(["GET", "HEAD"])],
["/api/auth/bootstrap", new Set(["GET", "HEAD"])],
Expand Down
32 changes: 0 additions & 32 deletions backend/src/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,50 +40,18 @@ async function ready() {
return json(snapshot, { status: snapshot.status === "isReady" ? 200 : 503 });
}

async function legacyReady() {
const snapshot = await readinessSnapshot();
const isReady = snapshot.status === "isReady";
return json(
{
status: isReady ? "isOk" : "notReady",
workerOnline: snapshot.checks.worker.ready,
},
{ status: isReady ? 200 : 503 }
);
}

async function diagnostics() {
return json(await diagnosticsSnapshot());
}

function retiredHealth() {
return json(
{
error: "Gone",
replacements: ["/api/health/live", "/api/health/ready"],
},
{ status: 410 }
);
}

function sessions() {
return json(gateway.getSessions());
}

const routeTable = {
Comment thread
mira-2026 marked this conversation as resolved.
"/health": {
GET: retiredHealth,
HEAD: retiredHealth,
},
"/api/health/diagnostics": {
GET: diagnostics,
},
// Transitional compatibility for the in-flight pre-readiness deploy
// executor. Remove after the atomic release executor has completed cutover.
"/api/health": {
GET: legacyReady,
HEAD: legacyReady,
},
"/api/health/live": {
GET: live,
HEAD: live,
Expand Down
Loading