diff --git a/backend/bun.lock b/backend/bun.lock index c141c77eb..9e6a0ded3 100644 --- a/backend/bun.lock +++ b/backend/bun.lock @@ -14,9 +14,9 @@ "eslint-plugin-prettier": "^5.5.6", "eslint-plugin-simple-import-sort": "^14.0.0", "eslint-plugin-unicorn": "^72.0.0", - "prettier": "^3.9.5", + "prettier": "^3.9.6", "typescript": "^6.0.3", - "typescript-eslint": "^8.64.0", + "typescript-eslint": "^8.65.0", }, }, }, @@ -59,25 +59,25 @@ "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], - "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.64.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/type-utils": "8.64.0", "@typescript-eslint/utils": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.64.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-CGvQPBxN3wZLu6Rz2kFUpZeoCm78xUic92ck39KPePkO1NPOwjCqdQnm5Q87tpWw9vcBvW8XLrDXjH9PWYtJ3Q=="], + "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.65.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/type-utils": "8.65.0", "@typescript-eslint/utils": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.65.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-IEgob78X12rHpUmtcwFsXhZdVGJtwTVP8FiCLZkR6GlYVrl2PcuB+KhCE5BlVC/eQpQnu8WXRtkHZuPar+gCRA=="], - "@typescript-eslint/parser": ["@typescript-eslint/parser@8.64.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-KA0OshtlcCCXmbfqyZkM5pV3/WNraJf7DkJRLpyrmwPtud57H5BDX7C3k0LPSPxpprfRL+cJDGabF10mvNCoCw=="], + "@typescript-eslint/parser": ["@typescript-eslint/parser@8.65.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.64.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.64.0", "@typescript-eslint/types": "^8.64.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-tk4WpOJ6IEbGrVHaNmM0YRrwAD3exZlIK3iadQNAxh4YKk6jvUQ4ecq18n+v7+meh+cJ3j+D8nbk8sRKhlwLQg=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.65.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.65.0", "@typescript-eslint/types": "^8.65.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q=="], - "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0" } }, "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w=="], + "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0" } }, "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.64.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-2yo8rRNKuzbVWQp5kslhANqZ2uDAeROQHBRZNPu8JDsHmeFNj/XJJhX/FhNUWmkHHvoNsKa6+tHJiig87EzsQw=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.65.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg=="], - "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/utils": "8.64.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-XWG4Fmmv/6SvyS9nH8jWrKs6terwJvE8cyRt1CzYYqzp9OrPhCT4cMc/f7C6RZCwG+qMmiffJS1/qJP8G1URtg=="], + "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/utils": "8.65.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-YjaZ7PRI5qY7ax2L3PbvX0rRyGtipAReCWs0mhhDBHjH/vl0g0BonaGXrKdKpMbIIsMIwDgbk/xzkBTyAltS5g=="], - "@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.64.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.64.0", "@typescript-eslint/tsconfig-utils": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-Pztpsn1aCE1oWDvDEfUk31nngvvF7vUB5SwHFEaZIFpvw7WJtqUHHL4plBZDA9HfWJJjL13BdG0YrJInTUvoVA=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.65.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.65.0", "@typescript-eslint/tsconfig-utils": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg=="], - "@typescript-eslint/utils": ["@typescript-eslint/utils@8.64.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ=="], + "@typescript-eslint/utils": ["@typescript-eslint/utils@8.65.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-mrtuL8Nsn6gi2H4mo5KMTp823M+3Q19Ew/i+Zlikq20tIMm99C3Ez0dCmkWWnxut20esQvTg8aUSEhMcAOXhEw=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-8C71BQkGjiMmXtop7pHVJu1l2NNShFdkCyD6a2ezzs5vU/L3LRtb69EtcteFwz0mYMPzIgOw0n6OV4VBUWZd7A=="], "@typescript/native": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], @@ -131,7 +131,7 @@ "baseline-browser-mapping": ["baseline-browser-mapping@2.10.42", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-c/jurFrDLyui7o1J86yLkRu4LMsTYcBohveus7/I2Hzdn9KIP2bdJPTue/lR1KH46enoPbD77GKeSYNdyPoD3Q=="], - "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], + "brace-expansion": ["brace-expansion@5.0.7", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA=="], "browserslist": ["browserslist@4.28.4", "", { "dependencies": { "baseline-browser-mapping": "^2.10.38", "caniuse-lite": "^1.0.30001799", "electron-to-chromium": "^1.5.376", "node-releases": "^2.0.48", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw=="], @@ -281,7 +281,7 @@ "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], - "prettier": ["prettier@3.9.5", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg=="], + "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], "prettier-linter-helpers": ["prettier-linter-helpers@1.0.1", "", { "dependencies": { "fast-diff": "^1.1.2" } }, "sha512-SxToR7P8Y2lWmv/kTzVLC1t/GDI2WGjMwNhLLE9qtH8Q13C+aEmuRlzDst4Up4s0Wc8sF2M+J57iB3cMLqftfg=="], @@ -319,7 +319,7 @@ "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], - "typescript-eslint": ["typescript-eslint@8.64.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.64.0", "@typescript-eslint/parser": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/utils": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-0qg+pDNMnqYzqH9AnNK+39tejHvsShUOUUoRUgtnTGE7QuMZhiFDnozq8nHJVq+Wae6NMLKNWLg5WmkcC/ndyQ=="], + "typescript-eslint": ["typescript-eslint@8.65.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.65.0", "@typescript-eslint/parser": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/utils": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-/ggrHAwyjENDusvyxbuqxAC2dTnZg/Z8F+fgQtYIz+L6n/9HfSlEZcFGV/NsMNa6CkGk0xUjUAFwC0vHOflvIA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], diff --git a/backend/package.json b/backend/package.json index f97ea0585..70dc53779 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,7 +8,9 @@ "scripts": { "build": "bun node_modules/@typescript/native/bin/tsc --noEmit && bun scripts/build.ts", "start": "NODE_ENV=production doppler run --config prd --project rajohan -- bun dist/serverStart.js", + "start:worker": "NODE_ENV=production doppler run --config prd --project rajohan -- bun dist/workerStart.js", "dev": "MIRA_DASHBOARD_DISABLE_SCHEDULER=1 doppler run --config prd --project rajohan -- bun --watch src/serverStart.ts", + "dev:worker": "doppler run --config prd --project rajohan -- bun --watch src/workerStart.ts", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write '**/*.{ts,js}'", @@ -26,8 +28,8 @@ "eslint-plugin-prettier": "^5.5.6", "eslint-plugin-simple-import-sort": "^14.0.0", "eslint-plugin-unicorn": "^72.0.0", - "prettier": "^3.9.5", + "prettier": "^3.9.6", "typescript": "^6.0.3", - "typescript-eslint": "^8.64.0" + "typescript-eslint": "^8.65.0" } } diff --git a/backend/scripts/build.ts b/backend/scripts/build.ts index e880b9380..4fecba2f3 100644 --- a/backend/scripts/build.ts +++ b/backend/scripts/build.ts @@ -8,7 +8,10 @@ await rm(outdir, { force: true, recursive: true }); await mkdir(outdir, { recursive: true }); const result = await Bun.build({ - entrypoints: [path.join(backendDirectory, "src/serverStart.ts")], + entrypoints: [ + path.join(backendDirectory, "src/serverStart.ts"), + path.join(backendDirectory, "src/workerStart.ts"), + ], format: "esm", outdir, packages: "external", diff --git a/backend/src/database.ts b/backend/src/database.ts index 3e9113456..c92d4ea5b 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -305,6 +305,60 @@ CREATE TABLE IF NOT EXISTS scheduled_job_runs ( CREATE INDEX IF NOT EXISTS idx_scheduled_job_runs_job_started ON scheduled_job_runs(job_id, started_at DESC); +CREATE TABLE IF NOT EXISTS scheduled_job_execution_policies ( + job_id TEXT PRIMARY KEY, + resource_class TEXT NOT NULL, + timeout_ms INTEGER NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY(job_id) REFERENCES scheduled_jobs(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS job_workers ( + id TEXT PRIMARY KEY, + capacity INTEGER NOT NULL, + started_at TEXT NOT NULL, + heartbeat_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_job_workers_heartbeat + ON job_workers(heartbeat_at DESC); + +CREATE TABLE IF NOT EXISTS job_executions ( + id TEXT PRIMARY KEY, + scheduled_job_id TEXT, + scheduled_run_id INTEGER, + action_key TEXT NOT NULL, + display_name TEXT NOT NULL, + resource_class TEXT NOT NULL, + priority INTEGER NOT NULL, + status TEXT NOT NULL, + trigger_type TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + queued_at TEXT NOT NULL, + available_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT, + lease_owner TEXT, + lease_expires_at TEXT, + heartbeat_at TEXT, + cancel_requested_at TEXT, + cancellable INTEGER NOT NULL DEFAULT 1, + attempt INTEGER NOT NULL DEFAULT 0, + timeout_ms INTEGER NOT NULL, + message TEXT, + output_json TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_job_executions_queue + ON job_executions(status, available_at, priority DESC, queued_at); + +CREATE INDEX IF NOT EXISTS idx_job_executions_scheduled_run + ON job_executions(scheduled_run_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_job_executions_active_scheduled_job + ON job_executions(scheduled_job_id) + WHERE scheduled_job_id IS NOT NULL AND status IN ('queued', 'running'); + CREATE TABLE IF NOT EXISTS chat_runtime_snapshots ( gateway_scope TEXT NOT NULL, session_key TEXT NOT NULL, diff --git a/backend/src/lib/jobResources.ts b/backend/src/lib/jobResources.ts new file mode 100644 index 000000000..110c333dc --- /dev/null +++ b/backend/src/lib/jobResources.ts @@ -0,0 +1,155 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +export const jobResourceClasses = [ + "interactive", + "light", + "network", + "host-heavy", + "exclusive", +] as const; + +export type JobResourceClass = (typeof jobResourceClasses)[number]; + +interface JobResourceContext { + resourceClass: JobResourceClass; +} + +interface JobResourcePolicy { + cpuWeight: number; + ioWeight: number; + memoryHigh: string; + memoryMax: string; + nice: number; + priority: number; + runtimeMaxSec: string; + tasksMax: number; +} + +const resourceContext = new AsyncLocalStorage(); + +const resourcePolicies: Record = { + interactive: { + cpuWeight: 60, + ioWeight: 60, + memoryHigh: "1G", + memoryMax: "2G", + nice: 5, + priority: 100, + runtimeMaxSec: "15m", + tasksMax: 64, + }, + light: { + cpuWeight: 30, + ioWeight: 40, + memoryHigh: "768M", + memoryMax: "1536M", + nice: 10, + priority: 60, + runtimeMaxSec: "30m", + tasksMax: 64, + }, + network: { + cpuWeight: 25, + ioWeight: 40, + memoryHigh: "1G", + memoryMax: "2G", + nice: 10, + priority: 50, + runtimeMaxSec: "30m", + tasksMax: 64, + }, + "host-heavy": { + cpuWeight: 15, + ioWeight: 15, + memoryHigh: "2G", + memoryMax: "4G", + nice: 15, + priority: 20, + runtimeMaxSec: "7h", + tasksMax: 128, + }, + exclusive: { + cpuWeight: 10, + ioWeight: 10, + memoryHigh: "3G", + memoryMax: "5G", + nice: 15, + priority: 10, + runtimeMaxSec: "7h", + tasksMax: 192, + }, +}; + +export function isJobResourceClass(value: unknown): value is JobResourceClass { + return ( + typeof value === "string" && + (jobResourceClasses as readonly string[]).includes(value) + ); +} + +export function jobResourcePriority(resourceClass: JobResourceClass): number { + return resourcePolicies[resourceClass].priority; +} + +export function withJobResourceClass( + resourceClass: JobResourceClass, + operation: () => T +): T { + return resourceContext.run({ resourceClass }, operation); +} + +function scopeOwnerProperties(environment: Record): string[] { + const owner = environment.MIRA_DASHBOARD_JOB_SCOPE_OWNER?.trim(); + if (!owner || !/^[A-Za-z0-9_.@-]+\.service$/u.test(owner)) return []; + return ["--property", `BindsTo=${owner}`, "--property", `After=${owner}`]; +} + +/** Wraps child commands in a constrained transient scope while a worker action runs. */ +export function scopedJobProcessCommand( + executable: string, + arguments_: readonly string[], + environment: Record = process.env +): { arguments: string[]; executable: string } { + const context = resourceContext.getStore(); + if ( + !context || + executable === "systemd-run" || + environment.MIRA_DASHBOARD_ENABLE_JOB_SCOPES !== "1" || + executable.endsWith("/systemd-run") + ) { + return { arguments: [...arguments_], executable }; + } + + const policy = resourcePolicies[context.resourceClass]; + return { + executable: "systemd-run", + arguments: [ + "--user", + "--scope", + "--expand-environment=no", + "--quiet", + "--collect", + `--nice=${policy.nice}`, + "--property", + `CPUWeight=${policy.cpuWeight}`, + "--property", + `IOWeight=${policy.ioWeight}`, + "--property", + `MemoryHigh=${policy.memoryHigh}`, + "--property", + `MemoryMax=${policy.memoryMax}`, + "--property", + `TasksMax=${policy.tasksMax}`, + "--property", + "KillMode=control-group", + "--property", + "TimeoutStopSec=20s", + "--property", + `RuntimeMaxSec=${policy.runtimeMaxSec}`, + ...scopeOwnerProperties(environment), + "--", + executable, + ...arguments_, + ], + }; +} diff --git a/backend/src/lib/processes.ts b/backend/src/lib/processes.ts index cf95c1318..b28c3b71d 100644 --- a/backend/src/lib/processes.ts +++ b/backend/src/lib/processes.ts @@ -1,9 +1,12 @@ +import { scopedJobProcessCommand } from "./jobResources.ts"; + export interface RunProcessOptions { cwd?: string; detached?: boolean; env?: Record; killSignal?: NodeJS.Signals; maxBuffer?: number; + signal?: AbortSignal; timeoutMs?: number; } @@ -52,8 +55,12 @@ export function spawnProcess( arguments_: readonly string[], options: RunProcessOptions = {} ): BunProcess { + if (options.signal?.aborted) { + throw new DOMException("Process aborted before start", "AbortError"); + } + const command = scopedJobProcessCommand(executable, arguments_); return Bun.spawn({ - cmd: [executable, ...arguments_], + cmd: [command.executable, ...command.arguments], cwd: options.cwd, detached: options.detached ?? true, env: options.env, @@ -86,14 +93,13 @@ export async function runProcess( let forceKillTimeout: Timer | undefined; let didTimeout = false; const timeoutMs = options.timeoutMs; - if (timeoutMs !== undefined) { - timeout = setTimeout(() => { - didTimeout = true; - try { - killProcessGroup(process, options.killSignal ?? "SIGTERM"); - } catch { - // The process may already have exited between scheduling and timeout. - } + const terminateProcess = () => { + try { + killProcessGroup(process, options.killSignal ?? "SIGTERM"); + } catch { + // The process may already have exited between scheduling and termination. + } + if (!forceKillTimeout) { forceKillTimeout = setTimeout(() => { try { killProcessGroup(process, "SIGKILL"); @@ -102,11 +108,20 @@ export async function runProcess( } }, DEFAULT_FORCE_KILL_GRACE_MS); forceKillTimeout.unref(); + } + }; + const abortFromSignal = () => terminateProcess(); + options.signal?.addEventListener("abort", abortFromSignal, { once: true }); + if (timeoutMs !== undefined) { + timeout = setTimeout(() => { + didTimeout = true; + terminateProcess(); }, timeoutMs); timeout.unref(); } function clearExitTimers(): void { + options.signal?.removeEventListener("abort", abortFromSignal); if (timeout) { clearTimeout(timeout); timeout = undefined; diff --git a/backend/src/requestPolicy.ts b/backend/src/requestPolicy.ts index a112025bb..644d5bd8e 100644 --- a/backend/src/requestPolicy.ts +++ b/backend/src/requestPolicy.ts @@ -3,6 +3,7 @@ import { isIP } from "node:net"; import type { Server } from "bun"; import { authUser, HttpError, isTrustedProxyAddress, json, requestIp } from "./http.ts"; +import { errorMessage, httpStatusCode } from "./lib/errors.ts"; type BunHandler = ( request: Request, @@ -208,6 +209,13 @@ function secureHandler(routePath: string, handler: BunHandler | Response): BunHa if (error instanceof SyntaxError) { return json({ error: "Invalid JSON" }, { status: 400 }); } + const mappedStatus = httpStatusCode(error); + if (mappedStatus !== 500) { + return json( + { error: errorMessage(error, "Request failed") }, + { status: mappedStatus } + ); + } console.error("[BunServer] Request failed:", error); return json({ error: "Internal server error" }, { status: 500 }); } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index d89b5f12e..d4e12ca67 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -13,6 +13,7 @@ import { databaseRoutes } from "./routes/databaseRoutes.ts"; import { dockerRoutes } from "./routes/dockerRoutes.ts"; import { execRoutes } from "./routes/execRoutes.ts"; import { fileRoutes } from "./routes/fileRoutes.ts"; +import { jobExecutionRoutes } from "./routes/jobExecutionRoutes.ts"; import { jobRoutes } from "./routes/jobRoutes.ts"; import { logRoutes } from "./routes/logRoutes.ts"; import { mediaRoutes } from "./routes/mediaRoutes.ts"; @@ -29,6 +30,7 @@ import { sttRoutes } from "./routes/sttRoutes.ts"; import { taskRoutes } from "./routes/taskRoutes.ts"; import { terminalRoutes } from "./routes/terminalRoutes.ts"; import { ttsRoutes } from "./routes/ttsRoutes.ts"; +import { getJobExecutionSummary } from "./services/jobExecutionQueue.ts"; const BACKEND_COMMIT = (() => { try { @@ -49,12 +51,22 @@ function backendCommit(): string { return BACKEND_COMMIT; } +function isWorkerOnline(): boolean { + try { + return getJobExecutionSummary().workerOnline; + } catch (error) { + console.warn("[Health] Failed to read job worker telemetry:", error); + return false; + } +} + function health() { return json({ status: "isOk", gatewayConnected: gateway.isConnected(), sessionCount: gateway.getSessions().length, backendCommit: backendCommit() || "unknown", + workerOnline: isWorkerOnline(), }); } @@ -83,6 +95,7 @@ const routeTable = { ...execRoutes, ...fileRoutes, ...jobRoutes, + ...jobExecutionRoutes, ...logRoutes, ...mediaRoutes, ...metricsRoutes, diff --git a/backend/src/routes/backupRoutes.ts b/backend/src/routes/backupRoutes.ts index b5315282e..49dbde21d 100644 --- a/backend/src/routes/backupRoutes.ts +++ b/backend/src/routes/backupRoutes.ts @@ -1,10 +1,9 @@ import { json } from "../http.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { - clearNeedsAttentionBackupJob, - getCurrentBackupJob, - mapBackupJob, - startManualBackup, + clearPersistedBackupAttention, + getPersistedBackupJob, + queueManualBackup, } from "../services/backups.ts"; type BackupType = "kopia" | "walg"; @@ -17,13 +16,12 @@ function backupResponseError(error: unknown, fallback: string): Response { } function backupStatus(type: BackupType): Response { - return json({ job: mapBackupJob(getCurrentBackupJob(type)) }); + return json({ job: getPersistedBackupJob(type) }); } async function runBackup(type: BackupType, fallback: string): Promise { try { - const job = await startManualBackup(type); - return json({ isOk: true, job: mapBackupJob(job) }); + return json({ isOk: true, job: queueManualBackup(type) }); } catch (error) { return backupResponseError(error, fallback); } @@ -34,8 +32,8 @@ async function clearNeedsAttention( fallback: string ): Promise { try { - const job = await clearNeedsAttentionBackupJob(type); - return json({ cleared: mapBackupJob(job), isOk: true }); + const job = await clearPersistedBackupAttention(type); + return json({ cleared: job, isOk: true }); } catch (error) { return backupResponseError(error, fallback); } diff --git a/backend/src/routes/cacheRoutes.ts b/backend/src/routes/cacheRoutes.ts index a5d1fead5..50d4bc531 100644 --- a/backend/src/routes/cacheRoutes.ts +++ b/backend/src/routes/cacheRoutes.ts @@ -8,7 +8,11 @@ import { } from "../lib/cacheStore.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { stringFallback } from "../lib/values.ts"; -import { refreshCacheProducer } from "../services/cacheRefresh.ts"; +import { cacheRefreshResourceClass } from "../services/cacheRefresh.ts"; +import { + enqueueAndWaitForJobExecution, + successfulJobExecutionOutput, +} from "../services/queuedJobExecution.ts"; import { listScheduledJobs } from "../services/scheduledJobs.ts"; import { getHeartbeatAutomationSnapshot } from "../services/taskAutomation.ts"; @@ -187,6 +191,7 @@ function compactDashboardJobs() { disableIntent: job.disableIntent, enabled: job.enabled, id: job.id, + isQueued: job.isQueued, isRunning: job.isRunning, lastRun: job.lastRun ? { @@ -199,6 +204,7 @@ function compactDashboardJobs() { : undefined, name: job.name, nextRunAt: job.nextRunAt, + resourceClass: job.resourceClass, })); } @@ -225,8 +231,7 @@ function mapCacheRowForResponse( }; } -async function refreshCacheKey(key: string) { - const result = await refreshCacheProducer(key); +async function refreshedCacheEntry(key: string, result: Record) { const refreshed = Array.isArray(result?.refreshed) ? result.refreshed : []; if (refreshed.length === 0) { throw Object.assign(new Error(`No cache keys refreshed for: ${key}`), { @@ -306,7 +311,22 @@ export const cacheRoutes = { const key = stringFallback(request.params.key).trim(); if (!key) return json({ error: "Missing cache key" }, { status: 400 }); try { - return json({ entry: await refreshCacheKey(key), isOk: true }); + const resourceClass = cacheRefreshResourceClass(key); + const execution = await enqueueAndWaitForJobExecution( + { + actionKey: "cache.refresh", + displayName: `Refresh cache: ${key}`, + payload: { key }, + resourceClass, + timeoutMs: 5 * 60 * 1000, + }, + { signal: request.signal } + ); + const entry = await refreshedCacheEntry( + key, + successfulJobExecutionOutput(execution) + ); + return json({ entry, isOk: true }); } catch (error) { return json( { error: errorMessage(error, "Cache refresh failed") }, diff --git a/backend/src/routes/dockerRoutes.ts b/backend/src/routes/dockerRoutes.ts index 20882d54b..aac75788c 100644 --- a/backend/src/routes/dockerRoutes.ts +++ b/backend/src/routes/dockerRoutes.ts @@ -1,13 +1,7 @@ import { database } from "../database.ts"; import { json, readJson } from "../http.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; -import { - type BunProcess, - killProcessGroup, - pipeProcessOutput, - runProcess, - spawnProcess, -} from "../lib/processes.ts"; +import { runProcess } from "../lib/processes.ts"; import { arrayFallback, nonEmptyEnvironmentFallback, @@ -18,16 +12,20 @@ import { import { type DockerUpdaterStepResult, isNonblockingRegistrationFailure, - registerDockerUpdaterScheduledJobs, - runDockerUpdaterService, } from "../services/dockerUpdater.ts"; import { - createManualScheduledJobRun, - finishScheduledJobRun, -} from "../services/scheduledJobs.ts"; + cancelJobExecution, + enqueueJobExecution, + getJobExecution, + type JobExecution, +} from "../services/jobExecutionQueue.ts"; +import { + successfulJobExecutionOutput, + waitForJobExecution, +} from "../services/queuedJobExecution.ts"; +import { enqueueScheduledJob } from "../services/scheduledJobs.ts"; const dockerBin = nonEmptyEnvironmentFallback("MIRA_DOCKER_BIN", "docker"); -const MAX_OUTPUT_CHARS = 100_000; const MAX_JOBS = 100; const MIN_LOG_TAIL = 50; const MAX_LOG_TAIL = 5000; @@ -114,19 +112,6 @@ interface DockerVolumeRow { Size: string; } -interface DockerExecJob { - code: number | undefined; - containerId: string; - containerPid?: number; - endedAt: number | undefined; - id: string; - process?: BunProcess; - startedAt: number; - status: "running" | "done"; - stderr: string; - stdout: string; -} - interface DockerUpdaterServiceRow { app_slug: string; compose_image_ref: string; @@ -146,8 +131,6 @@ interface DockerUpdaterServiceRow { service_name: string; } -const dockerExecJobs = new Map(); - const dockerUpdaterProjection = ` CAST(id AS TEXT) AS id, app_slug, @@ -171,13 +154,6 @@ function getDockerRoot(): string { return nonEmptyEnvironmentFallback("MIRA_DOCKER_ROOT", "/opt/docker"); } -function getDockerComposeWrapper(): string { - return nonEmptyEnvironmentFallback( - "MIRA_DOCKER_COMPOSE_WRAPPER", - `${getDockerRoot()}/bin/docker-compose-doppler` - ); -} - function parameters(request: Request): Record { return (request as Request & { params?: Record }).params ?? {}; } @@ -189,10 +165,6 @@ function queryNumber(request: Request, key: string, fallback: number): number { return Number.isFinite(parsed) ? parsed : fallback; } -function trimOutput(text: string): string { - return text.length <= MAX_OUTPUT_CHARS ? text : text.slice(-MAX_OUTPUT_CHARS); -} - function dockerIdentifier(value: unknown): string | undefined { const identifier = stringFallback(value).trim(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(identifier)) return undefined; @@ -298,11 +270,12 @@ function parseDockerSizeToBytes(sizeRaw: string | undefined): number { ); } -async function runDocker(arguments_: string[]): Promise { +async function runDocker(arguments_: string[], signal?: AbortSignal): Promise { const { code, stderr, stdout } = await runProcess(dockerBin, arguments_, { cwd: getDockerRoot(), env: process.env, maxBuffer: 10 * 1024 * 1024, + signal, timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, }); if (code !== 0) { @@ -315,25 +288,6 @@ async function runDocker(arguments_: string[]): Promise { return String(stdout); } -async function runCompose( - arguments_: string[] -): Promise<{ stderr: string; stdout: string }> { - const result = await runProcess(getDockerComposeWrapper(), arguments_, { - cwd: getDockerRoot(), - env: process.env, - maxBuffer: 20 * 1024 * 1024, - timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, - }); - if (result.code !== 0) { - throw new Error( - `docker compose ${arguments_.join(" ")} failed with exit code ${ - result.code - }: ${result.stderr.trim() || result.stdout.trim()}` - ); - } - return result; -} - async function getContainerInspectMap(containerIds: string[]) { if (containerIds.length === 0) return new Map(); const parsedRows = JSON.parse( @@ -610,6 +564,15 @@ function blockingDockerUpdaterFailures(steps: DockerUpdaterStepResult[]) { ); } +function dockerUpdaterSteps(execution: JobExecution): DockerUpdaterStepResult[] { + const steps = execution.output.steps; + if (!Array.isArray(steps)) { + successfulJobExecutionOutput(execution); + throw new Error("Docker updater result was missing"); + } + return steps as DockerUpdaterStepResult[]; +} + export async function getDockerUpdaterEvents(limit: number) { const boundedLimit = Math.max(1, Math.min(200, Math.floor(limit))); const rows = database @@ -683,6 +646,42 @@ function statusCodeFromError(error: unknown): number { : 500; } +async function runQueuedDockerAction(options: { + actionKey: string; + displayName: string; + payload?: Record; + resourceClass?: "host-heavy" | "interactive" | "exclusive"; + timeoutMs: number; +}): Promise> { + const execution = enqueueJobExecution({ + actionKey: options.actionKey, + displayName: options.displayName, + payload: options.payload, + resourceClass: options.resourceClass ?? "host-heavy", + timeoutMs: options.timeoutMs, + }); + return successfulJobExecutionOutput( + await waitForJobExecution(execution.id, { + timeoutMs: options.timeoutMs + 30 * 60 * 1000, + }) + ); +} + +function outputString(output: Record, key: string): string { + return typeof output[key] === "string" ? output[key] : ""; +} + +function outputNumber(output: Record, key: string): number | undefined { + return typeof output[key] === "number" && Number.isFinite(output[key]) + ? output[key] + : undefined; +} + +function dockerExecExecution(jobId: string): JobExecution | undefined { + const execution = getJobExecution(jobId); + return execution?.actionKey === "docker.exec" ? execution : undefined; +} + async function runStackAction(request: Request): Promise { const body = await readDockerJson<{ action?: unknown; service?: unknown }>(request); if (body instanceof Response) return body; @@ -699,130 +698,17 @@ async function runStackAction(request: Request): Promise { ) { return json({ error: "Invalid service name" }, { status: 400 }); } - const arguments_: string[] = [body.action]; - if (body.service !== undefined) arguments_.push(body.service); - const result = await runCompose(arguments_); + const result = await runQueuedDockerAction({ + actionKey: "docker.stack.action", + displayName: `Docker stack ${body.action}`, + payload: { action: body.action, service: body.service }, + timeoutMs: 2 * 60 * 1000, + }); return json({ - output: [result.stdout, result.stderr].filter(Boolean).join("\n").trim(), + output: outputString(result, "output"), }); } -function cleanupDockerExecJobs(): void { - if (dockerExecJobs.size <= MAX_JOBS) return; - const doneJobs = dockerExecJobs - .values() - .filter((job) => job.status === "done") - .toArray() - .toSorted((a, b) => a.startedAt - b.startedAt); - const jobsToDelete = doneJobs.slice(0, dockerExecJobs.size - MAX_JOBS); - for (const job of jobsToDelete) { - dockerExecJobs.delete(job.id); - } -} - -function activeDockerExecJobCount(): number { - return dockerExecJobs - .values() - .filter((job) => job.status !== "done") - .toArray().length; -} - -function settleDockerExecJob(containerId: string, command: string, jobId: string): void { - const pidMarker = `__MIRA_DOCKER_EXEC_PID_${jobId}:`; - let child: BunProcess; - try { - child = spawnProcess( - dockerBin, - [ - "exec", - "-e", - `MIRA_DASHBOARD_EXEC_COMMAND=${command}`, - containerId, - "sh", - "-lc", - String.raw`if command -v setsid >/dev/null 2>&1; then exec setsid sh -lc 'printf '\''${pidMarker}%s\n'\'' "$$"; exec sh -lc "$MIRA_DASHBOARD_EXEC_COMMAND"'; fi; printf '${pidMarker}%s\n' "$$"; exec sh -lc "$MIRA_DASHBOARD_EXEC_COMMAND"`, - ], - { - cwd: getDockerRoot(), - env: process.env, - } - ); - } catch (error) { - const job = dockerExecJobs.get(jobId); - if (job) { - job.status = "done"; - job.code = 1; - job.stderr = errorMessage(error, "Docker exec failed"); - job.endedAt = Date.now(); - cleanupDockerExecJobs(); - } - return; - } - const job = dockerExecJobs.get(jobId); - if (job) job.process = child; - - let stdoutPrefix = ""; - const stdoutDone = pipeProcessOutput( - child.stdout as ReadableStream | undefined, - (data) => { - const current = dockerExecJobs.get(jobId); - if (!current) return; - const output = stdoutPrefix + String(data); - const newlineIndex = output.indexOf("\n"); - if (newlineIndex === -1 && current.containerPid === undefined) { - stdoutPrefix = output; - return; - } - let userOutput = output; - if (current.containerPid === undefined) { - const firstLine = output.slice(0, newlineIndex).trim(); - if (firstLine.startsWith(pidMarker)) { - const pid = Number(firstLine.slice(pidMarker.length)); - if (Number.isSafeInteger(pid) && pid > 0) { - current.containerPid = pid; - } - userOutput = output.slice(newlineIndex + 1); - } - stdoutPrefix = ""; - } - current.stdout = trimOutput(current.stdout + userOutput); - } - ); - const stderrDone = pipeProcessOutput( - child.stderr as ReadableStream | undefined, - (data) => { - const current = dockerExecJobs.get(jobId); - if (current) current.stderr = trimOutput(current.stderr + String(data)); - } - ); - void (async () => { - const code = await child.exited; - await Promise.all([stdoutDone, stderrDone]); - return code; - })() - .then((code) => { - const current = dockerExecJobs.get(jobId); - if (!current) return; - current.status = "done"; - current.code = code; - current.endedAt = Date.now(); - current.process = undefined; - cleanupDockerExecJobs(); - }) - .catch((error: unknown) => { - const current = dockerExecJobs.get(jobId); - if (!current) return; - current.status = "done"; - current.code = 1; - current.stderr = trimOutput( - `${current.stderr}\n${errorMessage(error, "Docker exec failed")}`.trim() - ); - current.endedAt = Date.now(); - current.process = undefined; - cleanupDockerExecJobs(); - }); -} - export const dockerRoutes = { "/api/docker/containers": { GET: async () => json({ containers: await getContainers() }), @@ -867,10 +753,16 @@ export const dockerRoutes = { ) { return json({ error: "Invalid container action" }, { status: 400 }); } + const action = body.action; const details = await getContainerDetails(containerId); if (!details) return json({ error: "Container not found" }, { status: 404 }); - await runDocker([body.action, details.id]); - return json({ output: `${body.action} sent to ${details.name}` }); + await runQueuedDockerAction({ + actionKey: "docker.container.action", + displayName: `Docker container ${action}`, + payload: { action, containerId: details.id }, + timeoutMs: 2 * 60 * 1000, + }); + return json({ output: `${action} sent to ${details.name}` }); }, }, "/api/docker/containers/:containerId/logs": { @@ -906,70 +798,52 @@ export const dockerRoutes = { }, "/api/docker/exec/:jobId": { GET: (request: Request) => { - const job = dockerExecJobs.get(stringFallback(parameters(request).jobId)); - if (!job) + const jobId = stringFallback(parameters(request).jobId); + const execution = dockerExecExecution(jobId); + if (!execution) return json({ error: "Docker exec job not found" }, { status: 404 }); + const output = execution.output; + const isTerminal = ["success", "failed", "cancelled"].includes( + execution.status + ); return json({ - code: job.code, - containerId: job.containerId, - endedAt: job.endedAt, - jobId: job.id, - startedAt: job.startedAt, - status: job.status, - stderr: job.stderr, - stdout: job.stdout, + code: outputNumber(output, "code"), + containerId: + outputString(output, "containerId") || + stringFallback(execution.payload.containerId), + endedAt: isTerminal + ? (outputNumber(output, "endedAt") ?? + (execution.finishedAt + ? Date.parse(execution.finishedAt) + : undefined)) + : undefined, + jobId: execution.id, + startedAt: + outputNumber(output, "startedAt") ?? + Date.parse(execution.startedAt ?? execution.queuedAt), + status: isTerminal ? "done" : "running", + stderr: outputString(output, "stderr"), + stdout: outputString(output, "stdout"), }); }, }, "/api/docker/exec/:jobId/stop": { - POST: async (request: Request) => { - const job = dockerExecJobs.get(stringFallback(parameters(request).jobId)); - if (!job) + POST: (request: Request) => { + const jobId = stringFallback(parameters(request).jobId); + const execution = dockerExecExecution(jobId); + if (!execution) return json({ error: "Docker exec job not found" }, { status: 404 }); - if (job.status !== "running") { + if (execution.status !== "queued" && execution.status !== "running") { return json({ error: "Job is not running" }, { status: 400 }); } - if (!job.process) { - return json({ error: "Process not available" }, { status: 400 }); - } - let containerStopError: string | undefined; - if (job.containerPid) { - try { - await runDocker([ - "exec", - job.containerId, - "kill", - "-TERM", - `-${job.containerPid}`, - ]); - } catch (groupError) { - try { - await runDocker([ - "exec", - job.containerId, - "kill", - "-TERM", - String(job.containerPid), - ]); - } catch (processError) { - containerStopError = `${errorMessage(groupError, "Failed to stop in-container process group")}; ${errorMessage(processError, "Failed to stop in-container process")}`; - job.stderr = trimOutput( - `${job.stderr}\n${containerStopError}`.trim() - ); - } - } - } try { - killProcessGroup(job.process, "SIGTERM"); + cancelJobExecution(execution.id); } catch (error) { return json( { error: errorMessage(error, "Failed to stop Docker exec job") }, - { status: 500 } + { status: statusCodeFromError(error) } ); } - if (containerStopError) { - return json({ error: containerStopError }, { status: 500 }); - } return json({ isSuccess: true }); }, }, @@ -993,26 +867,35 @@ export const dockerRoutes = { if (!containerId) { return json({ error: "Container not found" }, { status: 404 }); } - cleanupDockerExecJobs(); - if (activeDockerExecJobCount() >= MAX_JOBS) { + const activeJobs = database + .prepare( + `SELECT COUNT(*) AS count FROM job_executions + WHERE action_key = 'docker.exec' + AND status IN ('queued', 'running')` + ) + .get() as { count: number }; + if (activeJobs.count >= MAX_JOBS) { return json( { error: "Too many active Docker exec jobs" }, { status: 429 } ); } - const jobId = Bun.randomUUIDv7(); - dockerExecJobs.set(jobId, { - code: undefined, - containerId, - endedAt: undefined, - id: jobId, - startedAt: Date.now(), - status: "running", - stderr: "", - stdout: "", - }); - settleDockerExecJob(containerId, body.command, jobId); - return json({ jobId }); + let execution: JobExecution; + try { + execution = enqueueJobExecution({ + actionKey: "docker.exec", + displayName: "Docker container exec", + payload: { command: body.command, containerId }, + resourceClass: "exclusive", + timeoutMs: 7 * 60 * 60 * 1000, + }); + } catch (error) { + return json( + { error: errorMessage(error, "Docker exec failed to start") }, + { status: statusCodeFromError(error) } + ); + } + return json({ jobId: execution.id }); }, }, "/api/docker/images": { @@ -1022,7 +905,12 @@ export const dockerRoutes = { DELETE: async (request: Request) => { const imageId = dockerImageIdentifier(parameters(request).imageId); if (!imageId) return invalidDockerIdentifier("imageId"); - await runDocker(["image", "rm", imageId]); + await runQueuedDockerAction({ + actionKey: "docker.image.delete", + displayName: "Delete Docker image", + payload: { imageId }, + timeoutMs: 2 * 60 * 1000, + }); return json({ isSuccess: true }); }, }, @@ -1033,13 +921,29 @@ export const dockerRoutes = { if (body?.target === "images") { return json({ isSuccess: true, - output: await runDocker(["image", "prune", "-a", "-f"]), + output: outputString( + await runQueuedDockerAction({ + actionKey: "docker.prune.images", + displayName: "Prune Docker images", + payload: { target: "images" }, + timeoutMs: 10 * 60 * 1000, + }), + "output" + ), }); } if (body?.target === "volumes") { return json({ isSuccess: true, - output: await runDocker(["volume", "prune", "-f"]), + output: outputString( + await runQueuedDockerAction({ + actionKey: "docker.prune.volumes", + displayName: "Prune Docker volumes", + payload: { target: "volumes" }, + timeoutMs: 10 * 60 * 1000, + }), + "output" + ), }); } return json({ error: "Invalid prune target" }, { status: 400 }); @@ -1056,37 +960,18 @@ export const dockerRoutes = { }, "/api/docker/updater/run": { POST: async () => { - let scheduledRun; - try { - registerDockerUpdaterScheduledJobs(); - scheduledRun = createManualScheduledJobRun("docker.updater"); - } catch (error) { - return json( - { error: errorMessage(error, "Docker updater failed") }, - { status: statusCodeFromError(error) } - ); - } try { - const steps = await runDockerUpdaterService(); - finishScheduledJobRun( - scheduledRun, - blockingDockerUpdaterFailures(steps).length === 0 - ? "success" - : "failed", - undefined, - { steps } + const scheduledRun = enqueueScheduledJob("docker.updater", "manual"); + const execution = await waitForJobExecution( + scheduledRun.executionId as string, + { timeoutMs: 60 * 60 * 1000 } ); + const steps = dockerUpdaterSteps(execution); return json({ isSuccess: blockingDockerUpdaterFailures(steps).length === 0, steps, }); } catch (error) { - finishScheduledJobRun( - scheduledRun, - "failed", - errorMessage(error, "Docker updater failed"), - { steps: [] } - ); return json( { error: errorMessage(error, "Docker updater failed") }, { status: statusCodeFromError(error) } @@ -1116,34 +1001,21 @@ export const dockerRoutes = { if (!service.enabled) { return json({ error: "Updater service is disabled" }, { status: 400 }); } - let scheduledRun; - try { - registerDockerUpdaterScheduledJobs(); - scheduledRun = createManualScheduledJobRun("docker.updater"); - } catch (error) { - return json( - { error: errorMessage(error, "Docker updater failed") }, - { status: statusCodeFromError(error) } - ); - } let steps: DockerUpdaterStepResult[]; try { - steps = await runDockerUpdaterService(serviceId); - finishScheduledJobRun( - scheduledRun, - blockingDockerUpdaterFailures(steps).length === 0 - ? "success" - : "failed", - undefined, - { serviceId, steps } + const execution = enqueueJobExecution({ + actionKey: "docker.updater", + displayName: `Update Docker service ${serviceId}`, + payload: { serviceId }, + resourceClass: "exclusive", + timeoutMs: 30 * 60 * 1000, + }); + steps = dockerUpdaterSteps( + await waitForJobExecution(execution.id, { + timeoutMs: 60 * 60 * 1000, + }) ); } catch (error) { - finishScheduledJobRun( - scheduledRun, - "failed", - errorMessage(error, "Docker updater failed"), - { serviceId } - ); return json( { error: errorMessage(error, "Docker updater failed") }, { status: statusCodeFromError(error) } @@ -1203,7 +1075,12 @@ export const dockerRoutes = { DELETE: async (request: Request) => { const volumeName = dockerIdentifier(parameters(request).volumeName); if (!volumeName) return invalidDockerIdentifier("volumeName"); - await runDocker(["volume", "rm", volumeName]); + await runQueuedDockerAction({ + actionKey: "docker.volume.delete", + displayName: "Delete Docker volume", + payload: { volumeName }, + timeoutMs: 2 * 60 * 1000, + }); return json({ isSuccess: true }); }, }, diff --git a/backend/src/routes/jobExecutionRoutes.ts b/backend/src/routes/jobExecutionRoutes.ts new file mode 100644 index 000000000..e382aeb46 --- /dev/null +++ b/backend/src/routes/jobExecutionRoutes.ts @@ -0,0 +1,129 @@ +import { json } from "../http.ts"; +import { errorMessage, httpStatusCode } from "../lib/errors.ts"; +import { + cancelJobExecution, + getJobExecution, + getJobExecutionSummary, + type JobExecution, + listJobExecutions, +} from "../services/jobExecutionQueue.ts"; + +type ParametersRequest = Request & { params: Record }; + +function publicExecution( + execution: JobExecution, + options: { includeOutput?: boolean } = {} +) { + return { + id: execution.id, + scheduledJobId: execution.scheduledJobId, + scheduledRunId: execution.scheduledRunId, + actionKey: execution.actionKey, + displayName: execution.displayName, + resourceClass: execution.resourceClass, + status: execution.status, + triggerType: execution.triggerType, + queuedAt: execution.queuedAt, + availableAt: execution.availableAt, + startedAt: execution.startedAt, + finishedAt: execution.finishedAt, + heartbeatAt: execution.heartbeatAt, + cancelRequestedAt: execution.cancelRequestedAt, + cancellable: execution.cancellable, + attempt: execution.attempt, + message: execution.message, + ...(options.includeOutput && { output: execution.output }), + }; +} + +function executionLimit(request: Request): number { + const value = new URL(request.url).searchParams.get("limit"); + if (!value || !/^\d{1,3}$/u.test(value)) return 50; + return Number(value); +} + +function isValidExecutionId(id: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test( + id + ); +} + +export const jobExecutionRoutes = { + "/api/job-executions": { + GET: (request: Request) => { + try { + return json({ + executions: listJobExecutions(executionLimit(request)).map( + (execution) => publicExecution(execution) + ), + summary: getJobExecutionSummary(), + }); + } catch (error) { + console.error("[jobExecutionRoutes] Queue lookup failed", error); + return json( + { error: "Job execution queue lookup failed" }, + { status: 500 } + ); + } + }, + }, + "/api/job-executions/:id": { + GET: (request: ParametersRequest<"id">) => { + const id = String(request.params.id); + if (!isValidExecutionId(id)) { + return json({ error: "Invalid job execution id" }, { status: 400 }); + } + try { + const execution = getJobExecution(id); + return execution + ? json({ + execution: publicExecution(execution, { + includeOutput: true, + }), + }) + : json({ error: "Job execution not found" }, { status: 404 }); + } catch (error) { + console.error("[jobExecutionRoutes] Queue detail lookup failed", error); + return json( + { error: "Job execution queue lookup failed" }, + { status: 500 } + ); + } + }, + }, + "/api/job-executions/:id/cancel": { + POST: (request: ParametersRequest<"id">) => { + const id = String(request.params.id); + if (!isValidExecutionId(id)) { + return json({ error: "Invalid job execution id" }, { status: 400 }); + } + try { + const execution = cancelJobExecution(id); + return json({ + execution: publicExecution(execution), + isOk: true, + }); + } catch (error) { + const status = httpStatusCode(error); + if (status === 500) { + console.error( + "[jobExecutionRoutes] Queue cancellation failed", + error + ); + } + return json( + { + error: + status === 500 + ? "Job execution cancellation failed" + : errorMessage( + error, + "Job execution cancellation failed" + ), + }, + { status } + ); + } + }, + }, +} as const; diff --git a/backend/src/routes/jobRoutes.ts b/backend/src/routes/jobRoutes.ts index b0399121a..94708e4e1 100644 --- a/backend/src/routes/jobRoutes.ts +++ b/backend/src/routes/jobRoutes.ts @@ -5,11 +5,11 @@ import { normalizeJobDisableIntent, } from "../services/jobDisableIntent.ts"; import { + enqueueScheduledJob, getScheduledJob, isScheduledJobValidationError, listScheduledJobRuns, listScheduledJobs, - runScheduledJob, type ScheduledJobScheduleType, updateScheduledJob, } from "../services/scheduledJobs.ts"; @@ -177,10 +177,10 @@ export const jobRoutes = { }, "/api/jobs/:id/run": { - POST: async (request: ParametersRequest<"id">) => { + POST: (request: ParametersRequest<"id">) => { try { - const run = await runScheduledJob(String(request.params.id), "manual"); - return json({ isOk: run.status === "success", run }); + const run = enqueueScheduledJob(String(request.params.id), "manual"); + return json({ isOk: true, run }, { status: 202 }); } catch (error) { return json( { error: errorMessage(error, "Scheduled job run failed") }, diff --git a/backend/src/routes/openclawConfigRoutes.ts b/backend/src/routes/openclawConfigRoutes.ts index b31e794fb..e6bf24113 100644 --- a/backend/src/routes/openclawConfigRoutes.ts +++ b/backend/src/routes/openclawConfigRoutes.ts @@ -5,8 +5,12 @@ import path from "node:path"; import gateway from "../gateway.ts"; import { json, readJson } from "../http.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; -import { runProcess } from "../lib/processes.ts"; import { objectFallback, stringFallback } from "../lib/values.ts"; +import { OPENCLAW_GATEWAY_RESTART_ACTION } from "../services/openclawActions.ts"; +import { + enqueueAndWaitForJobExecution, + successfulJobExecutionOutput, +} from "../services/queuedJobExecution.ts"; type ParametersRequest = Request & { params: Record }; @@ -83,14 +87,6 @@ function resolveOpenClawHome(): string | undefined { return resolveSafeAbsolutePath(path.join(homeDirectory, ".openclaw")); } -function getOpenClawBin(): string { - const homeDirectory = process.env.HOME?.trim() || os.homedir(); - return ( - process.env.OPENCLAW_BIN?.trim() || - path.join(homeDirectory, ".npm-global/bin/openclaw") - ); -} - function readSkillDescription(skillPath: string): string | undefined { try { const content = fs.readFileSync(path.join(skillPath, "SKILL.md"), "utf8"); @@ -310,23 +306,18 @@ export const openclawConfigRoutes = { "/api/restart": { POST: async () => { try { - const { code, stderr, stdout } = await runProcess( - getOpenClawBin(), - ["gateway", "restart"], - { - timeoutMs: 30_000, - } - ); - if (code !== 0) { - throw new Error( - stderr.trim() || stdout.trim() || `openclaw exited ${code}` - ); - } + const execution = await enqueueAndWaitForJobExecution({ + actionKey: OPENCLAW_GATEWAY_RESTART_ACTION, + displayName: "Restart OpenClaw Gateway", + resourceClass: "exclusive", + timeoutMs: 60_000, + }); + successfulJobExecutionOutput(execution); return json({ isOk: true }); } catch (error) { return json( { error: errorMessage(error, "Failed to restart gateway") }, - { status: 500 } + { status: httpStatusCode(error) } ); } }, diff --git a/backend/src/routes/opsRoutes.ts b/backend/src/routes/opsRoutes.ts index 6a6de828f..6944b8653 100644 --- a/backend/src/routes/opsRoutes.ts +++ b/backend/src/routes/opsRoutes.ts @@ -1,6 +1,11 @@ import { database } from "../database.ts"; import { json } from "../http.ts"; +import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { runElevatedLogRotationService } from "../services/logRotation.ts"; +import { + enqueueAndWaitForJobExecution, + successfulJobExecutionOutput, +} from "../services/queuedJobExecution.ts"; const LOG_ROTATION_STATE_KEY = "log_rotation.state"; @@ -93,15 +98,37 @@ export async function runLogRotation(options: { async function runLogRotationResponse(isDryRun: boolean) { try { - const { result, stderr } = await runLogRotation({ isDryRun }); + const execution = await enqueueAndWaitForJobExecution({ + actionKey: "ops.log-rotation", + displayName: isDryRun ? "Log rotation dry run" : "Log rotation manual run", + payload: { isDryRun }, + resourceClass: "host-heavy", + timeoutMs: 10 * 60 * 1000, + }); + const output = execution.output; + const logRotation = output.logRotation; + if (!logRotation || typeof logRotation !== "object") { + successfulJobExecutionOutput(execution); + throw new Error("Log rotation result was missing"); + } + const { result, stderr } = logRotation as unknown as LogRotationResult; return json({ isSuccess: result?.isOk === true, result, stderr, }); } catch (error) { - console.error("[opsRoutes] Ops route failed", error); - return json({ error: "Ops route failed" }, { status: 500 }); + const status = httpStatusCode(error); + if (status === 500) console.error("[opsRoutes] Ops route failed", error); + return json( + { + error: + status === 500 + ? "Ops route failed" + : errorMessage(error, "Ops route failed"), + }, + { status } + ); } } diff --git a/backend/src/routes/pullRequestRoutes.ts b/backend/src/routes/pullRequestRoutes.ts index 3870b042a..39e7c7a18 100644 --- a/backend/src/routes/pullRequestRoutes.ts +++ b/backend/src/routes/pullRequestRoutes.ts @@ -1,16 +1,14 @@ import { json, readJson } from "../http.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { - approvePullRequest, - approvePullRequestReview, - ensureProductionCheckout, - ensureProductionReadyForDeploy, getProductionCheckoutStatus, listDashboardPullRequests, + prepareAndStartDeployLatest, readDeploymentJobs, - rejectPullRequest, - startDeployLatest, - updatePullRequestBranch, + runPullRequestApproval, + runPullRequestBranchUpdate, + runPullRequestRejection, + runPullRequestReviewApproval, validatePrNumber, } from "../services/pullRequests.ts"; @@ -52,7 +50,7 @@ export const pullRequestRoutes = { const body = request.body ? await readJson<{ deploy?: unknown } | undefined>(request) : undefined; - return json(await approvePullRequest(number, body?.deploy === true)); + return json(await runPullRequestApproval(number, body?.deploy === true)); } catch (error) { return routeError(error); } @@ -70,7 +68,7 @@ export const pullRequestRoutes = { typeof body?.comment === "string" && body.comment.trim() ? body.comment.trim() : "Closed from Mira Dashboard after Rajohan rejected it."; - return json(await rejectPullRequest(number, comment)); + return json(await runPullRequestRejection(number, comment)); } catch (error) { return routeError(error); } @@ -81,7 +79,7 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json(await approvePullRequestReview(number)); + return json(await runPullRequestReviewApproval(number)); } catch (error) { return routeError(error); } @@ -92,7 +90,7 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json(await updatePullRequestBranch(number)); + return json(await runPullRequestBranchUpdate(number)); } catch (error) { return routeError(error); } @@ -101,9 +99,10 @@ export const pullRequestRoutes = { "/api/pull-requests/deploy": { POST: async () => { try { - await ensureProductionCheckout(); - await ensureProductionReadyForDeploy(); - return json({ deployment: startDeployLatest(), isOk: true }); + return json({ + deployment: await prepareAndStartDeployLatest(), + isOk: true, + }); } catch (error) { return routeError(error); } diff --git a/backend/src/serverStart.ts b/backend/src/serverStart.ts index c57aebac7..592d6732b 100644 --- a/backend/src/serverStart.ts +++ b/backend/src/serverStart.ts @@ -2,19 +2,11 @@ import { getPersistedGatewayToken } from "./auth.ts"; import gateway from "./gateway.ts"; import { createServer, resolveListenPort } from "./server.ts"; import { shouldStartScheduledJobs } from "./serverStartPolicy.ts"; -import { registerBackupScheduledJobs } from "./services/backups.ts"; -import { registerCacheRefreshScheduledJobs } from "./services/cacheRefresh.ts"; -import { registerDockerUpdaterScheduledJobs } from "./services/dockerUpdater.ts"; -import { registerGitHygieneScheduledJobs } from "./services/gitHygiene.ts"; -import { registerLogRotationScheduledJobs } from "./services/logRotation.ts"; -import { - startScheduledJobScheduler, - stopScheduledJobScheduler, -} from "./services/scheduledJobs.ts"; +import { startDashboardJobWorker, stopDashboardJobWorker } from "./services/jobWorker.ts"; +import { registerPullRequestJobLifecycleHandlers } from "./services/pullRequests.ts"; const serverStartState: { activeServer: ReturnType | undefined; - stopSchedulerOnServerClose?: () => void; isStarting: boolean; } = { activeServer: undefined, @@ -23,32 +15,20 @@ const serverStartState: { export { runLogRotationCli } from "./services/logRotation.ts"; -function installSchedulerCloseCleanup(): void { - if (serverStartState.stopSchedulerOnServerClose) { - return; - } - serverStartState.stopSchedulerOnServerClose = () => { - stopScheduledJobScheduler(); - serverStartState.stopSchedulerOnServerClose = undefined; - }; -} - -function rollbackBackgroundServiceStartup(function_: () => void, label: string): void { +function rollbackBackgroundServiceStartup( + function_: () => void | Promise, + label: string +): void { try { - function_(); + const result = function_(); + if (result instanceof Promise) { + void result.catch((cleanupError) => console.error(label, cleanupError)); + } } catch (cleanupError) { console.error(label, cleanupError); } } -function removeSchedulerCloseCleanup(): void { - if (!serverStartState.stopSchedulerOnServerClose) { - return; - } - - serverStartState.stopSchedulerOnServerClose = undefined; -} - export function resolveGatewayToken( environment = process.env, persistedToken = getPersistedGatewayToken @@ -64,8 +44,8 @@ export function resolveGatewayToken( /** Starts Gateway and notification monitors after the HTTP server is listening. */ export function handleServerListening(): void { let isGatewayStarted = false; - let isScheduledJobSchedulerStarted = false; try { + registerPullRequestJobLifecycleHandlers(); const token = resolveGatewayToken(); if (token) { gateway.init(token); @@ -77,24 +57,10 @@ export function handleServerListening(): void { } if (shouldStartScheduledJobs()) { - registerBackupScheduledJobs(); - registerCacheRefreshScheduledJobs(); - registerDockerUpdaterScheduledJobs(); - registerGitHygieneScheduledJobs(); - registerLogRotationScheduledJobs(); - startScheduledJobScheduler(); - isScheduledJobSchedulerStarted = true; - installSchedulerCloseCleanup(); + startDashboardJobWorker(); } } catch (error) { console.error("[Backend] Failed to start background services:", error); - if (isScheduledJobSchedulerStarted) { - removeSchedulerCloseCleanup(); - rollbackBackgroundServiceStartup( - stopScheduledJobScheduler, - "[Backend] Failed to stop scheduled job scheduler:" - ); - } if (isGatewayStarted) { rollbackBackgroundServiceStartup( () => gateway.shutdown(), @@ -135,8 +101,7 @@ export async function stopBackendServer(): Promise { const server = serverStartState.activeServer; serverStartState.activeServer = undefined; try { - removeSchedulerCloseCleanup(); - stopScheduledJobScheduler(); + await stopDashboardJobWorker(); gateway.shutdown(); } finally { await server?.stop(true); diff --git a/backend/src/serverStartPolicy.ts b/backend/src/serverStartPolicy.ts index 677dd7af2..99d850537 100644 --- a/backend/src/serverStartPolicy.ts +++ b/backend/src/serverStartPolicy.ts @@ -1,5 +1,17 @@ +export type DashboardExecutionRole = "combined" | "web" | "worker"; + +export function dashboardExecutionRole( + environment: Record = process.env +): DashboardExecutionRole { + const role = environment.MIRA_DASHBOARD_EXECUTION_ROLE?.trim(); + return role === "web" || role === "worker" ? role : "combined"; +} + export function shouldStartScheduledJobs( environment: Record = process.env ): boolean { - return environment.MIRA_DASHBOARD_DISABLE_SCHEDULER !== "1"; + return ( + environment.MIRA_DASHBOARD_DISABLE_SCHEDULER !== "1" && + dashboardExecutionRole(environment) !== "web" + ); } diff --git a/backend/src/services/backups.ts b/backend/src/services/backups.ts index 8f5f106a1..f3988fd12 100644 --- a/backend/src/services/backups.ts +++ b/backend/src/services/backups.ts @@ -9,12 +9,23 @@ import { } from "../lib/processes.ts"; import { refreshCacheProducer } from "./cacheRefresh.ts"; import { - createManualScheduledJobRun, - finishScheduledJobRun, + enqueueJobExecution, + getJobExecution, + getLatestScheduledJobExecution, + getPreviousScheduledJobExecution, + type JobExecution, +} from "./jobExecutionQueue.ts"; +import { + successfulJobExecutionOutput, + waitForJobExecution, +} from "./queuedJobExecution.ts"; +import { + enqueueScheduledJob, getScheduledJob, registerScheduledJobAction, removeScheduledJobsNotInAction, - type ScheduledJobRun, + type ScheduledJobActionContext, + ScheduledJobActionError, upsertScheduledJob, } from "./scheduledJobs.ts"; const MAX_OUTPUT_CHARS = 100_000; @@ -46,7 +57,6 @@ interface BackupJob { completed: Promise; process?: BunProcess; statusRefreshed?: boolean; - manualScheduledRun?: ScheduledJobRun; } const backupJobs = new Map(); @@ -139,7 +149,8 @@ function evictCompletedBackupJobs(type: BackupJob["type"]) { export async function clearNeedsAttentionBackupJob(type: BackupJob["type"]) { const job = getCurrentBackupJob(type); - if (!job) { + if (!job || job.status === "done") { + if (job) backupJobs.delete(job.id); throw Object.assign(new Error(`${type.toUpperCase()} backup job not found`), { statusCode: 404, }); @@ -739,28 +750,72 @@ async function startWalgBackupJob(signal?: AbortSignal) { ); } -async function startScheduledBackup(type: BackupJob["type"], signal?: AbortSignal) { +async function startScheduledBackup( + type: BackupJob["type"], + signal: AbortSignal | undefined, + context: ScheduledJobActionContext +) { if (signal?.aborted) { throw new Error("Backup aborted by scheduler"); } + const previousExecution = getPreviousScheduledJobExecution( + scheduledBackupJobId(type), + context.executionId + ); + const persistedJob = persistedBackupViewFromExecution(type, previousExecution); + if (persistedJob?.status === "needs_attention") { + throw Object.assign( + new ScheduledJobActionError(`${type.toUpperCase()} backup needs attention`, { + backup: persistedJob, + }), + { statusCode: 409 } + ); + } const currentJob = getCurrentBackupJob(type); - if (currentJob?.status === "running" || currentJob?.status === "needs_attention") { + if (currentJob?.status === "needs_attention") { throw Object.assign( - new Error( - currentJob.status === "needs_attention" - ? `${type.toUpperCase()} backup needs attention` - : `${type.toUpperCase()} backup is already running` - ), - { - statusCode: 409, - } + new ScheduledJobActionError(`${type.toUpperCase()} backup needs attention`, { + backup: mapBackupJob(currentJob), + }), + { statusCode: 409 } + ); + } + if (currentJob?.status === "running") { + throw Object.assign( + new Error(`${type.toUpperCase()} backup is already running`), + { statusCode: 409 } ); } - const job = - type === "kopia" - ? await startKopiaBackupJob(signal) - : await startWalgBackupJob(signal); - const completedJob = await job.completed; + let job: BackupJob; + try { + job = await startManualBackup(type, signal); + } catch (error) { + const attentionJob = getCurrentBackupJob(type); + if (attentionJob?.status === "needs_attention") { + const message = errorMessage( + error, + `${type.toUpperCase()} backup needs attention` + ); + throw Object.assign( + new ScheduledJobActionError(message, { + backup: mapBackupJob(attentionJob), + }), + { statusCode: 409 } + ); + } + throw error; + } + const publish = () => context.updateOutput({ backup: mapBackupJob(job) }); + publish(); + const progress = setInterval(publish, 1000); + progress.unref(); + let completedJob: BackupJob; + try { + completedJob = await job.completed; + } finally { + clearInterval(progress); + publish(); + } if (completedJob.code !== 0) { const details = completedJob.stderr || completedJob.stdout; throw new Error( @@ -776,66 +831,187 @@ function scheduledBackupJobId(type: BackupJob["type"]) { return type === "kopia" ? "backup.kopia" : "backup.walg"; } -function createBackupManualScheduledRun(type: BackupJob["type"]) { - const jobId = scheduledBackupJobId(type); - if (!getScheduledJob(jobId)) { - return; - } - return createManualScheduledJobRun(jobId); +export function getCurrentBackupJob(type: BackupJob["type"]): BackupJob | undefined { + return (type === "kopia" ? getCurrentKopiaJob : getCurrentWalgJob)(); } -function backupFailureMessage(job: BackupJob) { - if (job.status === "needs_attention") { - return `${job.type.toUpperCase()} backup needs attention`; - } - if (job.code === 0) { - return; +/** Worker primitive. HTTP callers must enqueue the registered backup action. */ +export async function startManualBackup(type: BackupJob["type"], signal?: AbortSignal) { + const existingJob = getCurrentBackupJob(type); + if (existingJob?.status === "running") return existingJob; + return type === "kopia" + ? await startKopiaBackupJob(signal) + : await startWalgBackupJob(signal); +} + +function backupViewFromExecution( + type: BackupJob["type"], + execution: JobExecution | undefined +) { + if (!execution) return; + const backup = execution.output.backup; + if (backup && typeof backup === "object" && !Array.isArray(backup)) { + const backupView = backup as NonNullable>; + if ( + backupView.status === "running" && + (execution.status === "failed" || execution.status === "cancelled") + ) { + return { + ...backupView, + endedAt: execution.finishedAt + ? Date.parse(execution.finishedAt) + : backupView.endedAt, + status: execution.status, + stderr: execution.message ?? backupView.stderr, + }; + } + return backupView; } - const details = job.stderr || job.stdout; - return `${job.type.toUpperCase()} backup failed with code ${job.code ?? 1}${ - details ? `: ${details}` : "" - }`; + return { + code: undefined, + endedAt: execution.finishedAt ? Date.parse(execution.finishedAt) : undefined, + id: execution.id, + startedAt: Date.parse(execution.startedAt ?? execution.queuedAt), + status: + execution.status === "failed" || execution.status === "cancelled" + ? execution.status + : execution.finishedAt + ? "done" + : "running", + stderr: execution.message ?? "", + stdout: "", + type, + } as const; } -function attachManualScheduledRun(job: BackupJob): void { - if (job.manualScheduledRun || job.status !== "running") { - return; +function persistedBackupViewFromExecution( + type: BackupJob["type"], + execution: JobExecution | undefined +) { + if (!execution || wasBackupAttentionClearedAfter(type, execution)) return; + return backupViewFromExecution(type, execution); +} + +export function getPersistedBackupJob(type: BackupJob["type"]) { + return persistedBackupViewFromExecution( + type, + getLatestScheduledJobExecution(scheduledBackupJobId(type)) + ); +} + +function wasBackupAttentionClearedAfter( + type: BackupJob["type"], + execution: JobExecution +): boolean { + return Boolean( + database + .prepare( + `SELECT 1 + FROM job_executions + WHERE action_key = 'backup.clear-attention' + AND status = 'success' + AND json_valid(payload_json) + AND json_extract(payload_json, '$.type') = ? + AND json_extract(payload_json, '$.backupExecutionId') = ? + ORDER BY queued_at DESC, id DESC + LIMIT 1` + ) + .get(type, execution.id) + ); +} + +export function queueManualBackup(type: BackupJob["type"]) { + if (getPersistedBackupJob(type)?.status === "needs_attention") { + throw Object.assign(new Error(`${type.toUpperCase()} backup needs attention`), { + statusCode: 409, + }); } - let run: ScheduledJobRun; + const scheduledRun = enqueueScheduledJob(scheduledBackupJobId(type), "manual"); + return backupViewFromExecution( + type, + scheduledRun.executionId ? getJobExecution(scheduledRun.executionId) : undefined + ); +} + +export async function clearPersistedBackupAttention(type: BackupJob["type"]) { + let execution: JobExecution; + database.run("BEGIN IMMEDIATE"); try { - const scheduledRun = createBackupManualScheduledRun(job.type); - if (!scheduledRun) { - return; + const backupExecutionId = getLatestScheduledJobExecution( + scheduledBackupJobId(type) + )?.id; + if (!backupExecutionId) { + throw Object.assign(new Error(`${type.toUpperCase()} backup job not found`), { + statusCode: 404, + }); } - run = scheduledRun; + execution = enqueueJobExecution({ + actionKey: "backup.clear-attention", + displayName: `Clear ${type.toUpperCase()} backup attention`, + payload: { backupExecutionId, type }, + resourceClass: "light", + timeoutMs: 5 * 60 * 1000, + }); + database.run("COMMIT"); } catch (error) { - console.warn("[Backups] Failed to record manual backup run:", error); - return; + try { + database.run("ROLLBACK"); + } catch { + // Preserve the queue error. + } + throw error; } - job.manualScheduledRun = run; - void finishManualScheduledRunWhenComplete(job, run); + const completed = await waitForJobExecution(execution.id, { + timeoutMs: 15 * 60 * 1000, + }); + const output = successfulJobExecutionOutput(completed); + return output.backup as ReturnType; } -async function finishManualScheduledRunWhenComplete( - job: BackupJob, - run: ScheduledJobRun -): Promise { - try { - const completedJob = await job.completed; - const isSuccess = completedJob.status === "done" && completedJob.code === 0; - finishScheduledJobRun( - run, - isSuccess ? "success" : "failed", - isSuccess ? undefined : backupFailureMessage(completedJob), - { backup: mapBackupJob(completedJob) } +async function clearBackupAttention(type: BackupJob["type"], backupExecutionId: string) { + const latestExecution = getLatestScheduledJobExecution(scheduledBackupJobId(type)); + if (!latestExecution) { + throw Object.assign(new Error(`${type.toUpperCase()} backup job not found`), { + statusCode: 404, + }); + } + if (latestExecution.id !== backupExecutionId) { + throw Object.assign( + new Error(`${type.toUpperCase()} backup attention changed before clearing`), + { statusCode: 409 } ); - } catch (error) { - console.warn("[Backups] Failed to finish manual backup run:", error); } -} -export function getCurrentBackupJob(type: BackupJob["type"]): BackupJob | undefined { - return (type === "kopia" ? getCurrentKopiaJob : getCurrentWalgJob)(); + const current = getCurrentBackupJob(type); + if (current) { + return mapBackupJob(await clearNeedsAttentionBackupJob(type)); + } + + const persisted = getPersistedBackupJob(type); + if (!persisted) { + throw Object.assign(new Error(`${type.toUpperCase()} backup job not found`), { + statusCode: 404, + }); + } + if (persisted.status !== "needs_attention") { + throw Object.assign( + new Error(`${type.toUpperCase()} backup does not need attention`), + { statusCode: 409 } + ); + } + + const cleared = { ...persisted }; + try { + await refreshCacheProducer(backupStatusCacheKey(type), undefined, { + force: true, + }); + } catch (error) { + const stderr = typeof cleared.stderr === "string" ? cleared.stderr : ""; + cleared.stderr = trimOutput( + `${stderr}\nStatus refresh failed: ${String(error)}`.trim() + ); + } + return cleared; } async function terminateContainerProcessSafely( @@ -854,37 +1030,6 @@ async function terminateContainerProcessSafely( } } -export async function startManualBackup(type: BackupJob["type"]) { - const existingJob = getCurrentBackupJob(type); - if (existingJob?.status === "running") { - return existingJob; - } - try { - const job = - type === "kopia" ? await startKopiaBackupJob() : await startWalgBackupJob(); - attachManualScheduledRun(job); - return job; - } catch (error) { - try { - const failedRun = createBackupManualScheduledRun(type); - if (failedRun) { - finishScheduledJobRun( - failedRun, - "failed", - errorMessage(error, `${type.toUpperCase()} backup failed to start`), - {} - ); - } - } catch (runError) { - console.warn( - "[Backups] Failed to record failed manual backup run:", - runError - ); - } - throw error; - } -} - const backupScheduledJobs = [ { id: "backup.walg", @@ -895,6 +1040,7 @@ const backupScheduledJobs = [ timeOfDay: "03:20", actionKey: "backup.run", actionPayload: { type: "walg" }, + resourceClass: "host-heavy", }, { id: "backup.kopia", @@ -905,13 +1051,14 @@ const backupScheduledJobs = [ timeOfDay: "03:50", actionKey: "backup.run", actionPayload: { type: "kopia" }, + resourceClass: "host-heavy", }, ] as const; export function registerBackupScheduledJobs(): void { registerScheduledJobAction( "backup.run", - (job, signal) => { + (job, signal, context) => { const type = getScheduledBackupType(job.actionPayload); if (type !== "kopia" && type !== "walg") { throw Object.assign( @@ -919,10 +1066,32 @@ export function registerBackupScheduledJobs(): void { { statusCode: 400 } ); } - return startScheduledBackup(type, signal); + return startScheduledBackup(type, signal, context); }, { timeoutMs: SCHEDULED_BACKUP_TIMEOUT_MS } ); + registerScheduledJobAction( + "backup.clear-attention", + async (job, _signal, context) => { + const type = getScheduledBackupType(job.actionPayload); + if (type !== "kopia" && type !== "walg") { + throw Object.assign(new Error("Invalid backup type"), { + statusCode: 400, + }); + } + const backupExecutionId = job.actionPayload.backupExecutionId; + if ( + typeof backupExecutionId !== "string" || + backupExecutionId.trim() === "" + ) { + throw Object.assign(new Error("Backup execution id is missing"), { + statusCode: 400, + }); + } + context.protectFromCancellation(); + return { backup: await clearBackupAttention(type, backupExecutionId) }; + } + ); database.run("BEGIN"); try { removeScheduledJobsNotInAction( diff --git a/backend/src/services/cacheRefresh.ts b/backend/src/services/cacheRefresh.ts index 92b7f0d82..b31012c87 100644 --- a/backend/src/services/cacheRefresh.ts +++ b/backend/src/services/cacheRefresh.ts @@ -11,6 +11,7 @@ import { import os from "node:os"; import { database } from "../database.ts"; +import type { JobResourceClass } from "../lib/jobResources.ts"; import { runProcess } from "../lib/processes.ts"; import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; import { @@ -26,6 +27,7 @@ import { getDatabaseOverview } from "./databaseOverview.ts"; import { evaluateOpenClawNotifications } from "./openclawNotifications.ts"; import { evaluateQuotaNotifications } from "./quotaNotifications.ts"; import { + enqueueScheduledJob, getScheduledJob, registerScheduledJobAction, removeScheduledJobsNotInAction, @@ -1884,6 +1886,31 @@ function redactOpenAiQuotaAccount(openai: Awaited>(); +class SerialOperationQueue { + private tail: Promise = Promise.resolve(); + + private async observe(result: Promise): Promise { + try { + await result; + } catch { + // A failed operation must not block later queue entries. + } + } + + run(operation: () => Promise): Promise { + const previous = this.tail; + const result = (async () => { + await previous; + return operation(); + })(); + this.tail = this.observe(result); + return result; + } +} + +const cacheRefreshQueue = new SerialOperationQueue(); +const localCacheSeedQueue = new SerialOperationQueue(); + function cacheRefreshScopeKey(key: string): string { if (key === "moltbook") { return "moltbook"; @@ -1911,6 +1938,24 @@ function isSupportedCacheProducerKey(key: string): boolean { ); } +export function cacheRefreshResourceClass(key: string): JobResourceClass { + if (!isSupportedCacheProducerKey(key)) { + throw Object.assign( + new Error(`No backend refresh producer configured for cache key: ${key}`), + { statusCode: 400 } + ); + } + if ( + key === "weather.spydeberg" || + key === "quotas.summary" || + key === "moltbook" || + MOLTBOOK_CACHE_KEYS.has(key) + ) { + return "network"; + } + return "host-heavy"; +} + async function refreshCacheWithFailureRecord( key: string, refresh: () => Promise<{ refreshed: string[] }>, @@ -2015,6 +2060,16 @@ function abortError(): Error { return error; } +function runBoundedCacheRefresh( + operation: () => Promise, + signal?: AbortSignal +): Promise { + return cacheRefreshQueue.run(async () => { + if (signal?.aborted) throw abortError(); + return await operation(); + }); +} + async function waitForRefreshWithSignal( refresh: Promise, signal: AbortSignal | undefined @@ -2025,22 +2080,20 @@ async function waitForRefreshWithSignal( if (signal.aborted) { throw abortError(); } - return await Promise.race([ - refresh, - new Promise((_resolve, reject) => { - const onAbort = () => reject(abortError()); - signal.addEventListener("abort", onAbort, { once: true }); - void (async () => { - try { - await refresh; - } catch { - // The refresh result is observed by the race winner. - } finally { - signal.removeEventListener("abort", onAbort); - } - })(); - }), - ]); + let isAborted = false; + const onAbort = () => { + isAborted = true; + }; + signal.addEventListener("abort", onAbort, { once: true }); + try { + // Keep the permit occupied until the producer really settles. Returning + // on abort would let a second heavy producer overlap the first one. + const result = await refresh; + if (isAborted) throw abortError(); + return result; + } finally { + signal.removeEventListener("abort", onAbort); + } } async function waitForExistingRefresh( @@ -2097,8 +2150,8 @@ export async function refreshCacheProducer( .map(([, refresh]) => refresh); const refresh = childRefreshes.length > 0 - ? refreshAfterChildRefreshes(childRefreshes, key) - : refreshCacheProducerUnlocked(key); + ? refreshAfterChildRefreshes(childRefreshes, key, signal) + : runBoundedCacheRefresh(() => refreshCacheProducerUnlocked(key), signal); inFlightCacheRefreshes.set(scopeKey, refresh); void (async () => { try { @@ -2116,10 +2169,11 @@ export async function refreshCacheProducer( async function refreshAfterChildRefreshes( childRefreshes: Array>, - key: string + key: string, + signal?: AbortSignal ): Promise<{ refreshed: string[] }> { await Promise.allSettled(childRefreshes); - return await refreshCacheProducerUnlocked(key); + return await runBoundedCacheRefresh(() => refreshCacheProducerUnlocked(key), signal); } const cacheRefreshScheduledJobs = [ @@ -2131,6 +2185,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 60 * 60, actionKey: "cache.refresh", actionPayload: { key: "weather.spydeberg" }, + resourceClass: "network", }, { id: "cache.quotas", @@ -2140,6 +2195,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 30 * 60, actionKey: "cache.refresh", actionPayload: { key: "quotas.summary" }, + resourceClass: "network", }, { id: "cache.system", @@ -2150,6 +2206,7 @@ const cacheRefreshScheduledJobs = [ timeOfDay: "02:50", actionKey: "cache.refresh", actionPayload: { key: "system.host" }, + resourceClass: "host-heavy", }, { id: "cache.git", @@ -2159,6 +2216,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 60 * 60, actionKey: "cache.refresh", actionPayload: { key: "git.workspace" }, + resourceClass: "host-heavy", }, { id: "cache.moltbook", @@ -2168,6 +2226,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 30 * 60, actionKey: "cache.refresh", actionPayload: { key: "moltbook" }, + resourceClass: "network", }, { id: "cache.backup.kopia", @@ -2177,6 +2236,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 60 * 60, actionKey: "cache.refresh", actionPayload: { key: "backup.kopia.status" }, + resourceClass: "host-heavy", }, { id: "cache.backup.walg", @@ -2186,6 +2246,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 60 * 60, actionKey: "cache.refresh", actionPayload: { key: "backup.walg.status" }, + resourceClass: "host-heavy", }, { id: "cache.docker.summary", @@ -2195,6 +2256,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 30 * 60, actionKey: "cache.refresh", actionPayload: { key: DOCKER_SUMMARY_KEY }, + resourceClass: "host-heavy", }, { id: "cache.database.summary", @@ -2204,6 +2266,7 @@ const cacheRefreshScheduledJobs = [ intervalSeconds: 60 * 60, actionKey: "cache.refresh", actionPayload: { key: DATABASE_SUMMARY_KEY }, + resourceClass: "host-heavy", }, ] as const; @@ -2241,15 +2304,21 @@ function isCacheEntryFresh(key: string): boolean { const localCacheSeedPromises = new Map>(); +type CacheSeedStrategy = "local" | "none" | "queue"; + +interface CacheRefreshScheduledJobOptions { + seedStrategy?: CacheSeedStrategy; +} + export function waitForLocalCacheSeed(key: string): Promise { return localCacheSeedPromises.get(key) ?? Promise.resolve(); } export function seedMissingLocalCacheEntry(key: string): void { - if (isCacheEntryFresh(key)) { + if (isCacheEntryFresh(key) || localCacheSeedPromises.has(key)) { return; } - const seedPromise = (async () => { + const seedPromise = localCacheSeedQueue.run(async () => { try { await refreshCacheProducer(key); } catch (error) { @@ -2259,7 +2328,7 @@ export function seedMissingLocalCacheEntry(key: string): void { ); throw error; } - })(); + }); localCacheSeedPromises.set(key, seedPromise); void (async () => { try { @@ -2274,13 +2343,43 @@ export function seedMissingLocalCacheEntry(key: string): void { })(); } -export function registerCacheRefreshScheduledJobs(): void { +function queueMissingCacheSeeds(seedJobs: Array<{ id: string; key: string }>): void { + const startedAt = Date.now(); + for (const [index, seedJob] of seedJobs.entries()) { + if (isCacheEntryFresh(seedJob.key)) continue; + const scheduledJob = getScheduledJob(seedJob.id); + if (scheduledJob?.nextRunAt && Date.parse(scheduledJob.nextRunAt) <= startedAt) { + continue; + } + const staggerMs = index * 5000 + Math.floor(Math.random() * 2500); + try { + enqueueScheduledJob(seedJob.id, "startup", { + availableAt: new Date(startedAt + staggerMs).toISOString(), + }); + } catch (error) { + const statusCode = + error instanceof Error && "statusCode" in error + ? (error as { statusCode?: unknown }).statusCode + : undefined; + if (statusCode !== 409) { + console.warn( + `[CacheRefresh] Failed to queue startup cache seed ${seedJob.key}:`, + error + ); + } + } + } +} + +export function registerCacheRefreshScheduledJobs( + options: CacheRefreshScheduledJobOptions = {} +): void { registerScheduledJobAction("cache.refresh", async (job, signal) => { const key = getScheduledCacheKey(job); const result = await refreshCacheProducer(key, signal); return { key, ...result }; }); - const seedKeys: string[] = []; + const seedJobs: Array<{ id: string; key: string }> = []; database.run("BEGIN"); try { removeScheduledJobsNotInAction( @@ -2307,7 +2406,7 @@ export function registerCacheRefreshScheduledJobs(): void { : undefined), }); if (existing?.enabled ?? true) { - seedKeys.push(job.actionPayload.key); + seedJobs.push({ id: job.id, key: job.actionPayload.key }); } } database.run("COMMIT"); @@ -2319,7 +2418,11 @@ export function registerCacheRefreshScheduledJobs(): void { } throw error; } - for (const key of seedKeys) { - seedMissingLocalCacheEntry(key); + if (options.seedStrategy === "queue") { + queueMissingCacheSeeds(seedJobs); + } else if (options.seedStrategy === "local") { + for (const seedJob of seedJobs) { + seedMissingLocalCacheEntry(seedJob.key); + } } } diff --git a/backend/src/services/dockerActions.ts b/backend/src/services/dockerActions.ts new file mode 100644 index 000000000..17441491d --- /dev/null +++ b/backend/src/services/dockerActions.ts @@ -0,0 +1,346 @@ +import { errorMessage } from "../lib/errors.ts"; +import { + type BunProcess, + killProcessGroup, + pipeProcessOutput, + runProcess, + spawnProcess, +} from "../lib/processes.ts"; +import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; +import { + registerScheduledJobAction, + type ScheduledJobActionContext, + ScheduledJobActionError, +} from "./scheduledJobs.ts"; + +const MAX_OUTPUT_CHARS = 100_000; +const MAX_EXEC_COMMAND_CHARS = 16_384; +const DOCKER_REQUEST_TIMEOUT_MS = 30_000; +const DOCKER_EXEC_TIMEOUT_MS = 7 * 60 * 60 * 1000; +const STREAM_UPDATE_INTERVAL_MS = 250; + +function dockerBin(): string { + return nonEmptyEnvironmentFallback("MIRA_DOCKER_BIN", "docker"); +} + +function dockerRoot(): string { + return nonEmptyEnvironmentFallback("MIRA_DOCKER_ROOT", "/opt/docker"); +} + +function dockerComposeWrapper(): string { + return nonEmptyEnvironmentFallback( + "MIRA_DOCKER_COMPOSE_WRAPPER", + `${dockerRoot()}/bin/docker-compose-doppler` + ); +} + +function trimOutput(text: string): string { + return text.length <= MAX_OUTPUT_CHARS ? text : text.slice(-MAX_OUTPUT_CHARS); +} + +function requiredIdentifier(value: unknown, label: string): string { + if (typeof value !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(value)) { + throw Object.assign(new Error(`Invalid ${label}`), { statusCode: 400 }); + } + return value; +} + +function requiredImageIdentifier(value: unknown): string { + if (typeof value === "string" && /^sha256:[a-f0-9]{64}$/iu.test(value)) { + return value; + } + return requiredIdentifier(value, "imageId"); +} + +export async function runDockerCommand( + arguments_: string[], + signal?: AbortSignal +): Promise { + const { code, stderr, stdout } = await runProcess(dockerBin(), arguments_, { + cwd: dockerRoot(), + env: process.env, + maxBuffer: 10 * 1024 * 1024, + signal, + timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, + }); + if (code !== 0) { + throw new Error( + `docker ${arguments_.join(" ")} failed with exit code ${code}: ${ + stderr.trim() || stdout.trim() + }` + ); + } + return String(stdout); +} + +export async function runDockerComposeCommand( + arguments_: string[], + signal?: AbortSignal +): Promise<{ stderr: string; stdout: string }> { + const result = await runProcess(dockerComposeWrapper(), arguments_, { + cwd: dockerRoot(), + env: process.env, + maxBuffer: 20 * 1024 * 1024, + signal, + timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, + }); + if (result.code !== 0) { + throw new Error( + `docker compose ${arguments_.join(" ")} failed with exit code ${ + result.code + }: ${result.stderr.trim() || result.stdout.trim()}` + ); + } + return { stderr: result.stderr, stdout: result.stdout }; +} + +async function terminateContainerProcess( + containerId: string, + containerPid: number | undefined +): Promise { + if (!containerPid) return; + try { + await runDockerCommand([ + "exec", + containerId, + "kill", + "-TERM", + `-${containerPid}`, + ]); + } catch (groupError) { + try { + await runDockerCommand([ + "exec", + containerId, + "kill", + "-TERM", + String(containerPid), + ]); + } catch (processError) { + console.warn( + "[Docker] In-container cancellation failed:", + errorMessage(groupError, "process group kill failed"), + errorMessage(processError, "process kill failed") + ); + } + } +} + +async function executeDockerCommand( + payload: Record, + signal: AbortSignal | undefined, + context: ScheduledJobActionContext +): Promise> { + const containerId = requiredIdentifier(payload.containerId, "containerId"); + const command = payload.command; + if ( + typeof command !== "string" || + command.trim() === "" || + command.length > MAX_EXEC_COMMAND_CHARS || + command.includes("\0") + ) { + throw Object.assign(new Error("Invalid Docker exec command"), { + statusCode: 400, + }); + } + + const startedAt = Date.now(); + const pidMarker = `__MIRA_DOCKER_EXEC_PID_${context.executionId}:`; + let containerPid: number | undefined; + let stdout = ""; + let stderr = ""; + let stdoutPrefix = ""; + let lastPublishedAt = 0; + const publish = (isForced = false) => { + const timestamp = Date.now(); + if (!isForced && timestamp - lastPublishedAt < STREAM_UPDATE_INTERVAL_MS) return; + lastPublishedAt = timestamp; + context.updateOutput({ + containerId, + containerPid, + endedAt: undefined, + startedAt, + status: "running", + stderr, + stdout, + }); + }; + publish(true); + + let child: BunProcess; + try { + child = spawnProcess( + dockerBin(), + [ + "exec", + "-e", + `MIRA_DASHBOARD_EXEC_COMMAND=${command}`, + containerId, + "sh", + "-lc", + String.raw`if command -v setsid >/dev/null 2>&1; then exec setsid sh -lc 'printf '\''${pidMarker}%s\n'\'' "$$"; exec sh -lc "$MIRA_DASHBOARD_EXEC_COMMAND"'; fi; printf '${pidMarker}%s\n' "$$"; exec sh -lc "$MIRA_DASHBOARD_EXEC_COMMAND"`, + ], + { + cwd: dockerRoot(), + env: process.env, + signal, + } + ); + } catch (error) { + throw new ScheduledJobActionError("Docker exec failed", { + code: 1, + containerId, + endedAt: Date.now(), + startedAt, + status: "done", + stderr: errorMessage(error, "Docker exec failed"), + stdout, + }); + } + const abort = () => { + void terminateContainerProcess(containerId, containerPid); + try { + killProcessGroup(child, "SIGTERM"); + } catch { + // The Docker CLI may already have exited. + } + }; + signal?.addEventListener("abort", abort, { once: true }); + if (signal?.aborted) abort(); + + const stdoutDone = pipeProcessOutput( + child.stdout as ReadableStream | undefined, + (data) => { + const output = stdoutPrefix + String(data); + const newlineIndex = output.indexOf("\n"); + if (newlineIndex === -1 && containerPid === undefined) { + stdoutPrefix = output; + return; + } + let userOutput = output; + if (containerPid === undefined) { + const firstLine = output.slice(0, newlineIndex).trim(); + if (firstLine.startsWith(pidMarker)) { + const pid = Number(firstLine.slice(pidMarker.length)); + if (Number.isSafeInteger(pid) && pid > 0) containerPid = pid; + userOutput = output.slice(newlineIndex + 1); + } + stdoutPrefix = ""; + } + stdout = trimOutput(stdout + userOutput); + publish(); + } + ); + const stderrDone = pipeProcessOutput( + child.stderr as ReadableStream | undefined, + (data) => { + stderr = trimOutput(stderr + String(data)); + publish(); + } + ); + + let code: number; + try { + code = await child.exited; + await Promise.all([stdoutDone, stderrDone]); + } catch (error) { + stderr = trimOutput( + `${stderr}\n${errorMessage(error, "Docker exec failed")}`.trim() + ); + throw new ScheduledJobActionError("Docker exec failed", { + code: 1, + containerId, + containerPid, + endedAt: Date.now(), + startedAt, + status: "done", + stderr, + stdout, + }); + } finally { + signal?.removeEventListener("abort", abort); + } + const output = { + code, + containerId, + containerPid, + endedAt: Date.now(), + startedAt, + status: "done", + stderr, + stdout, + }; + if (code !== 0) { + throw new ScheduledJobActionError("Docker exec exited non-zero", output); + } + return output; +} + +export function registerDockerExecutionActions(): void { + registerScheduledJobAction("docker.stack.action", async (job, signal, context) => { + const action = job.actionPayload.action; + if (action !== "restart" && action !== "start" && action !== "stop") { + throw Object.assign(new Error("Invalid stack action"), { statusCode: 400 }); + } + const service = job.actionPayload.service; + const arguments_: string[] = [action]; + if (service !== undefined) { + arguments_.push(requiredIdentifier(service, "service name")); + } + context.protectFromCancellation(); + const result = await runDockerComposeCommand(arguments_, signal); + return { + ...result, + output: [result.stdout, result.stderr].filter(Boolean).join("\n").trim(), + }; + }); + registerScheduledJobAction( + "docker.container.action", + async (job, signal, context) => { + const action = job.actionPayload.action; + if (action !== "restart" && action !== "start" && action !== "stop") { + throw Object.assign(new Error("Invalid container action"), { + statusCode: 400, + }); + } + const containerId = requiredIdentifier( + job.actionPayload.containerId, + "containerId" + ); + context.protectFromCancellation(); + return { output: await runDockerCommand([action, containerId], signal) }; + } + ); + registerScheduledJobAction("docker.image.delete", async (job, signal, context) => { + const imageId = requiredImageIdentifier(job.actionPayload.imageId); + context.protectFromCancellation(); + return { + output: await runDockerCommand(["image", "rm", imageId], signal), + }; + }); + registerScheduledJobAction("docker.prune.images", async (_job, signal, context) => { + context.protectFromCancellation(); + return { + output: await runDockerCommand(["image", "prune", "-a", "-f"], signal), + }; + }); + registerScheduledJobAction("docker.prune.volumes", async (_job, signal, context) => { + context.protectFromCancellation(); + return { + output: await runDockerCommand(["volume", "prune", "-f"], signal), + }; + }); + registerScheduledJobAction("docker.volume.delete", async (job, signal, context) => { + const volumeName = requiredIdentifier(job.actionPayload.volumeName, "volumeName"); + context.protectFromCancellation(); + return { + output: await runDockerCommand(["volume", "rm", volumeName], signal), + }; + }); + registerScheduledJobAction( + "docker.exec", + (job, signal, context) => + executeDockerCommand(job.actionPayload, signal, context), + { timeoutMs: DOCKER_EXEC_TIMEOUT_MS } + ); +} diff --git a/backend/src/services/dockerUpdater.ts b/backend/src/services/dockerUpdater.ts index 6b02795ba..6b0f0033d 100644 --- a/backend/src/services/dockerUpdater.ts +++ b/backend/src/services/dockerUpdater.ts @@ -12,6 +12,7 @@ import { getScheduledJob, registerScheduledJobAction, removeScheduledJobsNotInAction, + ScheduledJobActionError, upsertScheduledJob, } from "./scheduledJobs.ts"; @@ -651,6 +652,7 @@ interface DiscoveredComposeService { interface RegistryFetchOptions { accept?: string; authorization?: string; + signal?: AbortSignal; } interface RegistryCredentials { @@ -810,7 +812,13 @@ async function fetchRegistryResponse( ): Promise<{ authorization?: string; response: Response; clearTimer: () => void }> { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 30_000); - const clearTimer = () => clearTimeout(timeout); + const abortFromSignal = () => controller.abort(options.signal?.reason); + options.signal?.addEventListener("abort", abortFromSignal, { once: true }); + if (options.signal?.aborted) abortFromSignal(); + const clearTimer = () => { + clearTimeout(timeout); + options.signal?.removeEventListener("abort", abortFromSignal); + }; const headers = { Accept: options.accept || "application/json", ...(options.authorization && { Authorization: options.authorization }), @@ -933,6 +941,7 @@ async function fetchRegistryJsonWithHeaders( clearTimer(); } } catch (error) { + options.signal?.throwIfAborted(); if (error instanceof Error && error.name === "AbortError") { throw new Error(`Request timeout for ${url}`, { cause: error }); } @@ -1141,7 +1150,7 @@ function manifestDigestForPlatform( return typeof digest === "string" ? digest : undefined; } -async function lookupRegistryV2(service: ManagedServiceRow) { +async function lookupRegistryV2(service: ManagedServiceRow, signal?: AbortSignal) { const registry = imageRegistry(service.image_repo); const registryHost = registry === "docker.io" ? "registry-1.docker.io" : registry; const repo = @@ -1167,7 +1176,7 @@ async function lookupRegistryV2(service: ManagedServiceRow) { } const { authorization, body, headers } = await fetchRegistryJsonWithHeaders( tagsUrl, - { authorization: tagListAuthorization } + { authorization: tagListAuthorization, signal } ); tagListAuthorization = authorization; tags.push( @@ -1198,6 +1207,7 @@ async function lookupRegistryV2(service: ManagedServiceRow) { `https://${registryHost}/v2/${repo}/manifests/${tag}`, { accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json", + signal, } ); const manifestDigest = manifestDigestForPlatform(body, servicePlatform(service)); @@ -1210,7 +1220,8 @@ async function lookupRegistryV2(service: ManagedServiceRow) { }; } -async function lookupLatest(service: ManagedServiceRow) { +async function lookupLatest(service: ManagedServiceRow, signal?: AbortSignal) { + signal?.throwIfAborted(); if (process.env.MIRA_DOCKER_UPDATER_SKIP_REGISTRY === "1") { return { latestTag: service.current_tag, @@ -1225,7 +1236,7 @@ async function lookupLatest(service: ManagedServiceRow) { unsupported: true, }; } - return lookupRegistryV2(service); + return lookupRegistryV2(service, signal); } function hasUpdate(service: ManagedServiceRow): boolean { @@ -1572,7 +1583,8 @@ function composeUpdateLockKey(service: ManagedServiceRow): string { async function withComposeUpdateLock( service: ManagedServiceRow, - action: () => Promise + action: () => Promise, + signal?: AbortSignal ): Promise { const key = composeUpdateLockKey(service); const wasPrevious = composeUpdateLocks.get(key)?.promise ?? Promise.resolve(); @@ -1584,8 +1596,21 @@ async function withComposeUpdateLock( } const next = { promise: waitForCurrent() }; composeUpdateLocks.set(key, next); - await wasPrevious; try { + if (signal) { + signal.throwIfAborted(); + const aborted = Promise.withResolvers(); + const abort = () => aborted.reject(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + try { + await Promise.race([wasPrevious, aborted.promise]); + } finally { + signal.removeEventListener("abort", abort); + } + signal.throwIfAborted(); + } else { + await wasPrevious; + } return await action(); } finally { release(); @@ -1597,8 +1622,10 @@ async function withComposeUpdateLock( async function applyComposeUpdateUnlocked( service: ManagedServiceRow, - targetImageReference: string + targetImageReference: string, + signal?: AbortSignal ) { + signal?.throwIfAborted(); if (!service.compose_image_field) { throw new Error( `Service ${serviceLabel(service)} is missing compose image field` @@ -1608,12 +1635,16 @@ async function applyComposeUpdateUnlocked( const configuredComposePath = service.compose_path; const composePath = fs.realpathSync(configuredComposePath); const commandComposePaths = getComposeCommandPaths(configuredComposePath); - const dirtyBefore = await dirtyDockerUpdaterPaths([ - composePath, - ...commandComposePaths.map((commandComposePath) => - fs.realpathSync(commandComposePath) - ), - ]); + const dirtyBefore = await dirtyDockerUpdaterPaths( + [ + composePath, + ...commandComposePaths.map((commandComposePath) => + fs.realpathSync(commandComposePath) + ), + ], + signal + ); + signal?.throwIfAborted(); const raw = fs.readFileSync(composePath, "utf8"); const originalStats = fs.statSync(composePath); const document = YAML.parse(raw) as JsonRecord; @@ -1689,6 +1720,7 @@ async function applyComposeUpdateUnlocked( cwd: command.cwd, env: process.env, maxBuffer: 10 * 1024 * 1024, + signal, timeoutMs: 180_000, }); if (code !== 0) { @@ -1957,7 +1989,10 @@ function servicesFromCompose(composePath: string): } } -export async function registerDockerUpdaterServices(): Promise { +export async function registerDockerUpdaterServices( + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); let composeFiles: string[]; try { const appsRoot = getDockerAppsRoot(); @@ -2030,6 +2065,7 @@ export async function registerDockerUpdaterServices(): Promise { + signal?.throwIfAborted(); const timestamp = nowIso(); const services = normalizeManagedServiceRows( serviceId === undefined @@ -2155,7 +2194,8 @@ export async function pollDockerUpdaterRegistries( const failures: Array<{ service: string; error: string }> = []; for (const service of services) { try { - const latest = await lookupLatest(service); + signal?.throwIfAborted(); + const latest = await lookupLatest(service, signal); if ("unsupported" in latest && latest.unsupported) { skipped.push({ service: serviceLabel(service), @@ -2207,6 +2247,7 @@ export async function pollDockerUpdaterRegistries( } } } catch (error) { + signal?.throwIfAborted(); failures.push({ service: serviceLabel(service), error: caughtMessage(error), @@ -2248,83 +2289,88 @@ export async function pollDockerUpdaterRegistries( async function applyServiceUpdate( service: ManagedServiceRow, - eventPrefix: "auto" | "manual" + eventPrefix: "auto" | "manual", + signal?: AbortSignal ): Promise { - return withComposeUpdateLock(service, async () => { - const lockedService = normalizeManagedServiceRow( - database - .prepare("SELECT * FROM docker_managed_services WHERE id = ? LIMIT 1") - .get(service.id) as ManagedServiceRow | undefined - ); - if (!lockedService || lockedService.enabled !== 1) { - const code = lockedService ? "DISABLED" : "NOT_FOUND"; - return { - step: `${eventPrefix}-update:${serviceLabel(service)}`, - isOk: false, - code, - stdout: "", - stderr: "Docker updater service not found or disabled", - }; - } - if (!hasUpdate(lockedService)) { - return { - step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, - isOk: false, - code: "CONFLICT", - stdout: "", - stderr: "No update available", - }; - } - const target = buildTargetImageReference(lockedService); - let result: Awaited>; - try { - // Compose writes tag-only refs for non-digest pins, then pulls so - // digest drift still refreshes mutable tags without storing @digest. - result = await applyComposeUpdateUnlocked(lockedService, target); - } catch (error) { - const message = caughtMessage(error); - database - .prepare( - `UPDATE docker_managed_services + return withComposeUpdateLock( + service, + async () => { + signal?.throwIfAborted(); + const lockedService = normalizeManagedServiceRow( + database + .prepare("SELECT * FROM docker_managed_services WHERE id = ? LIMIT 1") + .get(service.id) as ManagedServiceRow | undefined + ); + if (!lockedService || lockedService.enabled !== 1) { + const code = lockedService ? "DISABLED" : "NOT_FOUND"; + return { + step: `${eventPrefix}-update:${serviceLabel(service)}`, + isOk: false, + code, + stdout: "", + stderr: "Docker updater service not found or disabled", + }; + } + if (!hasUpdate(lockedService)) { + return { + step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, + isOk: false, + code: "CONFLICT", + stdout: "", + stderr: "No update available", + }; + } + const target = buildTargetImageReference(lockedService); + let result: Awaited>; + try { + // Compose writes tag-only refs for non-digest pins, then pulls so + // digest drift still refreshes mutable tags without storing @digest. + result = await applyComposeUpdateUnlocked(lockedService, target, signal); + } catch (error) { + signal?.throwIfAborted(); + const message = caughtMessage(error); + database + .prepare( + `UPDATE docker_managed_services SET last_checked_at = ?, last_status = ? WHERE id = ?` - ) - .run(nowIso(), `${eventPrefix}_update_failed`, lockedService.id); - insertEventBestEffort( - lockedService, - `${eventPrefix}_update_failed`, - message, - { - targetComposeImageRef: target, - } - ); - const [os = "linux", architecture] = servicePlatform(lockedService).split( - "/", - 2 - ); - createNotificationBestEffort( - `Docker ${eventPrefix} update failed`, - `${serviceLabel(lockedService)}: ${message}`, - `docker:updater:${eventPrefix}-failed:${lockedService.id}:${nowIso().slice(0, 10)}`, - "error", - { - architecture, - digest: lockedService.latest_digest, - os, - } - ); - return { - step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, - isOk: false, - stdout: "", - stderr: message, - }; - } + ) + .run(nowIso(), `${eventPrefix}_update_failed`, lockedService.id); + insertEventBestEffort( + lockedService, + `${eventPrefix}_update_failed`, + message, + { + targetComposeImageRef: target, + } + ); + const [os = "linux", architecture] = servicePlatform(lockedService).split( + "/", + 2 + ); + createNotificationBestEffort( + `Docker ${eventPrefix} update failed`, + `${serviceLabel(lockedService)}: ${message}`, + `docker:updater:${eventPrefix}-failed:${lockedService.id}:${nowIso().slice(0, 10)}`, + "error", + { + architecture, + digest: lockedService.latest_digest, + os, + } + ); + return { + step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, + isOk: false, + stdout: "", + stderr: message, + }; + } - try { - database - .prepare( - `UPDATE docker_managed_services + try { + database + .prepare( + `UPDATE docker_managed_services SET compose_image_ref = ?, current_tag = ?, current_digest = ?, tag_match_pattern = CASE WHEN tag_match_type = 'exact' THEN ? @@ -2332,75 +2378,78 @@ async function applyServiceUpdate( END, last_updated_at = ?, last_checked_at = ?, last_status = 'updated' WHERE id = ?` - ) - .run( - target, - sqlNullable(lockedService.latest_tag), - sqlNullable(lockedService.latest_digest), - sqlNullable(lockedService.latest_tag), - nowIso(), - nowIso(), - lockedService.id + ) + .run( + target, + sqlNullable(lockedService.latest_tag), + sqlNullable(lockedService.latest_digest), + sqlNullable(lockedService.latest_tag), + nowIso(), + nowIso(), + lockedService.id + ); + insertEventBestEffort( + lockedService, + `${eventPrefix}_update_succeeded`, + "Docker service updated", + { targetComposeImageRef: target } ); - insertEventBestEffort( - lockedService, - `${eventPrefix}_update_succeeded`, - "Docker service updated", - { targetComposeImageRef: target } - ); - createNotificationBestEffort( - "Docker service updated", - `${serviceLabel(lockedService)} updated to ${target}`, - `docker:updater:updated:${lockedService.id}:${target}` - ); - return { - step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, - changedPaths: result.changedPaths, - isOk: true, - stdout: result.stdout, - stderr: result.stderr, - }; - } catch (error) { - const message = caughtMessage(error); - insertEventBestEffort( - lockedService, - `${eventPrefix}_update_reconcile_failed`, - `Docker service updated but failed to persist updater state: ${message}`, - { - targetComposeImageRef: target, - } - ); - const [os = "linux", architecture] = servicePlatform(lockedService).split( - "/", - 2 - ); - createNotificationBestEffort( - `Docker ${eventPrefix} update needs reconciliation`, - `${serviceLabel(lockedService)} updated to ${target}, but state persistence failed: ${message}`, - `docker:updater:${eventPrefix}-reconcile-failed:${lockedService.id}:${nowIso().slice(0, 10)}`, - "error", - { - architecture, - digest: lockedService.latest_digest, - os, - } - ); - return { - step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, - changedPaths: result.changedPaths, - isOk: false, - stdout: result.stdout, - stderr: `Docker service updated but failed to persist updater state: ${message}`, - }; - } - }); + createNotificationBestEffort( + "Docker service updated", + `${serviceLabel(lockedService)} updated to ${target}`, + `docker:updater:updated:${lockedService.id}:${target}` + ); + return { + step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, + changedPaths: result.changedPaths, + isOk: true, + stdout: result.stdout, + stderr: result.stderr, + }; + } catch (error) { + const message = caughtMessage(error); + insertEventBestEffort( + lockedService, + `${eventPrefix}_update_reconcile_failed`, + `Docker service updated but failed to persist updater state: ${message}`, + { + targetComposeImageRef: target, + } + ); + const [os = "linux", architecture] = servicePlatform(lockedService).split( + "/", + 2 + ); + createNotificationBestEffort( + `Docker ${eventPrefix} update needs reconciliation`, + `${serviceLabel(lockedService)} updated to ${target}, but state persistence failed: ${message}`, + `docker:updater:${eventPrefix}-reconcile-failed:${lockedService.id}:${nowIso().slice(0, 10)}`, + "error", + { + architecture, + digest: lockedService.latest_digest, + os, + } + ); + return { + step: `${eventPrefix}-update:${serviceLabel(lockedService)}`, + changedPaths: result.changedPaths, + isOk: false, + stdout: result.stdout, + stderr: `Docker service updated but failed to persist updater state: ${message}`, + }; + } + }, + signal + ); } -async function pruneDanglingImagesBestEffort(): Promise { +async function pruneDanglingImagesBestEffort(signal?: AbortSignal): Promise { try { const result = await runProcess(getDockerBin(), ["image", "prune", "-f"], { env: process.env, maxBuffer: 10 * 1024 * 1024, + signal, timeoutMs: 120_000, }); if (result.code !== 0) { @@ -2409,6 +2458,7 @@ async function pruneDanglingImagesBestEffort(): Promise { ); } } catch (error) { + signal?.throwIfAborted(); console.error("[DockerUpdater] Failed to prune dangling images", { error: caughtMessage(error), }); @@ -2416,12 +2466,18 @@ async function pruneDanglingImagesBestEffort(): Promise { } async function syncDockerUpdaterChangesBestEffort( - steps: DockerUpdaterStepResult[] + steps: DockerUpdaterStepResult[], + signal?: AbortSignal, + protectFromCancellation?: () => void ): Promise { const updateSteps = steps.filter((step) => step.step.includes("-update:")); if (updateSteps.length === 0) { try { - const pendingResult = await syncDockerUpdaterChanges([]); + const pendingResult = await syncDockerUpdaterChanges( + [], + signal, + protectFromCancellation + ); if (pendingResult.pushed) { steps.push({ step: "git-sync:docker", @@ -2431,6 +2487,7 @@ async function syncDockerUpdaterChangesBestEffort( }); } } catch (error) { + signal?.throwIfAborted(); steps.push({ step: "git-sync:docker", isOk: false, @@ -2443,7 +2500,11 @@ async function syncDockerUpdaterChangesBestEffort( const changedPaths = updateSteps.flatMap((step) => step.changedPaths ?? []); if (changedPaths.length === 0) { try { - const pendingResult = await syncDockerUpdaterChanges([]); + const pendingResult = await syncDockerUpdaterChanges( + [], + signal, + protectFromCancellation + ); steps.push({ step: "git-sync:docker", isOk: true, @@ -2459,6 +2520,7 @@ async function syncDockerUpdaterChangesBestEffort( stderr: "", }); } catch (error) { + signal?.throwIfAborted(); steps.push({ step: "git-sync:docker", isOk: false, @@ -2469,7 +2531,11 @@ async function syncDockerUpdaterChangesBestEffort( return; } try { - const result = await syncDockerUpdaterChanges(changedPaths); + const result = await syncDockerUpdaterChanges( + changedPaths, + signal, + protectFromCancellation + ); steps.push({ step: "git-sync:docker", isOk: true, @@ -2477,6 +2543,7 @@ async function syncDockerUpdaterChangesBestEffort( stderr: "", }); } catch (error) { + signal?.throwIfAborted(); steps.push({ step: "git-sync:docker", isOk: false, @@ -2487,8 +2554,17 @@ async function syncDockerUpdaterChangesBestEffort( } export async function runDockerUpdaterService( - serviceId?: number + serviceId?: number, + signal?: AbortSignal, + protectFromCancellation?: () => void ): Promise { + signal?.throwIfAborted(); + let isMutationProtected = false; + const protectMutation = () => { + if (isMutationProtected) return; + protectFromCancellation?.(); + isMutationProtected = true; + }; const requestedService = serviceId === undefined ? undefined @@ -2499,7 +2575,7 @@ export async function runDockerUpdaterService( ) .get(serviceId) as ManagedServiceRow | undefined ); - const register = await registerDockerUpdaterServices(); + const register = await registerDockerUpdaterServices(signal); if (serviceId === undefined && shouldBlockGlobalUpdateForDiscoveryFailure(register)) { return [register]; } @@ -2565,7 +2641,7 @@ export async function runDockerUpdaterService( }, ]; } - const poll = await pollDockerUpdaterRegistries(service.id); + const poll = await pollDockerUpdaterRegistries(service.id, signal); if (!poll?.isOk) { return [register, poll].filter( (step): step is DockerUpdaterStepResult => step !== undefined @@ -2628,16 +2704,17 @@ export async function runDockerUpdaterService( }, ]; } - const apply = await applyServiceUpdate(refreshedService, "manual"); + protectMutation(); + const apply = await applyServiceUpdate(refreshedService, "manual", signal); if (apply.isOk) { - await pruneDanglingImagesBestEffort(); + await pruneDanglingImagesBestEffort(signal); } const steps = [register, poll, apply]; - await syncDockerUpdaterChangesBestEffort(steps); + await syncDockerUpdaterChangesBestEffort(steps, signal, protectMutation); return steps; } const blockedAppSlugs = failedDiscoveryAppSlugs(register); - const poll = await pollDockerUpdaterRegistries(); + const poll = await pollDockerUpdaterRegistries(undefined, signal); const autoServices = normalizeManagedServiceRows( database .prepare( @@ -2647,6 +2724,7 @@ export async function runDockerUpdaterService( ); const applyResults: DockerUpdaterStepResult[] = []; for (const service of autoServices) { + signal?.throwIfAborted(); if ( blockedAppSlugs.has(service.app_slug) || service.last_status !== "update_available" || @@ -2654,13 +2732,14 @@ export async function runDockerUpdaterService( ) { continue; } - applyResults.push(await applyServiceUpdate(service, "auto")); + protectMutation(); + applyResults.push(await applyServiceUpdate(service, "auto", signal)); } if (applyResults.some((step) => step.isOk)) { - await pruneDanglingImagesBestEffort(); + await pruneDanglingImagesBestEffort(signal); } const steps = [register, poll, ...applyResults]; - await syncDockerUpdaterChangesBestEffort(steps); + await syncDockerUpdaterChangesBestEffort(steps, signal, protectMutation); return steps; } @@ -2684,11 +2763,30 @@ export function registerDockerUpdaterScheduledJobs(): void { timeOfDay: "04:10", actionKey: "docker.updater", actionPayload: {}, + resourceClass: "exclusive", } as const; registerScheduledJobAction( "docker.updater", - async () => { - const steps = await runDockerUpdaterService(); + async (executionJob, signal, context) => { + const rawServiceId = executionJob.actionPayload.serviceId; + const serviceId = + rawServiceId === undefined + ? undefined + : typeof rawServiceId === "number" && + Number.isSafeInteger(rawServiceId) && + rawServiceId > 0 + ? rawServiceId + : NaN; + if (Number.isNaN(serviceId)) { + throw Object.assign(new Error("Invalid Docker updater service id"), { + statusCode: 400, + }); + } + const steps = await runDockerUpdaterService( + serviceId, + signal, + context.protectFromCancellation + ); const failed = steps.filter( (step) => !step.isOk && @@ -2696,11 +2794,12 @@ export function registerDockerUpdaterScheduledJobs(): void { step.step !== "git-sync:docker" ); if (failed.length > 0) { - throw new Error( - failed.map((step) => `${step.step}: ${step.stderr}`).join("\n") + throw new ScheduledJobActionError( + failed.map((step) => `${step.step}: ${step.stderr}`).join("\n"), + { serviceId, steps } ); } - return { steps }; + return { serviceId, steps }; }, { timeoutMs: 30 * 60 * 1000 } ); diff --git a/backend/src/services/execJobs.ts b/backend/src/services/execJobs.ts index 6572266ea..db315f547 100644 --- a/backend/src/services/execJobs.ts +++ b/backend/src/services/execJobs.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; +import { database } from "../database.ts"; import { errorMessage } from "../lib/errors.ts"; import { type BunProcess, @@ -8,6 +9,17 @@ import { pipeProcessOutput, spawnProcess, } from "../lib/processes.ts"; +import { + cancelJobExecution, + enqueueJobExecution, + getJobExecution, + type JobExecution, +} from "./jobExecutionQueue.ts"; +import { + successfulJobExecutionOutput, + waitForJobExecution, +} from "./queuedJobExecution.ts"; +import { registerScheduledJobAction, ScheduledJobActionError } from "./scheduledJobs.ts"; const OPS_SHELL_COMMANDS = new Set([ "__mira_dashboard_shell_smoke_test__", @@ -49,24 +61,14 @@ export interface ExecJobResponse { } class ExecValidationError extends Error { + readonly statusCode = 400; + constructor(message: string) { super(message); this.name = "ExecValidationError"; } } -interface ExecJob { - closePending?: boolean; - code: number | undefined; - endedAt: number | undefined; - id: string; - process?: BunProcess; - startedAt: number; - status: "running" | "signaled" | "done"; - stderr: string; - stdout: string; -} - const MAX_COMMAND_LENGTH = 4096; const SHELL_METACHARACTERS_RE = /[\n\r\0]/u; const EXECUTABLE_RE = /^(?:[\w./-]+)$/u; @@ -75,7 +77,8 @@ const MAX_JOBS = 100; const EXEC_ONCE_TIMEOUT_MS = 60_000; const ALLOWED_DIRECT_EXECUTABLES = new Set(["bash"]); const BASH_LOGIN_COMMAND_ARGUMENTS = 2; -const jobs = new Map(); +const TRACKED_EXEC_TIMEOUT_MS = 7 * 60 * 60 * 1000; +const STREAM_UPDATE_INTERVAL_MS = 250; function trimOutput(text: string): string { return text.length <= MAX_OUTPUT_CHARS ? text : text.slice(-MAX_OUTPUT_CHARS); @@ -165,7 +168,7 @@ function validateExecRequest(payload: unknown, mode: ExecRequestMode): ExecReque if (cwd !== undefined && typeof cwd !== "string") { throw new ExecValidationError("cwd must be a string"); } - return { args, command, cwd, shell }; + return { args, command, cwd: resolveCwd(cwd), shell }; } function resolveCwd(cwd: string | undefined): string { @@ -222,12 +225,17 @@ export function execErrorResponse(error: unknown): { error: string; status: numb function runExecCommand( request: ExecRequest, - jobId: string, - onUpdate?: (update: Pick) => void, - timeoutMs?: number + onUpdate?: (update: Pick) => void, + timeoutMs?: number, + signal?: AbortSignal ): Promise { const { args, command, cwd, shell } = request; - const cwdOption = { cwd: resolveCwd(cwd), detached: true, env: process.env }; + const cwdOption = { + cwd: resolveCwd(cwd), + detached: true, + env: process.env, + signal, + }; let childFactory: () => BunProcess; if (shell) { childFactory = () => @@ -244,27 +252,24 @@ function runExecCommand( return new Promise((resolve, reject) => { const child = childFactory(); - const job = jobs.get(jobId); - if (job) job.process = child; + let stdout = ""; + let stderr = ""; const recordKillError = (signal: NodeJS.Signals, error: unknown) => { const message = errorMessage(error, `Failed to send ${signal}`); console.error("[Exec] Process group kill failed:", message); - const current = jobs.get(jobId); - if (current) { - current.stderr = trimOutput(`${current.stderr}\n${message}`.trim()); - } + stderr = trimOutput(`${stderr}\n${message}`.trim()); + onUpdate?.({ stderr, stdout }); }; let timeout: Timer | undefined; let forceKillTimeout: Timer | undefined; let didTimeout = false; - if (timeoutMs !== undefined) { - timeout = setTimeout(() => { - didTimeout = true; - try { - killProcessGroup(child, "SIGTERM"); - } catch (error) { - recordKillError("SIGTERM", error); - } + const terminate = () => { + try { + killProcessGroup(child, "SIGTERM"); + } catch (error) { + recordKillError("SIGTERM", error); + } + if (!forceKillTimeout) { forceKillTimeout = setTimeout(() => { try { killProcessGroup(child, "SIGKILL"); @@ -273,12 +278,17 @@ function runExecCommand( } }, 3000); forceKillTimeout.unref(); + } + }; + const abortFromSignal = () => terminate(); + signal?.addEventListener("abort", abortFromSignal, { once: true }); + if (timeoutMs !== undefined) { + timeout = setTimeout(() => { + didTimeout = true; + terminate(); }, timeoutMs); timeout.unref(); } - - let stdout = ""; - let stderr = ""; const outputUpdate = () => onUpdate?.({ stderr, stdout }); const stdoutDone = pipeProcessOutput( child.stdout as ReadableStream | undefined, @@ -300,6 +310,7 @@ function runExecCommand( return code; })() .then((code) => { + signal?.removeEventListener("abort", abortFromSignal); if (timeout) clearTimeout(timeout); if (forceKillTimeout) clearTimeout(forceKillTimeout); resolve({ @@ -309,6 +320,7 @@ function runExecCommand( }); }) .catch((error: unknown) => { + signal?.removeEventListener("abort", abortFromSignal); if (timeout) clearTimeout(timeout); if (forceKillTimeout) clearTimeout(forceKillTimeout); reject(error); @@ -316,179 +328,198 @@ function runExecCommand( }); } -function cleanupJobs(): void { - if (jobs.size < MAX_JOBS) return; - const entries = jobs - .values() - .toArray() - .toSorted((a, b) => a.startedAt - b.startedAt); - let overflow = entries.length - (MAX_JOBS - 1); - for (const job of entries) { - if (overflow <= 0) break; - if ( - job.closePending || - ((job.status === "running" || job.status === "signaled") && job.process) - ) { - continue; - } - jobs.delete(job.id); - overflow -= 1; - } - if (overflow > 0) { - console.warn("[Exec] Job cleanup skipped active jobs while enforcing cap"); - } +function outputString(output: Record, key: string): string { + return typeof output[key] === "string" ? output[key] : ""; } -function updateExecJobOutput( - jobId: string, - update: Pick -): void { - const current = jobs.get(jobId); - if (!current) return; - current.stdout = update.stdout; - current.stderr = update.stderr; +function outputNumber(output: Record, key: string): number | undefined { + return typeof output[key] === "number" && Number.isFinite(output[key]) + ? output[key] + : undefined; } -function completeExecJob(jobId: string, result: ExecResponse): void { - const current = jobs.get(jobId); - if (!current) return; - current.status = "done"; - if (!current.closePending) current.code = result.code; - current.stdout = result.stdout; - current.stderr = result.stderr; - current.endedAt = Date.now(); - current.closePending = false; - current.process = undefined; - cleanupJobs(); +function execResponseFromExecution(execution: JobExecution): ExecResponse { + const output = execution.output; + if (typeof output.stdout !== "string" || typeof output.stderr !== "string") { + successfulJobExecutionOutput(execution); + } + return { + code: outputNumber(output, "code"), + stderr: outputString(output, "stderr").slice(-10_000), + stdout: outputString(output, "stdout").slice(-10_000), + }; } -function markExecJobForcedKilled(job: ExecJob): void { - job.closePending = true; - job.status = "done"; - job.code = 137; - job.endedAt = Date.now(); - cleanupJobs(); +async function executeCommandInWorker( + payload: Record, + mode: ExecRequestMode, + signal: AbortSignal | undefined, + updateOutput: (output: Record) => void +): Promise> { + const request = validateExecRequest(payload.request, mode); + const startedAt = Date.now(); + let lastPublishedAt = 0; + let latestOutput = { stderr: "", stdout: "" }; + const publish = ( + update: Pick, + isForced = false + ) => { + const timestamp = Date.now(); + if (!isForced && timestamp - lastPublishedAt < STREAM_UPDATE_INTERVAL_MS) return; + lastPublishedAt = timestamp; + updateOutput({ + endedAt: undefined, + startedAt, + status: "running", + stderr: update.stderr, + stdout: update.stdout, + }); + }; + publish(latestOutput, true); + let result: ExecResponse; + try { + result = await runExecCommand( + request, + (update) => { + latestOutput = update; + publish(update); + }, + mode === "once" ? EXEC_ONCE_TIMEOUT_MS : undefined, + signal + ); + } catch (error) { + const output = { + code: 1, + endedAt: Date.now(), + startedAt, + status: "done", + stderr: trimOutput( + `${latestOutput.stderr}\n${errorMessage(error, "Tracked command failed")}`.trim() + ), + stdout: latestOutput.stdout, + }; + throw new ScheduledJobActionError("Tracked command failed", output); + } + const output = { + code: result.code, + endedAt: Date.now(), + startedAt, + status: "done", + stderr: result.stderr, + stdout: result.stdout, + }; + publish(result, true); + if (result.code !== 0) { + throw new ScheduledJobActionError("Tracked command exited non-zero", output); + } + return output; } -function failExecJob(jobId: string, error: unknown): void { - const current = jobs.get(jobId); - if (!current) return; - current.status = "done"; - current.code = 1; - const message = error instanceof Error ? error.message : String(error); - current.stderr = trimOutput(`${current.stderr}\n${message}`.trim()); - current.endedAt = Date.now(); - current.closePending = false; - current.process = undefined; - cleanupJobs(); +export function registerExecExecutionActions(): void { + registerScheduledJobAction( + "exec.once", + (job, signal, context) => + executeCommandInWorker( + job.actionPayload, + "once", + signal, + context.updateOutput + ), + { timeoutMs: EXEC_ONCE_TIMEOUT_MS } + ); + registerScheduledJobAction( + "exec.tracked", + (job, signal, context) => + executeCommandInWorker( + job.actionPayload, + "start", + signal, + context.updateOutput + ), + { timeoutMs: TRACKED_EXEC_TIMEOUT_MS } + ); } export async function runExecOnce(payload: unknown): Promise { const request = validateExecRequest(payload, "once"); - const result = await runExecCommand( - request, - Bun.randomUUIDv7(), - undefined, - EXEC_ONCE_TIMEOUT_MS + const execution = enqueueJobExecution({ + actionKey: "exec.once", + displayName: "Tracked ops command", + payload: { request }, + resourceClass: "exclusive", + timeoutMs: EXEC_ONCE_TIMEOUT_MS, + }); + return execResponseFromExecution( + await waitForJobExecution(execution.id, { + timeoutMs: EXEC_ONCE_TIMEOUT_MS + 30 * 60 * 1000, + }) ); - return { - code: result.code, - stderr: result.stderr.slice(-10_000), - stdout: result.stdout.slice(-10_000), - }; +} + +function activeTrackedExecCount(): number { + const row = database + .prepare( + `SELECT COUNT(*) AS count + FROM job_executions + WHERE action_key = 'exec.tracked' + AND status IN ('queued', 'running')` + ) + .get() as { count: number }; + return row.count; } export function startExecJob(payload: unknown): ExecStartResponse { - const request = validateExecRequest(payload, "start"); - cleanupJobs(); - if (jobs.size >= MAX_JOBS) { + if (activeTrackedExecCount() >= MAX_JOBS) { throw Object.assign(new Error("Too many exec jobs"), { statusCode: 429 }); } - - const jobId = Bun.randomUUIDv7(); - jobs.set(jobId, { - closePending: false, - code: undefined, - endedAt: undefined, - id: jobId, - startedAt: Date.now(), - status: "running", - stderr: "", - stdout: "", + const request = validateExecRequest(payload, "start"); + const execution = enqueueJobExecution({ + actionKey: "exec.tracked", + displayName: "Tracked shell job", + payload: { request }, + resourceClass: "exclusive", + timeoutMs: TRACKED_EXEC_TIMEOUT_MS, }); + return { jobId: execution.id }; +} - let runPromise: Promise; - try { - runPromise = runExecCommand(request, jobId, (update) => - updateExecJobOutput(jobId, update) - ); - } catch (error) { - jobs.delete(jobId); - throw error; +function trackedExecExecution(jobId: string): JobExecution { + const execution = getJobExecution(jobId); + if (!execution || execution.actionKey !== "exec.tracked") { + throw Object.assign(new Error("Exec job not found"), { statusCode: 404 }); } - - void (async () => { - try { - completeExecJob(jobId, await runPromise); - } catch (error) { - failExecJob(jobId, error); - } - })(); - - return { jobId }; + return execution; } export function stopExecJob(jobId: string): { isSuccess: boolean; message: string } { - const job = jobs.get(jobId); - if (!job) { - throw Object.assign(new Error("Exec job not found"), { statusCode: 404 }); - } - if (job.status !== "running") { + const execution = trackedExecExecution(jobId); + if (execution.status !== "queued" && execution.status !== "running") { throw Object.assign(new Error("Job is not running"), { statusCode: 400 }); } - - if (!job.process) { - throw Object.assign(new Error("Process not available"), { statusCode: 400 }); - } - - try { - killProcessGroup(job.process, "SIGTERM"); - } catch (error) { - throw Object.assign( - new Error(errorMessage(error, "Failed to stop exec job process")), - { statusCode: 409 } - ); - } - job.status = "signaled"; - - const forceKillTimer = setTimeout(() => { - if (!job.process || job.status !== "signaled") return; - try { - killProcessGroup(job.process, "SIGKILL"); - markExecJobForcedKilled(job); - } catch (error) { - const message = errorMessage(error, "Failed to force kill exec job"); - console.error("[Exec] Force kill failed:", message); - job.stderr = trimOutput(`${job.stderr}\n${message}`.trim()); - } - }, 3000); - forceKillTimer.unref(); - + cancelJobExecution(jobId); return { isSuccess: true, message: "Stop signal sent" }; } export function getExecJob(jobId: string): ExecJobResponse { - const job = jobs.get(jobId); - if (!job) { - throw Object.assign(new Error("Exec job not found"), { statusCode: 404 }); - } + const execution = trackedExecExecution(jobId); + const output = execution.output; + const isTerminal = ["success", "failed", "cancelled"].includes(execution.status); return { - code: job.code, - endedAt: job.endedAt, - jobId: job.id, - startedAt: job.startedAt, - status: job.status, - stderr: job.stderr, - stdout: job.stdout, + code: outputNumber(output, "code"), + endedAt: isTerminal + ? (outputNumber(output, "endedAt") ?? + (execution.finishedAt ? Date.parse(execution.finishedAt) : undefined)) + : undefined, + jobId: execution.id, + startedAt: + outputNumber(output, "startedAt") ?? + Date.parse(execution.startedAt ?? execution.queuedAt), + status: isTerminal + ? "done" + : execution.cancelRequestedAt + ? "signaled" + : "running", + stderr: outputString(output, "stderr"), + stdout: outputString(output, "stdout"), }; } diff --git a/backend/src/services/gitHygiene.ts b/backend/src/services/gitHygiene.ts index 0bd501ed0..264b1c901 100644 --- a/backend/src/services/gitHygiene.ts +++ b/backend/src/services/gitHygiene.ts @@ -21,6 +21,7 @@ interface GitSyncResult { interface GitCommandOptions { cwd: string; + signal?: AbortSignal; timeoutMs?: number; } @@ -105,6 +106,7 @@ async function git(arguments_: string[], options: GitCommandOptions): Promise { +async function resolveDockerGitScope( + signal?: AbortSignal +): Promise<{ appsPath: string; repoPath: string }> { const appsRoot = realpathSync(getDockerAppsRoot()); - const repoPath = await git(["rev-parse", "--show-toplevel"], { cwd: appsRoot }); + const repoPath = await git(["rev-parse", "--show-toplevel"], { + cwd: appsRoot, + signal, + }); const appsPath = relativePath(repoPath, appsRoot); if (!appsPath) { throw new Error(`Docker apps root is outside git repository: ${appsRoot}`); @@ -153,15 +160,18 @@ async function resolveDockerGitScope(): Promise<{ appsPath: string; repoPath: st return { appsPath, repoPath }; } -async function dockerGitScope(): Promise<{ appsPath: string; repoPath: string }> { - return await resolveDockerGitScope(); +async function dockerGitScope( + signal?: AbortSignal +): Promise<{ appsPath: string; repoPath: string }> { + return await resolveDockerGitScope(signal); } export async function dirtyDockerUpdaterPaths( - paths: string[] + paths: string[], + signal?: AbortSignal ): Promise | undefined> { try { - const scope = await dockerGitScope(); + const scope = await dockerGitScope(signal); const statusPathspecs = normalizeDockerChangedPaths(scope.repoPath, paths) ?? []; if (statusPathspecs.length === 0) return new Set(); const status = await git( @@ -172,7 +182,7 @@ export async function dirtyDockerUpdaterPaths( "--", ...statusPathspecs.map((path_) => literalPathspec(path_)), ], - { cwd: scope.repoPath } + { cwd: scope.repoPath, signal } ); return new Set( parseStatusPaths(status).map((statusPath) => @@ -180,6 +190,7 @@ export async function dirtyDockerUpdaterPaths( ) ); } catch { + signal?.throwIfAborted(); return undefined; } } @@ -199,16 +210,18 @@ function normalizeDockerChangedPaths( async function commitAndPushPaths( repoPath: string, paths: string[], - message: string + message: string, + signal?: AbortSignal, + protectFromCancellation?: () => void ): Promise { const changedPaths = uniqueSorted(paths); if (changedPaths.length === 0) { return { changedPaths, pushed: false, skippedReason: "no safe changes" }; } - await assertPendingCommitsAreAutomation(repoPath, [message]); + await assertPendingCommitsAreAutomation(repoPath, [message], signal); const changedPathspecs = changedPaths.map((path_) => literalPathspec(path_)); - await git(["add", "--", ...changedPathspecs], { cwd: repoPath }); + await git(["add", "--", ...changedPathspecs], { cwd: repoPath, signal }); try { const stagedDiff = await runProcess( "git", @@ -216,6 +229,7 @@ async function commitAndPushPaths( { cwd: repoPath, env: process.env, + signal, timeoutMs: GIT_SYNC_TIMEOUT_MS, } ); @@ -230,20 +244,26 @@ async function commitAndPushPaths( ); } + protectFromCancellation?.(); await git(["commit", "--only", "-m", message, "--", ...changedPathspecs], { cwd: repoPath, + signal, }); } catch (error) { await git(["restore", "--staged", "--", ...changedPathspecs], { cwd: repoPath }); throw error; } - const upstream = await inspectUpstream(repoPath); + const upstream = await inspectUpstream(repoPath, signal); if (!upstream) { throw new Error("Refusing to push without an inspectable upstream"); } - const commit = await git(["rev-parse", "--short", "HEAD"], { cwd: repoPath }); + const commit = await git(["rev-parse", "--short", "HEAD"], { + cwd: repoPath, + signal, + }); await git(["push", upstream.remote, upstream.refspec], { cwd: repoPath, + signal, timeoutMs: GIT_PUSH_TIMEOUT_MS, }); return { changedPaths, commit, pushed: true }; @@ -255,13 +275,17 @@ interface InspectedUpstream { remote: string; } -async function inspectUpstream(repoPath: string): Promise { +async function inspectUpstream( + repoPath: string, + signal?: AbortSignal +): Promise { const upstream = await runProcess( "git", ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], { cwd: repoPath, env: process.env, + signal, timeoutMs: GIT_SYNC_TIMEOUT_MS, } ); @@ -277,21 +301,24 @@ async function inspectUpstream(repoPath: string): Promise { - const upstream = await inspectUpstream(repoPath); + const upstream = await inspectUpstream(repoPath, signal); if (!upstream) return undefined; const subjects = await git(["log", "--format=%s", `${upstream.name}..HEAD`], { cwd: repoPath, + signal, }); return { subjects: subjects.split("\n").filter(Boolean), upstream }; } async function assertPendingCommitsAreAutomation( repoPath: string, - allowedMessages: string[] + allowedMessages: string[], + signal?: AbortSignal ): Promise { - const pendingState = await pendingCommitState(repoPath); + const pendingState = await pendingCommitState(repoPath, signal); if (pendingState === undefined) { throw new Error("Refusing to push without an inspectable upstream"); } @@ -302,9 +329,11 @@ async function assertPendingCommitsAreAutomation( async function pushPendingAutomationCommits( repoPath: string, - allowedMessages: string[] + allowedMessages: string[], + signal?: AbortSignal, + protectFromCancellation?: () => void ): Promise { - const pendingState = await pendingCommitState(repoPath); + const pendingState = await pendingCommitState(repoPath, signal); if ( pendingState === undefined || pendingState.subjects.length === 0 || @@ -313,17 +342,23 @@ async function pushPendingAutomationCommits( return undefined; } + protectFromCancellation?.(); await git(["push", pendingState.upstream.remote, pendingState.upstream.refspec], { cwd: repoPath, + signal, timeoutMs: GIT_PUSH_TIMEOUT_MS, }); - const commit = await git(["rev-parse", "--short", "HEAD"], { cwd: repoPath }); + const commit = await git(["rev-parse", "--short", "HEAD"], { + cwd: repoPath, + signal, + }); return { changedPaths: [], commit, pushed: true }; } async function withGitSyncLock( repoPath: string, - action: () => Promise + action: () => Promise, + signal?: AbortSignal ): Promise { const wasPrevious = gitSyncLocks.get(repoPath)?.promise ?? Promise.resolve(); const current = Promise.withResolvers(); @@ -336,6 +371,7 @@ async function withGitSyncLock( gitSyncLocks.set(repoPath, next); await wasPrevious; try { + signal?.throwIfAborted(); return await action(); } finally { release(); @@ -345,64 +381,107 @@ async function withGitSyncLock( } } -export async function syncOpenClawWorkspaceSafePaths(): Promise { +export async function syncOpenClawWorkspaceSafePaths( + signal?: AbortSignal, + protectFromCancellation?: () => void +): Promise { const repoPath = getOpenClawRoot(); - return withGitSyncLock(repoPath, async () => { - const status = await git(["status", "--porcelain=v1", "-z", "-uall"], { - cwd: repoPath, - }); - const changedPaths = parseStatusPaths(status); - const safePaths = changedPaths.filter((path_) => isOpenClawSafePath(path_)); - if (safePaths.length === 0) { - const pushedPending = await pushPendingAutomationCommits(repoPath, [ + return withGitSyncLock( + repoPath, + async () => { + const status = await git(["status", "--porcelain=v1", "-z", "-uall"], { + cwd: repoPath, + signal, + }); + const changedPaths = parseStatusPaths(status); + const safePaths = changedPaths.filter((path_) => isOpenClawSafePath(path_)); + if (safePaths.length === 0) { + const pushedPending = await pushPendingAutomationCommits( + repoPath, + [OPENCLAW_SYNC_COMMIT_MESSAGE], + signal, + protectFromCancellation + ); + if (pushedPending) return pushedPending; + return { + changedPaths: [], + pushed: false, + skippedReason: "no safe changes", + }; + } + return commitAndPushPaths( + repoPath, + safePaths, OPENCLAW_SYNC_COMMIT_MESSAGE, - ]); - if (pushedPending) return pushedPending; - return { changedPaths: [], pushed: false, skippedReason: "no safe changes" }; - } - return commitAndPushPaths(repoPath, safePaths, OPENCLAW_SYNC_COMMIT_MESSAGE); - }); + signal, + protectFromCancellation + ); + }, + signal + ); } -export async function syncDockerUpdaterChanges(paths?: string[]): Promise { - const scope = await dockerGitScope(); +export async function syncDockerUpdaterChanges( + paths?: string[], + signal?: AbortSignal, + protectFromCancellation?: () => void +): Promise { + const scope = await dockerGitScope(signal); const { appsPath, repoPath } = scope; - return withGitSyncLock(repoPath, async () => { - const statusPathspecs = normalizeDockerChangedPaths(repoPath, paths); - const safePaths = - statusPathspecs?.length === 0 - ? [] - : parseStatusPaths( - await git( - [ - "status", - "--porcelain=v1", - "-z", - "--", - ...(statusPathspecs ?? [appsPath]).map((path_) => - literalPathspec(path_) - ), - ], - { - cwd: repoPath, - } - ) - ).filter((path_) => - isDockerUpdaterSafePath( - path_, - appsPath, - statusPathspecs !== undefined - ) - ); - if (safePaths.length === 0) { - const pushedPending = await pushPendingAutomationCommits(repoPath, [ + return withGitSyncLock( + repoPath, + async () => { + const statusPathspecs = normalizeDockerChangedPaths(repoPath, paths); + const safePaths = + statusPathspecs?.length === 0 + ? [] + : parseStatusPaths( + await git( + [ + "status", + "--porcelain=v1", + "-z", + "--", + ...(statusPathspecs ?? [appsPath]).map((path_) => + literalPathspec(path_) + ), + ], + { + cwd: repoPath, + signal, + } + ) + ).filter((path_) => + isDockerUpdaterSafePath( + path_, + appsPath, + statusPathspecs !== undefined + ) + ); + if (safePaths.length === 0) { + const pushedPending = await pushPendingAutomationCommits( + repoPath, + [DOCKER_SYNC_COMMIT_MESSAGE], + signal, + protectFromCancellation + ); + if (pushedPending) return pushedPending; + return { + changedPaths: [], + pushed: false, + skippedReason: "no safe changes", + }; + } + return commitAndPushPaths( + repoPath, + safePaths, DOCKER_SYNC_COMMIT_MESSAGE, - ]); - if (pushedPending) return pushedPending; - return { changedPaths: [], pushed: false, skippedReason: "no safe changes" }; - } - return commitAndPushPaths(repoPath, safePaths, DOCKER_SYNC_COMMIT_MESSAGE); - }); + signal, + protectFromCancellation + ); + }, + signal + ); } export function registerGitHygieneScheduledJobs(): void { @@ -415,11 +494,15 @@ export function registerGitHygieneScheduledJobs(): void { timeOfDay: "05:20", actionKey: "git.openclaw.workspace-sync", actionPayload: {}, + resourceClass: "host-heavy", } as const; registerScheduledJobAction( "git.openclaw.workspace-sync", - async () => { - const result = await syncOpenClawWorkspaceSafePaths(); + async (_job, signal, context) => { + const result = await syncOpenClawWorkspaceSafePaths( + signal, + context.protectFromCancellation + ); return { ...result }; }, { timeoutMs: GIT_WORKSPACE_SYNC_TIMEOUT_MS } diff --git a/backend/src/services/jobExecutionQueue.ts b/backend/src/services/jobExecutionQueue.ts new file mode 100644 index 000000000..da05e0137 --- /dev/null +++ b/backend/src/services/jobExecutionQueue.ts @@ -0,0 +1,726 @@ +import { database, sqlNullable } from "../database.ts"; +import { + isJobResourceClass, + type JobResourceClass, + jobResourcePriority, +} from "../lib/jobResources.ts"; + +const DEFAULT_LEASE_MS = 2 * 60 * 1000; +const MAX_EXECUTION_LIST_LIMIT = 200; + +export type JobExecutionStatus = + "queued" | "running" | "success" | "failed" | "cancelled"; +export type JobExecutionTriggerType = "manual" | "schedule" | "startup" | "system"; + +export interface JobExecution { + id: string; + scheduledJobId: string | undefined; + scheduledRunId: number | undefined; + actionKey: string; + displayName: string; + resourceClass: JobResourceClass; + priority: number; + status: JobExecutionStatus; + triggerType: JobExecutionTriggerType; + payload: Record; + queuedAt: string; + availableAt: string; + startedAt: string | undefined; + finishedAt: string | undefined; + leaseOwner: string | undefined; + leaseExpiresAt: string | undefined; + heartbeatAt: string | undefined; + cancelRequestedAt: string | undefined; + cancellable: boolean; + attempt: number; + timeoutMs: number; + message: string | undefined; + output: Record; +} + +export interface JobExecutionSummary { + activeResourceClasses: JobResourceClass[]; + oldestQueuedAgeMs: number | undefined; + oldestQueuedAt: string | undefined; + queued: number; + running: number; + workerCapacity: number; + workerCount: number; + workerLastHeartbeatAt: string | undefined; + workerOnline: boolean; +} + +export interface InsertJobExecutionInput { + actionKey: string; + availableAt?: string; + cancellable?: boolean; + displayName: string; + id?: string; + leaseOwner?: string; + payload?: Record; + priority?: number; + queuedAt: string; + resourceClass: JobResourceClass; + scheduledJobId?: string; + scheduledRunId?: number; + status?: "queued" | "running"; + timeoutMs: number; + triggerType: JobExecutionTriggerType; +} + +export interface EnqueueJobExecutionInput { + actionKey: string; + availableAt?: string; + cancellable?: boolean; + displayName: string; + id?: string; + payload?: Record; + priority?: number; + resourceClass: JobResourceClass; + timeoutMs: number; + triggerType?: JobExecutionTriggerType; +} + +type QueuedJobCancellationHandler = (execution: JobExecution, timestamp: string) => void; +type ExpiredJobExecutionHandler = (execution: JobExecution) => void; + +const queuedJobCancellationHandlers = new Map(); +const expiredJobExecutionHandlers = new Map(); + +/** Registers domain cleanup that participates in a queued cancellation transaction. */ +export function registerQueuedJobCancellationHandler( + actionKey: string, + handler: QueuedJobCancellationHandler +): void { + queuedJobCancellationHandlers.set(actionKey, handler); +} + +/** Registers domain cleanup that participates in expired-lease recovery. */ +export function registerExpiredJobExecutionHandler( + actionKey: string, + handler: ExpiredJobExecutionHandler +): void { + expiredJobExecutionHandlers.set(actionKey, handler); +} + +interface JobExecutionRow { + id: string; + scheduled_job_id: string | null | undefined; + scheduled_run_id: number | null | undefined; + action_key: string; + display_name: string; + resource_class: string; + priority: number; + status: string; + trigger_type: string; + payload_json: string; + queued_at: string; + available_at: string; + started_at: string | null | undefined; + finished_at: string | null | undefined; + lease_owner: string | null | undefined; + lease_expires_at: string | null | undefined; + heartbeat_at: string | null | undefined; + cancel_requested_at: string | null | undefined; + cancellable: number; + attempt: number; + timeout_ms: number; + message: string | null | undefined; + output_json: string; +} + +function nowIso(): string { + return new Date().toISOString(); +} + +function fromSqlNullable(value: T | null | undefined): T | undefined { + return value ?? undefined; +} + +function parseJsonObject(value: string): Record { + try { + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +function mapExecution(row: JobExecutionRow | undefined): JobExecution | undefined { + if (!row) return undefined; + return { + id: row.id, + scheduledJobId: fromSqlNullable(row.scheduled_job_id), + scheduledRunId: fromSqlNullable(row.scheduled_run_id), + actionKey: row.action_key, + displayName: row.display_name, + resourceClass: isJobResourceClass(row.resource_class) + ? row.resource_class + : "light", + priority: row.priority, + status: row.status as JobExecutionStatus, + triggerType: row.trigger_type as JobExecutionTriggerType, + payload: parseJsonObject(row.payload_json), + queuedAt: row.queued_at, + availableAt: row.available_at, + startedAt: fromSqlNullable(row.started_at), + finishedAt: fromSqlNullable(row.finished_at), + leaseOwner: fromSqlNullable(row.lease_owner), + leaseExpiresAt: fromSqlNullable(row.lease_expires_at), + heartbeatAt: fromSqlNullable(row.heartbeat_at), + cancelRequestedAt: fromSqlNullable(row.cancel_requested_at), + cancellable: row.cancellable === 1, + attempt: row.attempt, + timeoutMs: row.timeout_ms, + message: fromSqlNullable(row.message), + output: parseJsonObject(row.output_json), + }; +} + +function statusError(message: string, statusCode: number): Error { + return Object.assign(new Error(message), { statusCode }); +} + +function leaseExpiry(timestamp: string, leaseMs = DEFAULT_LEASE_MS): string { + return new Date(Date.parse(timestamp) + leaseMs).toISOString(); +} + +/** Inserts a queue row inside the caller's transaction. */ +export function insertJobExecution(input: InsertJobExecutionInput): JobExecution { + const id = input.id ?? Bun.randomUUIDv7(); + const status = input.status ?? "queued"; + const startedAt = status === "running" ? input.queuedAt : undefined; + const leaseOwner = status === "running" ? input.leaseOwner : undefined; + if (status === "running" && !leaseOwner) { + throw new Error("Running job executions require a lease owner"); + } + database + .prepare( + `INSERT INTO job_executions ( + id, scheduled_job_id, scheduled_run_id, action_key, display_name, + resource_class, priority, status, trigger_type, payload_json, + queued_at, available_at, started_at, lease_owner, lease_expires_at, + heartbeat_at, cancellable, attempt, timeout_ms, output_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '{}')` + ) + .run( + id, + sqlNullable(input.scheduledJobId), + sqlNullable(input.scheduledRunId), + input.actionKey, + input.displayName, + input.resourceClass, + input.priority ?? jobResourcePriority(input.resourceClass), + status, + input.triggerType, + JSON.stringify(input.payload ?? {}), + input.queuedAt, + input.availableAt ?? input.queuedAt, + sqlNullable(startedAt), + sqlNullable(leaseOwner), + sqlNullable(startedAt ? leaseExpiry(startedAt) : undefined), + sqlNullable(startedAt), + input.cancellable === false ? 0 : 1, + status === "running" ? 1 : 0, + input.timeoutMs + ); + return getJobExecution(id) as JobExecution; +} + +export function getJobExecution(id: string): JobExecution | undefined { + return mapExecution( + database.prepare("SELECT * FROM job_executions WHERE id = ?").get(id) as + JobExecutionRow | undefined + ); +} + +export function getLatestScheduledJobExecution( + scheduledJobId: string +): JobExecution | undefined { + return mapExecution( + database + .prepare( + `SELECT * FROM job_executions + WHERE scheduled_job_id = ? + ORDER BY queued_at DESC, id DESC + LIMIT 1` + ) + .get(scheduledJobId) as JobExecutionRow | undefined + ); +} + +export function getPreviousScheduledJobExecution( + scheduledJobId: string, + executionId: string +): JobExecution | undefined { + return mapExecution( + database + .prepare( + `SELECT candidate.* + FROM job_executions candidate + JOIN job_executions current ON current.id = ? + WHERE candidate.scheduled_job_id = ? + AND ( + candidate.queued_at < current.queued_at + OR ( + candidate.queued_at = current.queued_at + AND candidate.id < current.id + ) + ) + ORDER BY candidate.queued_at DESC, candidate.id DESC + LIMIT 1` + ) + .get(executionId, scheduledJobId) as JobExecutionRow | undefined + ); +} + +/** Adds non-scheduled work to the same persistent queue used by the scheduler. */ +export function enqueueJobExecution( + input: EnqueueJobExecutionInput, + queuedAt = nowIso() +): JobExecution { + return insertJobExecution({ + ...input, + queuedAt, + triggerType: input.triggerType ?? "manual", + }); +} + +export function listJobExecutions(limit = 50): JobExecution[] { + const normalizedLimit = + Number.isSafeInteger(limit) && limit > 0 + ? Math.min(limit, MAX_EXECUTION_LIST_LIMIT) + : 50; + return ( + database + .prepare( + `SELECT * FROM job_executions + ORDER BY + CASE WHEN status IN ('queued', 'running') THEN 0 ELSE 1 END, + CASE WHEN status IN ('queued', 'running') THEN queued_at END, + queued_at DESC, + id DESC + LIMIT ?` + ) + .all(normalizedLimit) as unknown as JobExecutionRow[] + ).map((row) => mapExecution(row) as JobExecution); +} + +export function getJobExecutionSummary(timestamp = Date.now()): JobExecutionSummary { + const counts = database + .prepare( + `SELECT + SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS queued, + SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running, + MIN(CASE WHEN status = 'queued' THEN queued_at END) AS oldest_queued_at + FROM job_executions` + ) + .get() as { + oldest_queued_at: string | null | undefined; + queued: number | null | undefined; + running: number | null | undefined; + }; + const activeRows = database + .prepare( + `SELECT DISTINCT resource_class + FROM job_executions + WHERE status = 'running' + ORDER BY resource_class` + ) + .all() as Array<{ resource_class: string }>; + const oldestQueuedAt = fromSqlNullable(counts.oldest_queued_at); + const parsedOldestQueuedAt = oldestQueuedAt ? Date.parse(oldestQueuedAt) : NaN; + const workerFreshAfter = new Date(timestamp - 30_000).toISOString(); + const worker = database + .prepare( + `SELECT COUNT(*) AS count, + COALESCE(MAX(capacity), 0) AS capacity, + MAX(heartbeat_at) AS last_heartbeat_at + FROM job_workers + WHERE heartbeat_at >= ?` + ) + .get(workerFreshAfter) as { + capacity: number | null | undefined; + count: number; + last_heartbeat_at: string | null | undefined; + }; + return { + activeResourceClasses: activeRows + .map((row) => row.resource_class) + .filter(isJobResourceClass), + oldestQueuedAgeMs: Number.isFinite(parsedOldestQueuedAt) + ? Math.max(0, timestamp - parsedOldestQueuedAt) + : undefined, + oldestQueuedAt, + queued: Number(counts.queued ?? 0), + running: Number(counts.running ?? 0), + workerCapacity: Number(worker.capacity ?? 0), + workerCount: Number(worker.count ?? 0), + workerLastHeartbeatAt: fromSqlNullable(worker.last_heartbeat_at), + workerOnline: worker.count > 0, + }; +} + +export function registerJobWorker( + id: string, + capacity: number, + timestamp = nowIso() +): void { + if (!Number.isSafeInteger(capacity) || capacity < 1) { + throw new Error("Job worker capacity must be a positive integer"); + } + database + .prepare( + `INSERT INTO job_workers (id, capacity, started_at, heartbeat_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + capacity = excluded.capacity, + started_at = excluded.started_at, + heartbeat_at = excluded.heartbeat_at` + ) + .run(id, capacity, timestamp, timestamp); + database + .prepare("DELETE FROM job_workers WHERE heartbeat_at < ?") + .run(new Date(Date.parse(timestamp) - 24 * 60 * 60 * 1000).toISOString()); +} + +export function didHeartbeatJobWorker(id: string, timestamp = nowIso()): boolean { + return ( + database + .prepare("UPDATE job_workers SET heartbeat_at = ? WHERE id = ?") + .run(timestamp, id).changes > 0 + ); +} + +export function unregisterJobWorker(id: string): void { + database.prepare("DELETE FROM job_workers WHERE id = ?").run(id); +} + +function finishExpiredExecution(row: JobExecutionRow, finishedAt: string): void { + const status: JobExecutionStatus = row.cancel_requested_at ? "cancelled" : "failed"; + const message = row.cancel_requested_at + ? "Job cancelled after its worker lease expired" + : "Job failed after its worker lease expired"; + const update = database + .prepare( + `UPDATE job_executions + SET status = ?, finished_at = ?, lease_owner = NULL, + lease_expires_at = NULL, message = ? + WHERE id = ? AND status = 'running'` + ) + .run(status, finishedAt, message, row.id); + if (update.changes === 0) return; + if (row.scheduled_run_id !== null && row.scheduled_run_id !== undefined) { + database + .prepare( + `UPDATE scheduled_job_runs + SET status = ?, finished_at = ?, message = ? + WHERE id = ? AND status = 'running'` + ) + .run(status, finishedAt, message, row.scheduled_run_id); + } + const recoveryHandler = expiredJobExecutionHandlers.get(row.action_key); + const execution = mapExecution({ + ...row, + finished_at: finishedAt, + lease_expires_at: undefined, + lease_owner: undefined, + message, + status, + }); + if (recoveryHandler && execution) { + recoveryHandler(execution); + } +} + +function recoverExpiredJobExecutionsInTransaction(timestamp: string): number { + const rows = database + .prepare( + `SELECT * FROM job_executions + WHERE status = 'running' + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= ?` + ) + .all(timestamp) as unknown as JobExecutionRow[]; + for (const row of rows) finishExpiredExecution(row, timestamp); + return rows.length; +} + +export function recoverExpiredJobExecutions(timestamp = nowIso()): number { + database.run("BEGIN IMMEDIATE"); + try { + const recovered = recoverExpiredJobExecutionsInTransaction(timestamp); + database.run("COMMIT"); + return recovered; + } catch (error) { + try { + database.run("ROLLBACK"); + } catch { + // Preserve the recovery error. + } + throw error; + } +} + +function claimExecutionRow( + row: JobExecutionRow, + workerId: string, + timestamp: string, + leaseMs: number +): JobExecution | undefined { + const update = database + .prepare( + `UPDATE job_executions + SET status = 'running', started_at = ?, heartbeat_at = ?, + lease_owner = ?, lease_expires_at = ?, attempt = attempt + 1 + WHERE id = ? AND status = 'queued' AND cancel_requested_at IS NULL` + ) + .run(timestamp, timestamp, workerId, leaseExpiry(timestamp, leaseMs), row.id); + if (update.changes === 0) return undefined; + if (row.scheduled_run_id !== null && row.scheduled_run_id !== undefined) { + database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'running', started_at = ? + WHERE id = ? AND status = 'queued'` + ) + .run(timestamp, row.scheduled_run_id); + } + return getJobExecution(row.id); +} + +export function claimNextJobExecution( + workerId: string, + capacity = 1, + timestamp = nowIso(), + leaseMs = DEFAULT_LEASE_MS +): JobExecution | undefined { + database.run("BEGIN IMMEDIATE"); + try { + recoverExpiredJobExecutionsInTransaction(timestamp); + const active = database + .prepare( + "SELECT COUNT(*) AS count FROM job_executions WHERE status = 'running'" + ) + .get() as { count: number }; + if (active.count >= capacity) { + database.run("COMMIT"); + return undefined; + } + const row = database + .prepare( + `SELECT * FROM job_executions + WHERE status = 'queued' + AND cancel_requested_at IS NULL + AND available_at <= ? + ORDER BY priority DESC, queued_at, id + LIMIT 1` + ) + .get(timestamp) as JobExecutionRow | undefined; + const claimed = row + ? claimExecutionRow(row, workerId, timestamp, leaseMs) + : undefined; + database.run("COMMIT"); + return claimed; + } catch (error) { + try { + database.run("ROLLBACK"); + } catch { + // Preserve the claim error. + } + throw error; + } +} + +export function heartbeatJobExecution( + id: string, + workerId: string, + timestamp = nowIso(), + leaseMs = DEFAULT_LEASE_MS +): { cancelRequested: boolean; hasLease: boolean } { + const update = database + .prepare( + `UPDATE job_executions + SET heartbeat_at = ?, lease_expires_at = ? + WHERE id = ? AND status = 'running' AND lease_owner = ?` + ) + .run(timestamp, leaseExpiry(timestamp, leaseMs), id, workerId); + if (update.changes === 0) { + return { cancelRequested: false, hasLease: false }; + } + const row = database + .prepare("SELECT cancel_requested_at FROM job_executions WHERE id = ?") + .get(id) as { cancel_requested_at: string | null | undefined } | undefined; + return { + cancelRequested: Boolean(row?.cancel_requested_at), + hasLease: true, + }; +} + +/** + * Atomically closes the UI cancellation window before an irreversible action. + * Queued executions remain cancellable until the worker starts the action. + */ +export function protectRunningJobExecutionFromCancellation(id: string): JobExecution { + const update = database + .prepare( + `UPDATE job_executions + SET cancellable = 0 + WHERE id = ? AND status = 'running' AND cancel_requested_at IS NULL` + ) + .run(id); + if (update.changes === 0) { + const execution = getJobExecution(id); + if (!execution) throw statusError("Job execution not found", 404); + if (execution.cancelRequestedAt) { + throw statusError("Job cancellation was already requested", 409); + } + throw statusError("Job execution is not running", 409); + } + return getJobExecution(id) as JobExecution; +} + +/** Replaces the bounded progress snapshot for an execution with an active lease. */ +export function updateJobExecutionOutput( + id: string, + workerId: string, + output: Record +): JobExecution { + const update = database + .prepare( + `UPDATE job_executions + SET output_json = ? + WHERE id = ? AND status = 'running' AND lease_owner = ?` + ) + .run(JSON.stringify(output), id, workerId); + if (update.changes === 0) { + throw statusError("Job execution lease is no longer active", 409); + } + return getJobExecution(id) as JobExecution; +} + +export function finishJobExecution( + id: string, + workerId: string, + status: "success" | "failed" | "cancelled", + message: string | undefined, + output: Record, + finishedAt = nowIso() +): JobExecution { + database.run("BEGIN IMMEDIATE"); + try { + const row = database + .prepare("SELECT * FROM job_executions WHERE id = ?") + .get(id) as JobExecutionRow | undefined; + if (!row) throw statusError("Job execution not found", 404); + if (row.status !== "running" || row.lease_owner !== workerId) { + throw statusError("Job execution lease is no longer active", 409); + } + const wasCancellationRequested = Boolean(row.cancel_requested_at); + const finalStatus: JobExecutionStatus = + wasCancellationRequested || status === "cancelled" ? "cancelled" : status; + const finalMessage = wasCancellationRequested + ? "Job cancelled" + : finalStatus === "cancelled" + ? (message ?? "Job cancelled") + : message; + database + .prepare( + `UPDATE job_executions + SET status = ?, finished_at = ?, lease_owner = NULL, + lease_expires_at = NULL, message = ?, output_json = ? + WHERE id = ? AND status = 'running' AND lease_owner = ?` + ) + .run( + finalStatus, + finishedAt, + sqlNullable(finalMessage), + JSON.stringify(output), + id, + workerId + ); + if (row.scheduled_run_id !== null && row.scheduled_run_id !== undefined) { + database + .prepare( + `UPDATE scheduled_job_runs + SET status = ?, finished_at = ?, message = ?, output_json = ? + WHERE id = ?` + ) + .run( + finalStatus, + finishedAt, + sqlNullable(finalMessage), + JSON.stringify(output), + row.scheduled_run_id + ); + } + database.run("COMMIT"); + return getJobExecution(id) as JobExecution; + } catch (error) { + try { + database.run("ROLLBACK"); + } catch { + // Preserve the finish error. + } + throw error; + } +} + +export function cancelJobExecution(id: string, timestamp = nowIso()): JobExecution { + database.run("BEGIN IMMEDIATE"); + try { + const row = database + .prepare("SELECT * FROM job_executions WHERE id = ?") + .get(id) as JobExecutionRow | undefined; + if (!row) throw statusError("Job execution not found", 404); + if (!row.cancellable) { + throw statusError("This job execution cannot be cancelled here", 409); + } + if (row.status === "queued") { + database + .prepare( + `UPDATE job_executions + SET status = 'cancelled', cancel_requested_at = ?, finished_at = ?, + message = 'Job cancelled before execution' + WHERE id = ? AND status = 'queued'` + ) + .run(timestamp, timestamp, id); + if (row.scheduled_run_id !== null && row.scheduled_run_id !== undefined) { + database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'cancelled', finished_at = ?, + message = 'Job cancelled before execution' + WHERE id = ? AND status = 'queued'` + ) + .run(timestamp, row.scheduled_run_id); + } + const cancellationHandler = queuedJobCancellationHandlers.get(row.action_key); + const execution = mapExecution(row); + if (cancellationHandler && execution) { + cancellationHandler(execution, timestamp); + } + } else if (row.status === "running") { + database + .prepare( + `UPDATE job_executions + SET cancel_requested_at = COALESCE(cancel_requested_at, ?) + WHERE id = ? AND status = 'running'` + ) + .run(timestamp, id); + } else { + throw statusError("Completed job executions cannot be cancelled", 409); + } + database.run("COMMIT"); + return getJobExecution(id) as JobExecution; + } catch (error) { + try { + database.run("ROLLBACK"); + } catch { + // Preserve the cancellation error. + } + throw error; + } +} diff --git a/backend/src/services/jobWorker.ts b/backend/src/services/jobWorker.ts new file mode 100644 index 000000000..b4630be20 --- /dev/null +++ b/backend/src/services/jobWorker.ts @@ -0,0 +1,95 @@ +import { registerBackupScheduledJobs } from "./backups.ts"; +import { registerCacheRefreshScheduledJobs } from "./cacheRefresh.ts"; +import { registerDockerExecutionActions } from "./dockerActions.ts"; +import { registerDockerUpdaterScheduledJobs } from "./dockerUpdater.ts"; +import { registerExecExecutionActions } from "./execJobs.ts"; +import { registerGitHygieneScheduledJobs } from "./gitHygiene.ts"; +import { registerLogRotationScheduledJobs } from "./logRotation.ts"; +import { registerOpenClawExecutionActions } from "./openclawActions.ts"; +import { registerPullRequestExecutionActions } from "./pullRequests.ts"; +import { + startScheduledJobExecutor, + startScheduledJobScheduler, + stopScheduledJobExecutor, + stopScheduledJobScheduler, +} from "./scheduledJobs.ts"; + +const workerState: { + isStarted: boolean; + pendingStop?: Promise; + stopGeneration: number; +} = { + isStarted: false, + stopGeneration: 0, +}; + +function trackWorkerStop(operation: () => Promise): Promise { + const generation = ++workerState.stopGeneration; + const pendingStop = (async () => { + try { + await operation(); + } finally { + if (workerState.stopGeneration === generation) { + workerState.pendingStop = undefined; + } + } + })(); + workerState.pendingStop = pendingStop; + return pendingStop; +} + +function registerScheduledActions(): void { + registerBackupScheduledJobs(); + registerCacheRefreshScheduledJobs({ seedStrategy: "queue" }); + registerDockerExecutionActions(); + registerDockerUpdaterScheduledJobs(); + registerExecExecutionActions(); + registerGitHygieneScheduledJobs(); + registerLogRotationScheduledJobs(); + registerOpenClawExecutionActions(); + registerPullRequestExecutionActions(); +} + +/** Starts the persistent queue scheduler and its single-concurrency executor. */ +export function startDashboardJobWorker(): void { + if (workerState.isStarted || workerState.pendingStop) return; + workerState.isStarted = true; + try { + registerScheduledActions(); + startScheduledJobExecutor(); + startScheduledJobScheduler(); + } catch (error) { + stopScheduledJobScheduler(); + void trackWorkerStop(async () => { + try { + await stopScheduledJobExecutor(); + workerState.isStarted = false; + } catch (cleanupError) { + console.error( + "[JobWorker] Failed to roll back executor startup:", + cleanupError + ); + } + }); + throw error; + } +} + +/** Stops claims first, then cooperatively aborts the active execution. */ +export async function stopDashboardJobWorker(): Promise { + for (;;) { + const pendingStop = workerState.pendingStop; + if (pendingStop) { + await pendingStop; + continue; + } + if (!workerState.isStarted) return; + await trackWorkerStop(async () => { + stopScheduledJobScheduler(); + await stopScheduledJobExecutor(); + // Release the startup guard only after executor cleanup succeeds. + workerState.isStarted = false; + }); + return; + } +} diff --git a/backend/src/services/logRotation.ts b/backend/src/services/logRotation.ts index 27e20ad2b..5f8fbe7fe 100644 --- a/backend/src/services/logRotation.ts +++ b/backend/src/services/logRotation.ts @@ -77,6 +77,7 @@ type ExecFileRunner = ( encoding?: BufferEncoding; env: NodeJS.ProcessEnv; maxBuffer: number; + signal?: AbortSignal; timeout?: number; } ) => Promise<{ stderr: string; stdout: string }>; @@ -89,6 +90,7 @@ const elevatedLogRotationExecFileRunner: ExecFileRunner = async ( const result = await runProcess(file, arguments_, { env: options.env, maxBuffer: options.maxBuffer, + signal: options.signal, timeoutMs: options.timeout, }); if (result.code !== 0) { @@ -1775,6 +1777,7 @@ export async function runLogRotationService( export async function runElevatedLogRotationService(options: { isDryRun: boolean; + signal?: AbortSignal; }): Promise { const modulePath = Bun.fileURLToPath(import.meta.url); const arguments_ = buildElevatedLogRotationCliArguments(modulePath, options); @@ -1785,6 +1788,7 @@ export async function runElevatedLogRotationService(options: { encoding: "utf8", env: elevatedLogRotationEnvironment(), maxBuffer: ELEVATED_LOG_ROTATION_MAX_BUFFER, + signal: options.signal, timeout: ELEVATED_LOG_ROTATION_TIMEOUT_MS, }); stderr = output.stderr; @@ -1964,11 +1968,16 @@ function persistLogRotationScheduledFailure( /** Registers the scheduled real log rotation job. */ export function registerLogRotationScheduledJobs(): void { - registerScheduledJobAction(LOG_ROTATION_JOB_ID, async () => { - const logRotation = await runElevatedLogRotationService({ isDryRun: false }); + registerScheduledJobAction(LOG_ROTATION_JOB_ID, async (job, signal, context) => { + const isDryRun = job.actionPayload.isDryRun === true; + if (!isDryRun) context.protectFromCancellation(); + const logRotation = await runElevatedLogRotationService({ + isDryRun, + signal, + }); if (logRotation.result?.isOk !== true) { const message = logRotationFailureMessage(logRotation); - persistLogRotationScheduledFailure(logRotation, message); + if (!isDryRun) persistLogRotationScheduledFailure(logRotation, message); throw new ScheduledJobActionError(message, { logRotation: capScheduledLogRotationFailure(logRotation), }); @@ -1991,6 +2000,7 @@ export function registerLogRotationScheduledJobs(): void { cronExpression: existing?.cronExpression ?? undefined, actionKey: LOG_ROTATION_JOB_ID, actionPayload: { key: STATE_CACHE_KEY }, + resourceClass: "host-heavy", }); database.run("COMMIT"); } catch (error) { diff --git a/backend/src/services/openclawActions.ts b/backend/src/services/openclawActions.ts new file mode 100644 index 000000000..78da23fc0 --- /dev/null +++ b/backend/src/services/openclawActions.ts @@ -0,0 +1,41 @@ +import os from "node:os"; +import path from "node:path"; + +import { runProcess } from "../lib/processes.ts"; +import { registerScheduledJobAction } from "./scheduledJobs.ts"; + +export const OPENCLAW_GATEWAY_RESTART_ACTION = "openclaw.gateway.restart"; + +function getOpenClawBin(): string { + const homeDirectory = process.env.HOME?.trim() || os.homedir(); + return ( + process.env.OPENCLAW_BIN?.trim() || + path.join(homeDirectory, ".npm-global/bin/openclaw") + ); +} + +export function registerOpenClawExecutionActions(): void { + registerScheduledJobAction( + OPENCLAW_GATEWAY_RESTART_ACTION, + async (_job, signal, context) => { + context.protectFromCancellation(); + const result = await runProcess(getOpenClawBin(), ["gateway", "restart"], { + signal, + timeoutMs: 30_000, + }); + if (result.code !== 0) { + throw new Error( + result.stderr.trim() || + result.stdout.trim() || + `openclaw exited ${result.code}` + ); + } + return { + code: result.code, + stderr: result.stderr, + stdout: result.stdout, + }; + }, + { timeoutMs: 60_000 } + ); +} diff --git a/backend/src/services/pullRequests.ts b/backend/src/services/pullRequests.ts index fab7be37c..cdb89bf11 100644 --- a/backend/src/services/pullRequests.ts +++ b/backend/src/services/pullRequests.ts @@ -10,6 +10,22 @@ import { spawnProcess, } from "../lib/processes.ts"; import { nonEmptyEnvironmentFallback } from "../lib/values.ts"; +import { + enqueueJobExecution, + type JobExecution, + registerExpiredJobExecutionHandler, + registerQueuedJobCancellationHandler, +} from "./jobExecutionQueue.ts"; +import { + successfulJobExecutionOutput, + waitForJobExecution, +} from "./queuedJobExecution.ts"; +import { + registerScheduledJobAction, + type ScheduledJob, + type ScheduledJobActionContext, + ScheduledJobActionError, +} from "./scheduledJobs.ts"; function dateToISOString(date: Date): string { return date.toISOString(); @@ -28,7 +44,10 @@ function resolveConfiguredRoot(environmentName: string, fallback: string): strin } const DASHBOARD_REPO = "rajohan/Mira-Dashboard"; -const DASHBOARD_SERVICE = "mira-dashboard.service"; +const DASHBOARD_SERVICES = [ + "mira-dashboard.service", + "mira-dashboard-worker.service", +] as const; const DEFAULT_REVIEWER_AUTHOR = "rajohan"; const DEFAULT_BASE = "main"; const DEPLOYMENT_LOCK_STALE_MS = 30 * 60 * 1000; @@ -36,6 +55,8 @@ const RECENT_DEPLOYMENTS_LIMIT = 10; const MAX_BUFFER = 20 * 1024 * 1024; const MAX_JSON_LINE_LENGTH = 1024 * 1024; const PR_LIST_TIMEOUT_MS = 180_000; +const DEPLOYMENT_RESTART_STATUS_POLL_MS = 1000; +const DEPLOYMENT_RESTART_CLAIM_PAUSE_TIMEOUT_MS = 2 * 60 * 1000; const PASSING_CHECK_VALUES = new Set(["success", "successful", "neutral", "skipped"]); const OPINIONATED_REVIEW_STATES = new Set(["APPROVED", "CHANGES_REQUESTED", "DISMISSED"]); const ACTIVE_DEPLOYMENT_STATUSES = new Set(["building", "restart-scheduled"]); @@ -283,6 +304,10 @@ interface DeploymentLockRow { updated_at: string; } +interface DeploymentLockExecutionRow { + status: JobExecution["status"]; +} + /** Checks whether an active deployment lock row is stale enough to replace. */ function isDeploymentLockStale(lock: DeploymentLockRow, now = Date.now()): boolean { const updatedAt = Date.parse(lock.updated_at); @@ -299,6 +324,30 @@ function readDeploymentLockRow(): DeploymentLockRow | undefined { .get() as DeploymentLockRow | undefined; } +function readDeploymentLockExecution( + lockOwner: string +): DeploymentLockExecutionRow | undefined { + return database + .prepare( + `SELECT status + FROM job_executions + WHERE json_valid(payload_json) + AND ( + ( + action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ? + ) + OR ( + action_key IN ('github.merge', 'github.merge-deploy') + AND json_extract(payload_json, '$.deploymentLockId') = ? + ) + ) + ORDER BY queued_at DESC, id DESC + LIMIT 1` + ) + .get(lockOwner, lockOwner) as DeploymentLockExecutionRow | undefined; +} + /** Releases the active deploy lock if it still belongs to the given job. */ function releaseDeploymentLock(jobId: string): void { try { @@ -310,14 +359,107 @@ function releaseDeploymentLock(jobId: string): void { } } +function cleanupTerminatedDeploymentExecution( + execution: JobExecution, + timestamp: string, + note: string +): void { + if (execution.actionKey === "dashboard.deploy") { + const deploymentId = execution.payload.deploymentId; + if (typeof deploymentId !== "string" || deploymentId.trim() === "") { + return; + } + const deployment = readDeploymentJob(deploymentId); + if (deployment && ACTIVE_DEPLOYMENT_STATUSES.has(deployment.status)) { + writeDeploymentJob({ + ...deployment, + note, + status: "failed", + updatedAt: timestamp, + }); + } + database + .prepare("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") + .run(deploymentId); + return; + } + + const deploymentLockId = execution.payload.deploymentLockId; + if (typeof deploymentLockId === "string" && deploymentLockId.trim() !== "") { + database + .prepare("DELETE FROM deployment_lock WHERE id = 1 AND job_id = ?") + .run(deploymentLockId); + } +} + +function cleanupQueuedDeploymentCancellation( + execution: JobExecution, + timestamp: string +): void { + cleanupTerminatedDeploymentExecution( + execution, + timestamp, + "Deploy cancelled before execution" + ); +} + +function cleanupExpiredDeploymentExecution(execution: JobExecution): void { + cleanupTerminatedDeploymentExecution( + execution, + execution.finishedAt ?? dateToISOString(new Date()), + execution.status === "cancelled" + ? "Deploy cancelled after its worker lease expired" + : "Deploy failed after its worker lease expired" + ); +} + +/** Restores action-specific cleanup for queued cancellations and expired leases. */ +export function registerPullRequestJobLifecycleHandlers(): void { + registerQueuedJobCancellationHandler( + "dashboard.deploy", + cleanupQueuedDeploymentCancellation + ); + registerQueuedJobCancellationHandler( + "github.merge", + cleanupQueuedDeploymentCancellation + ); + registerQueuedJobCancellationHandler( + "github.merge-deploy", + cleanupQueuedDeploymentCancellation + ); + registerExpiredJobExecutionHandler( + "dashboard.deploy", + cleanupExpiredDeploymentExecution + ); + registerExpiredJobExecutionHandler("github.merge", cleanupExpiredDeploymentExecution); + registerExpiredJobExecutionHandler( + "github.merge-deploy", + cleanupExpiredDeploymentExecution + ); +} + /** Ensures no active deploy owns the production checkout. */ function ensureNoActiveDeployment(): void { const activeLock = readDeploymentLockRow(); const activeJobId = activeLock?.job_id; if (activeJobId) { const activeJob = readDeploymentJob(activeJobId); + const lockExecution = readDeploymentLockExecution(activeJobId); + if (lockExecution?.status === "queued" || lockExecution?.status === "running") { + throw new Error(`Dashboard deploy already in progress (${activeJobId})`); + } + if (lockExecution && activeJob?.status === "building") { + writeDeploymentJob({ + ...activeJob, + note: "Deploy execution ended before build completion", + status: "failed", + updatedAt: dateToISOString(new Date()), + }); + database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); + return; + } if (!activeJob) { - if (!isDeploymentLockStale(activeLock)) { + if (!lockExecution && !isDeploymentLockStale(activeLock)) { throw new Error(`Dashboard deploy already in progress (${activeJobId})`); } database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); @@ -351,6 +493,15 @@ function acquireDeploymentLock(jobId: string): void { } } +function refreshDeploymentLockOwner(jobId: string): void { + const result = database + .prepare("UPDATE deployment_lock SET updated_at = ? WHERE id = 1 AND job_id = ?") + .run(dateToISOString(new Date()), jobId); + if (result.changes !== 1) { + throw new Error("Dashboard deploy lock ownership was lost"); + } +} + /** Refreshes the active deploy heartbeat while long-running work continues. */ function refreshDeploymentHeartbeat(job: DeploymentJob): DeploymentJob { const updatedJob = { ...job, updatedAt: dateToISOString(new Date()) }; @@ -512,6 +663,7 @@ async function runCommand( options: { cwd?: string; environment?: NodeJS.ProcessEnv; + signal?: AbortSignal; timeoutMs?: number; } = {} ): Promise { @@ -519,6 +671,7 @@ async function runCommand( cwd: options.cwd || getDashboardRoot(), env: options.environment || buildCommandEnvironment(), maxBuffer: MAX_BUFFER, + signal: options.signal, timeoutMs: options.timeoutMs || 120_000, }); if (code !== 0) { @@ -536,11 +689,12 @@ async function runCommand( } /** Runs a GitHub CLI command and parses its JSON output. */ -async function runGhJson(arguments_: string[]): Promise { +async function runGhJson(arguments_: string[], signal?: AbortSignal): Promise { const { code, stderr, stdout } = await runProcess("gh", arguments_, { cwd: getDashboardRoot(), env: buildCommandEnvironment(), maxBuffer: MAX_BUFFER, + signal, timeoutMs: 60_000, }); if (code !== 0) { @@ -838,37 +992,43 @@ function shouldRefreshBlockedMergeState(pr: PullRequestSummary): boolean { } /** Returns the current GitHub metadata for one pull request. */ -async function getPullRequest(number: number): Promise { +async function getPullRequest( + number: number, + signal?: AbortSignal +): Promise { return normalizePullRequest( - await runGhJson([ - "pr", - "view", - String(number), - "--repo", - DASHBOARD_REPO, - "--json", + await runGhJson( [ - "number", - "title", - "body", - "url", - "headRefName", - "headRefOid", - "baseRefName", - "author", - "createdAt", - "updatedAt", - "isDraft", - "mergeable", - "mergeStateStatus", - "reviewDecision", - "reviews", - "statusCheckRollup", - "additions", - "deletions", - "changedFiles", - ].join(","), - ]) + "pr", + "view", + String(number), + "--repo", + DASHBOARD_REPO, + "--json", + [ + "number", + "title", + "body", + "url", + "headRefName", + "headRefOid", + "baseRefName", + "author", + "createdAt", + "updatedAt", + "isDraft", + "mergeable", + "mergeStateStatus", + "reviewDecision", + "reviews", + "statusCheckRollup", + "additions", + "deletions", + "changedFiles", + ].join(","), + ], + signal + ) ); } @@ -917,8 +1077,12 @@ function isPathInsideRoot(value: string, root: string): boolean { } /** Performs find worktree for branch. */ -async function findWorktreeForBranch(branch: string): Promise { +async function findWorktreeForBranch( + branch: string, + signal?: AbortSignal +): Promise { const { stdout } = await runCommand("git", ["worktree", "list", "--porcelain"], { + signal, timeoutMs: 30_000, }); const expectedReference = `refs/heads/${branch}`; @@ -932,10 +1096,11 @@ async function findWorktreeForBranch(branch: string): Promise { try { - const worktree = await findWorktreeForBranch(branch); + const worktree = await findWorktreeForBranch(branch, signal); if (!worktree) { return { status: "skipped", @@ -958,7 +1123,7 @@ async function cleanupPullRequestWorktree( const { stdout: status } = await runCommand( "git", ["-C", worktreePath, "status", "--short"], - { timeoutMs: 30_000 } + { signal, timeoutMs: 30_000 } ); if (status.trim()) { return { @@ -970,6 +1135,7 @@ async function cleanupPullRequestWorktree( } await runCommand("git", ["worktree", "remove", worktreePath], { + signal, timeoutMs: 60_000, }); @@ -1108,19 +1274,27 @@ function normalizedCheckValue(value: unknown): string { } /** Returns production checkout status. */ -export async function getProductionCheckoutStatus(): Promise { +export async function getProductionCheckoutStatus( + signal?: AbortSignal +): Promise { const [{ stdout: root }, { stdout: branch }, { stdout: head }, { stdout: status }] = await Promise.all([ runCommand("git", ["rev-parse", "--show-toplevel"], { + signal, timeoutMs: 30_000, }), runCommand("git", ["rev-parse", "--abbrev-ref", "HEAD"], { + signal, timeoutMs: 30_000, }), runCommand("git", ["rev-parse", "--short", "HEAD"], { + signal, + timeoutMs: 30_000, + }), + runCommand("git", ["status", "--short"], { + signal, timeoutMs: 30_000, }), - runCommand("git", ["status", "--short"], { timeoutMs: 30_000 }), ]); let upstream: string | undefined; @@ -1128,7 +1302,7 @@ export async function getProductionCheckoutStatus(): Promise { - const status = await getProductionCheckoutStatus(); +export async function ensureProductionCheckout(signal?: AbortSignal): Promise { + const status = await getProductionCheckoutStatus(signal); if (!status.isProductionRoot) { throw new Error( @@ -1174,8 +1348,10 @@ export async function ensureProductionCheckout(): Promise { } /** Performs ensure production ready for deploy. */ -export async function ensureProductionReadyForDeploy(): Promise { - const status = await getProductionCheckoutStatus(); +export async function ensureProductionReadyForDeploy( + signal?: AbortSignal +): Promise { + const status = await getProductionCheckoutStatus(signal); if (!status.isSafeForDeploy) { throw new Error( @@ -1185,14 +1361,21 @@ export async function ensureProductionReadyForDeploy(): Promise { } /** Performs sync main. */ -async function syncMain(): Promise { - await ensureProductionCheckout(); - await runCommand("git", ["fetch", "--prune", "origin"], { timeoutMs: 120_000 }); - await runCommand("git", ["checkout", DEFAULT_BASE], { timeoutMs: 60_000 }); +async function syncMain(signal?: AbortSignal): Promise { + await ensureProductionCheckout(signal); + await runCommand("git", ["fetch", "--prune", "origin"], { + signal, + timeoutMs: 120_000, + }); + await runCommand("git", ["checkout", DEFAULT_BASE], { + signal, + timeoutMs: 60_000, + }); await runCommand("git", ["pull", "--ff-only", "origin", DEFAULT_BASE], { + signal, timeoutMs: 120_000, }); - await ensureProductionReadyForDeploy(); + await ensureProductionReadyForDeploy(signal); } /** Performs shell quote. */ @@ -1268,12 +1451,15 @@ try { } /** Performs schedule restart health check. */ -async function scheduleRestartHealthCheck(job: DeploymentJob): Promise { +async function scheduleRestartHealthCheck( + job: DeploymentJob, + signal?: AbortSignal +): Promise { const okJob: DeploymentJob = { ...job, status: "isOk", updatedAt: dateToISOString(new Date()), - note: "Restarted service and health check passed", + note: "Restarted web and worker services; health checks passed", }; const failedJob: DeploymentJob = { ...job, @@ -1283,10 +1469,11 @@ async function scheduleRestartHealthCheck(job: DeploymentJob): Promise/dev/null; then', + `if [ "$restart_status" -eq 0 ] && curl -fsS http://127.0.0.1:3100/api/health | grep -Fq '"workerOnline":true'; then`, ` ${deploymentJobUpdateCommand(okJob)}`, "else", ` ${deploymentJobUpdateCommand(failedJob)}`, @@ -1303,17 +1490,20 @@ async function scheduleRestartHealthCheck(job: DeploymentJob): Promise { +async function runDeploymentJob( + job: DeploymentJob, + signal?: AbortSignal +): Promise { let currentJob = job; const dashboardRoot = getDashboardRoot(); try { currentJob = refreshDeploymentHeartbeat(currentJob); - await syncMain(); + await syncMain(signal); currentJob = refreshDeploymentHeartbeat(currentJob); currentJob = refreshDeploymentHeartbeat(currentJob); @@ -1322,10 +1512,14 @@ async function runDeploymentJob(job: DeploymentJob): Promise { recursive: true, }); await runCommand("bun", ["install", "--frozen-lockfile"], { + signal, timeoutMs: 180_000, }); currentJob = refreshDeploymentHeartbeat(currentJob); - await runCommand("bun", ["run", "build"], { timeoutMs: 180_000 }); + await runCommand("bun", ["run", "build"], { + signal, + timeoutMs: 180_000, + }); currentJob = refreshDeploymentHeartbeat(currentJob); await rm(path.join(dashboardRoot, "backend", "node_modules"), { force: true, @@ -1333,11 +1527,13 @@ async function runDeploymentJob(job: DeploymentJob): Promise { }); await runCommand("bun", ["install", "--frozen-lockfile"], { cwd: path.join(dashboardRoot, "backend"), + signal, timeoutMs: 120_000, }); currentJob = refreshDeploymentHeartbeat(currentJob); await runCommand("bun", ["run", "build"], { cwd: path.join(dashboardRoot, "backend"), + signal, timeoutMs: 120_000, }); currentJob = refreshDeploymentHeartbeat(currentJob); @@ -1345,13 +1541,14 @@ async function runDeploymentJob(job: DeploymentJob): Promise { "git", ["rev-parse", "--short", "HEAD"], { + signal, timeoutMs: 30_000, } ); const { stdout: commitTitle } = await runCommand( "git", ["log", "-1", "--pretty=%s"], - { timeoutMs: 30_000 } + { signal, timeoutMs: 30_000 } ); currentJob = refreshDeploymentHeartbeat(currentJob); @@ -1364,7 +1561,8 @@ async function runDeploymentJob(job: DeploymentJob): Promise { note: "Build passed; restart + health check scheduled", }; writeDeploymentJob(restartScheduled); - await scheduleRestartHealthCheck(restartScheduled); + await scheduleRestartHealthCheck(restartScheduled, signal); + return true; } catch (error) { const failed: DeploymentJob = { ...currentJob, @@ -1377,30 +1575,54 @@ async function runDeploymentJob(job: DeploymentJob): Promise { } finally { releaseDeploymentLock(job.id); } + return false; } } -/** Reports background deployment failures after the API response has returned. */ -function reportBackgroundDeploymentError(error: unknown): void { - console.error( - "[pullRequestsRoutes] Background deploy failed:", - errorMessage(error, "Deploy failed") +function resumeWorkerClaimsWhenDeploymentRestartSettles( + deploymentId: string, + resumeWorkerClaims: () => void +): void { + let isSettled = false; + const settle = () => { + if (isSettled) return; + isSettled = true; + clearInterval(restartPoll); + clearTimeout(failSafe); + resumeWorkerClaims(); + }; + const checkRestartStatus = () => { + try { + const deployment = readDeploymentJob(deploymentId); + if (!deployment || !ACTIVE_DEPLOYMENT_STATUSES.has(deployment.status)) { + settle(); + } + } catch (error) { + console.warn( + "[PullRequests] Failed to inspect detached deployment restart:", + error + ); + } + }; + const restartPoll = setInterval( + checkRestartStatus, + DEPLOYMENT_RESTART_STATUS_POLL_MS ); + restartPoll.unref(); + const failSafe = setTimeout(() => { + console.warn( + "[PullRequests] Resuming worker claims after deployment restart pause timed out", + { deploymentId } + ); + settle(); + }, DEPLOYMENT_RESTART_CLAIM_PAUSE_TIMEOUT_MS); + failSafe.unref(); + queueMicrotask(checkRestartStatus); } -async function runDeploymentJobAndReportErrors( - job: DeploymentJob, - runner = runDeploymentJob -): Promise { - try { - await runner(job); - } catch (error) { - reportBackgroundDeploymentError(error); - } -} - -/** Starts deploy latest in the background. */ +/** Persists a deployment and puts its execution behind the worker lease. */ export function startDeployLatest(lockHeldBy?: string): DeploymentJob { + registerPullRequestJobLifecycleHandlers(); const now = dateToISOString(new Date()); const job: DeploymentJob = { id: Bun.randomUUIDv7(), @@ -1423,25 +1645,47 @@ export function startDeployLatest(lockHeldBy?: string): DeploymentJob { } try { writeDeploymentJob(job); - void runDeploymentJobAndReportErrors(job); + enqueueJobExecution({ + actionKey: "dashboard.deploy", + displayName: "Deploy Mira Dashboard", + payload: { deploymentId: job.id }, + resourceClass: "exclusive", + timeoutMs: 45 * 60 * 1000, + }); return job; } catch (error) { releaseDeploymentLock(job.id); if (lockHeldBy) { releaseDeploymentLock(lockHeldBy); } + writeDeploymentJob({ + ...job, + note: errorMessage(error, "Dashboard deploy failed to queue"), + status: "failed", + updatedAt: dateToISOString(new Date()), + }); throw error; } } +/** Queues a direct deploy; production validation is owned by the worker action. */ +export async function prepareAndStartDeployLatest(): Promise { + return startDeployLatest(); +} + +interface PullRequestApprovalExecutionOptions { + lockHeldBy?: string; + signal?: AbortSignal; +} + /** Performs approve pull request. */ -export async function approvePullRequest(number: number, willDeploy: boolean) { - await ensureProductionCheckout(); - const pr = await getPullRequest(number); - validateDashboardPrForApproval(pr); - const lockId = `approve-${Bun.randomUUIDv7()}`; - acquireDeploymentLock(lockId); - let isReleaseLock = true; +export async function approvePullRequest( + number: number, + willDeploy: boolean, + options: PullRequestApprovalExecutionOptions = {} +) { + const lockId = options.lockHeldBy ?? `approve-${Bun.randomUUIDv7()}`; + let isReleaseLock = options.lockHeldBy !== undefined; let syncError: string | undefined; let deployError: string | undefined; @@ -1449,6 +1693,16 @@ export async function approvePullRequest(number: number, willDeploy: boolean) { let cleanup: WorktreeCleanupResult; try { + if (options.lockHeldBy) { + refreshDeploymentLockOwner(lockId); + } + await ensureProductionCheckout(options.signal); + const pr = await getPullRequest(number, options.signal); + validateDashboardPrForApproval(pr); + if (!options.lockHeldBy) { + acquireDeploymentLock(lockId); + isReleaseLock = true; + } await runCommand( "gh", [ @@ -1460,12 +1714,12 @@ export async function approvePullRequest(number: number, willDeploy: boolean) { "--repo", DASHBOARD_REPO, ], - { timeoutMs: 120_000 } + { signal: options.signal, timeoutMs: 120_000 } ); - cleanup = await cleanupPullRequestWorktree(pr.headRefName); + cleanup = await cleanupPullRequestWorktree(pr.headRefName, options.signal); try { - await syncMain(); + await syncMain(options.signal); } catch (error) { syncError = errorMessage(error, "Failed to sync main after merge"); } @@ -1500,18 +1754,53 @@ export async function approvePullRequest(number: number, willDeploy: boolean) { }; } +function queuedPullRequestResult(execution: JobExecution): T { + if ("result" in execution.output) return execution.output.result as T; + successfulJobExecutionOutput(execution); + throw new Error("Pull request result was missing"); +} + +/** Runs PR merge/deploy through the shared persistent execution plane. */ +export async function runPullRequestApproval(number: number, willDeploy: boolean) { + registerPullRequestJobLifecycleHandlers(); + const deploymentLockId = `approve-${Bun.randomUUIDv7()}`; + acquireDeploymentLock(deploymentLockId); + let execution: JobExecution; + try { + execution = enqueueJobExecution({ + actionKey: willDeploy ? "github.merge-deploy" : "github.merge", + displayName: willDeploy + ? `Merge and deploy PR #${number}` + : `Merge PR #${number}`, + payload: { deploymentLockId, number, willDeploy }, + resourceClass: "exclusive", + timeoutMs: (willDeploy ? 45 : 10) * 60 * 1000, + }); + } catch (error) { + releaseDeploymentLock(deploymentLockId); + throw error; + } + return queuedPullRequestResult>>( + await waitForJobExecution(execution.id, { timeoutMs: 60 * 60 * 1000 }) + ); +} + /** Performs approve pull request review. */ -export async function approvePullRequestReview(number: number) { - const pr = await getPullRequest(number); +export async function approvePullRequestReview(number: number, signal?: AbortSignal) { + const pr = await getPullRequest(number, signal); validateDashboardPrForReviewApproval(pr); await runCommand( "gh", ["pr", "review", String(number), "--approve", "--repo", DASHBOARD_REPO], - { environment: buildReviewCommandEnvironment(), timeoutMs: 60_000 } + { + environment: buildReviewCommandEnvironment(), + signal, + timeoutMs: 60_000, + } ); - const pullRequest = await getPullRequest(number); + const pullRequest = await getPullRequest(number, signal); return { isOk: true, @@ -1521,8 +1810,8 @@ export async function approvePullRequestReview(number: number) { } /** Updates one pull request branch with the latest base branch. */ -export async function updatePullRequestBranch(number: number) { - const pr = await getPullRequest(number); +export async function updatePullRequestBranch(number: number, signal?: AbortSignal) { + const pr = await getPullRequest(number, signal); validateDashboardPrForBranchUpdate(pr); const repo = parseRepoParts(DASHBOARD_REPO); const arguments_ = [ @@ -1535,26 +1824,30 @@ export async function updatePullRequestBranch(number: number) { arguments_.push("-f", `expected_head_sha=${pr.headRefOid}`); } - await runCommand("gh", arguments_, { timeoutMs: 60_000 }); + await runCommand("gh", arguments_, { signal, timeoutMs: 60_000 }); return { isOk: true, message: `PR #${number} branch update started`, - pullRequest: await getPullRequest(number), + pullRequest: await getPullRequest(number, signal), }; } /** Performs reject pull request. */ -export async function rejectPullRequest(number: number, comment: string) { - const pr = await getPullRequest(number); +export async function rejectPullRequest( + number: number, + comment: string, + signal?: AbortSignal +) { + const pr = await getPullRequest(number, signal); validateDashboardPr(pr); await runCommand( "gh", ["pr", "close", String(number), "--repo", DASHBOARD_REPO, "--comment", comment], - { timeoutMs: 60_000 } + { signal, timeoutMs: 60_000 } ); - const cleanup = await cleanupPullRequestWorktree(pr.headRefName); + const cleanup = await cleanupPullRequestWorktree(pr.headRefName, signal); return { isOk: true, @@ -1562,3 +1855,139 @@ export async function rejectPullRequest(number: number, comment: string) { cleanup, }; } + +/** Records a GitHub review approval in the shared execution plane. */ +export async function runPullRequestReviewApproval(number: number) { + const execution = enqueueJobExecution({ + actionKey: "github.review-approval", + displayName: `Approve review for PR #${number}`, + payload: { number }, + resourceClass: "network", + timeoutMs: 2 * 60 * 1000, + }); + return queuedPullRequestResult>>( + await waitForJobExecution(execution.id, { timeoutMs: 15 * 60 * 1000 }) + ); +} + +/** Records a GitHub branch update in the shared execution plane. */ +export async function runPullRequestBranchUpdate(number: number) { + const execution = enqueueJobExecution({ + actionKey: "github.update-branch", + displayName: `Update branch for PR #${number}`, + payload: { number }, + resourceClass: "network", + timeoutMs: 2 * 60 * 1000, + }); + return queuedPullRequestResult>>( + await waitForJobExecution(execution.id, { timeoutMs: 15 * 60 * 1000 }) + ); +} + +/** Records a PR close and local worktree cleanup in the execution plane. */ +export async function runPullRequestRejection(number: number, comment: string) { + const execution = enqueueJobExecution({ + actionKey: "github.reject", + displayName: `Reject PR #${number}`, + payload: { comment, number }, + resourceClass: "exclusive", + timeoutMs: 5 * 60 * 1000, + }); + return queuedPullRequestResult>>( + await waitForJobExecution(execution.id, { timeoutMs: 15 * 60 * 1000 }) + ); +} + +function executionPullRequestNumber(payload: Record): number { + const number = payload.number; + if (typeof number === "number" && Number.isSafeInteger(number) && number > 0) { + return number; + } + return validatePrNumber(number); +} + +async function executePullRequestMerge( + job: ScheduledJob, + signal: AbortSignal | undefined, + context: ScheduledJobActionContext +) { + context.protectFromCancellation(); + const number = executionPullRequestNumber(job.actionPayload); + const willDeploy = job.actionPayload.willDeploy === true; + const deploymentLockId = job.actionPayload.deploymentLockId; + if ( + deploymentLockId !== undefined && + (typeof deploymentLockId !== "string" || deploymentLockId.trim() === "") + ) { + throw Object.assign(new Error("Deployment lock id is invalid"), { + statusCode: 400, + }); + } + const result = await approvePullRequest(number, willDeploy, { + lockHeldBy: deploymentLockId, + signal, + }); + const message = result.syncError || result.deployError; + if (message) { + throw new ScheduledJobActionError(message, { result }); + } + return { result }; +} + +/** Registers every mutating GitHub/deploy action exclusively in the worker. */ +export function registerPullRequestExecutionActions(): void { + registerPullRequestJobLifecycleHandlers(); + registerScheduledJobAction("dashboard.deploy", async (job, signal, context) => { + const deploymentId = job.actionPayload.deploymentId; + if (typeof deploymentId !== "string" || deploymentId.trim() === "") { + throw Object.assign(new Error("Deployment id is missing"), { + statusCode: 400, + }); + } + const deployment = readDeploymentJob(deploymentId); + if (!deployment) { + throw Object.assign(new Error("Deployment job not found"), { + statusCode: 404, + }); + } + context.protectFromCancellation(); + const isSuccess = await runDeploymentJob(deployment, signal); + if (!isSuccess) { + throw new ScheduledJobActionError("Dashboard deploy failed", { + deploymentId, + }); + } + const resumeWorkerClaims = context.pauseWorkerClaims(); + resumeWorkerClaimsWhenDeploymentRestartSettles(deploymentId, resumeWorkerClaims); + return { deploymentId }; + }); + registerScheduledJobAction("github.merge", executePullRequestMerge); + registerScheduledJobAction("github.merge-deploy", executePullRequestMerge); + registerScheduledJobAction("github.review-approval", async (job, signal, context) => { + const number = executionPullRequestNumber(job.actionPayload); + context.protectFromCancellation(); + return { + result: await approvePullRequestReview(number, signal), + }; + }); + registerScheduledJobAction("github.update-branch", async (job, signal, context) => { + const number = executionPullRequestNumber(job.actionPayload); + context.protectFromCancellation(); + return { + result: await updatePullRequestBranch(number, signal), + }; + }); + registerScheduledJobAction("github.reject", async (job, signal, context) => { + const comment = job.actionPayload.comment; + if (typeof comment !== "string" || comment.trim() === "") { + throw Object.assign(new Error("Pull request rejection comment is missing"), { + statusCode: 400, + }); + } + const number = executionPullRequestNumber(job.actionPayload); + context.protectFromCancellation(); + return { + result: await rejectPullRequest(number, comment, signal), + }; + }); +} diff --git a/backend/src/services/queuedJobExecution.ts b/backend/src/services/queuedJobExecution.ts new file mode 100644 index 000000000..f8f7f18f6 --- /dev/null +++ b/backend/src/services/queuedJobExecution.ts @@ -0,0 +1,124 @@ +import { errorMessage } from "../lib/errors.ts"; +import { + cancelJobExecution, + enqueueJobExecution, + type EnqueueJobExecutionInput, + getJobExecution, + type JobExecution, +} from "./jobExecutionQueue.ts"; + +const DEFAULT_POLL_INTERVAL_MS = 100; +const DEFAULT_WAIT_TIMEOUT_MS = 30 * 60 * 1000; + +export interface WaitForJobExecutionOptions { + pollIntervalMs?: number; + signal?: AbortSignal; + timeoutMs?: number; +} + +function waitForDelay(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(new DOMException("Request aborted", "AbortError")); + } + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal?.removeEventListener("abort", abort); + resolve(); + }, delayMs); + const abort = () => { + clearTimeout(timeout); + reject(new DOMException("Request aborted", "AbortError")); + }; + signal?.addEventListener("abort", abort, { once: true }); + timeout.unref(); + }); +} + +export function isTerminalJobExecution(execution: JobExecution): boolean { + return ["success", "failed", "cancelled"].includes(execution.status); +} + +/** Observes a persisted result. Observer aborts do not cancel the execution. */ +export async function waitForJobExecution( + id: string, + options: WaitForJobExecutionOptions = {} +): Promise { + const startedAt = Date.now(); + const timeoutMs = options.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + while (true) { + const execution = getJobExecution(id); + if (!execution) { + throw Object.assign(new Error("Job execution not found"), { + statusCode: 404, + }); + } + if (isTerminalJobExecution(execution)) return execution; + if (Date.now() - startedAt >= timeoutMs) { + if (execution.status === "queued" && execution.cancellable) { + try { + cancelJobExecution(id); + } catch (error) { + const current = getJobExecution(id); + if (current && isTerminalJobExecution(current)) { + return current; + } + throw error; + } + } + throw Object.assign( + new Error("Timed out while waiting for the queued job result"), + { executionId: id, statusCode: 504 } + ); + } + await waitForDelay(pollIntervalMs, options.signal); + } +} + +export async function enqueueAndWaitForJobExecution( + input: EnqueueJobExecutionInput, + options: WaitForJobExecutionOptions = {} +): Promise { + const execution = enqueueJobExecution(input); + return await waitForJobExecution(execution.id, options); +} + +/** Returns a successful result or rethrows the worker's persisted failure. */ +export function successfulJobExecutionOutput( + execution: JobExecution +): Record { + if (execution.status === "success") return execution.output; + const statusCode = Number(execution.output.statusCode); + throw Object.assign( + new Error( + execution.message || + (execution.status === "cancelled" + ? "Job execution was cancelled" + : "Job execution failed") + ), + { + executionId: execution.id, + statusCode: + Number.isSafeInteger(statusCode) && statusCode >= 400 && statusCode < 600 + ? statusCode + : execution.status === "cancelled" + ? 409 + : 500, + } + ); +} + +export function jobExecutionFailureOutput( + error: unknown, + fallback = "Job execution failed" +): Record { + const statusCode = Number( + (error as { statusCode?: unknown } | undefined)?.statusCode + ); + return { + error: errorMessage(error, fallback), + ...(Number.isSafeInteger(statusCode) && + statusCode >= 400 && + statusCode < 600 && { statusCode }), + }; +} diff --git a/backend/src/services/scheduledJobs.ts b/backend/src/services/scheduledJobs.ts index 9f09dc670..1584ea801 100644 --- a/backend/src/services/scheduledJobs.ts +++ b/backend/src/services/scheduledJobs.ts @@ -1,6 +1,26 @@ import { database, sqlNullable } from "../database.ts"; import { errorMessage } from "../lib/errors.ts"; +import { + isJobResourceClass, + type JobResourceClass, + withJobResourceClass, +} from "../lib/jobResources.ts"; import { type JobDisableIntent, parseJobDisableIntent } from "./jobDisableIntent.ts"; +import { + claimNextJobExecution, + didHeartbeatJobWorker, + finishJobExecution, + getJobExecution, + heartbeatJobExecution, + insertJobExecution, + type JobExecution, + protectRunningJobExecutionFromCancellation, + recoverExpiredJobExecutions, + registerJobWorker, + unregisterJobWorker, + updateJobExecutionOutput, +} from "./jobExecutionQueue.ts"; +import { waitForJobExecution } from "./queuedJobExecution.ts"; function dateToISOString(date: Date): string { return date.toISOString(); @@ -10,26 +30,52 @@ const schedulerTickMs = 30_000; const defaultScheduledJobRunTimeoutMs = 5 * 60 * 1000; const minimumIntervalSeconds = 60; const latestRunsJobIdChunkSize = 900; -const runningJobs = new Set(); -const scheduledJobRuns = new Set>(); +const executorTickMs = 1000; +const executorHeartbeatMs = 1000; +const executorCapacity = 1; +const interruptedHandlerGraceMs = 30_000; const actionHandlers = new Map(); -const abortHandlerSettled = new WeakMap>(); +const interruptedHandlerSettled = new WeakMap< + ScheduledJobInterruptionError, + Promise +>(); +const activeExecutionControllers = new Map(); +const activeExecutionRuns = new Map>(); const scheduledJobRuntimeState: { scheduler: NodeJS.Timeout | undefined; + executor: NodeJS.Timeout | undefined; + workerHeartbeat: NodeJS.Timeout | undefined; + executorClaimPauseGeneration: number; isSchedulerTickRunning: boolean; + isExecutorClaimingPaused: boolean; + isExecutorTickRunning: boolean; + workerId: string; } = { scheduler: undefined, + executor: undefined, + workerHeartbeat: undefined, + executorClaimPauseGeneration: 0, isSchedulerTickRunning: false, + isExecutorClaimingPaused: false, + isExecutorTickRunning: false, + workerId: `dashboard-worker:${process.pid}:${Bun.randomUUIDv7()}`, }; -const scheduledJobRunTimeoutMs = defaultScheduledJobRunTimeoutMs; export type ScheduledJobScheduleType = "interval" | "daily" | "cron"; -export type ScheduledJobRunStatus = "running" | "success" | "failed"; -export type ScheduledJobTriggerType = "manual" | "schedule"; +export type ScheduledJobRunStatus = + "queued" | "running" | "success" | "failed" | "cancelled"; +export type ScheduledJobTriggerType = "manual" | "schedule" | "startup"; +export interface ScheduledJobActionContext { + executionId: string; + pauseWorkerClaims: () => () => void; + protectFromCancellation: () => void; + updateOutput: (output: Record) => void; +} export type ScheduledJobActionHandler = ( job: ScheduledJob, - signal?: AbortSignal + signal: AbortSignal | undefined, + context: ScheduledJobActionContext ) => Promise | void> | Record | void; export interface ScheduledJobActionOptions { @@ -52,14 +98,14 @@ export class ScheduledJobActionError extends Error { } } -class ScheduledJobAbortError extends Error { - constructor(handlerSettled: Promise) { - super("Scheduled job aborted"); - abortHandlerSettled.set(this, handlerSettled); +class ScheduledJobInterruptionError extends Error { + constructor(message: string, handlerSettled: Promise) { + super(message); + interruptedHandlerSettled.set(this, handlerSettled); } getHandlerSettled(): Promise { - return abortHandlerSettled.get(this)!; + return interruptedHandlerSettled.get(this)!; } } @@ -79,6 +125,9 @@ export interface ScheduledJob { createdAt: string; updatedAt: string; lastRun: ScheduledJobRun | undefined; + resourceClass: JobResourceClass; + timeoutMs: number; + isQueued: boolean; isRunning: boolean; } @@ -91,6 +140,11 @@ export interface ScheduledJobRun { finishedAt: string | undefined; message: string | undefined; output: Record; + executionId: string | undefined; + queuedAt: string; + resourceClass: JobResourceClass; + cancelRequestedAt: string | undefined; + cancellable: boolean; } export interface ScheduledJobDefinition { @@ -104,6 +158,8 @@ export interface ScheduledJobDefinition { cronExpression?: string | undefined; actionKey: string; actionPayload?: Record; + resourceClass?: JobResourceClass; + timeoutMs?: number; } export interface ScheduledJobPatch { @@ -131,6 +187,8 @@ interface ScheduledJobRow { next_run_at: string | null | undefined; created_at: string; updated_at: string; + resource_class?: string | null; + timeout_ms?: number | null; } interface ScheduledJobRunRow { @@ -142,6 +200,11 @@ interface ScheduledJobRunRow { finished_at: string | null | undefined; message: string | null | undefined; output_json: string; + execution_id?: string | null; + execution_queued_at?: string | null; + execution_resource_class?: string | null; + execution_cancel_requested_at?: string | null; + execution_cancellable?: number | null; } export class ScheduledJobValidationError extends Error { @@ -423,6 +486,9 @@ function mapRun(row: ScheduledJobRunRow | undefined): ScheduledJobRun | undefine if (!row) { return undefined; } + const resourceClass = isJobResourceClass(row.execution_resource_class) + ? row.execution_resource_class + : "light"; return { id: row.id, jobId: row.job_id, @@ -432,6 +498,11 @@ function mapRun(row: ScheduledJobRunRow | undefined): ScheduledJobRun | undefine finishedAt: fromSqlNullable(row.finished_at), message: fromSqlNullable(row.message), output: parseJsonObject(row.output_json), + executionId: fromSqlNullable(row.execution_id), + queuedAt: fromSqlNullable(row.execution_queued_at) ?? row.started_at, + resourceClass, + cancelRequestedAt: fromSqlNullable(row.execution_cancel_requested_at), + cancellable: row.execution_cancellable !== 0, }; } @@ -463,11 +534,18 @@ function latestRunsByJobId(jobIds: string[]): Map { FROM ( SELECT run.*, + execution.id AS execution_id, + execution.queued_at AS execution_queued_at, + execution.resource_class AS execution_resource_class, + execution.cancel_requested_at AS execution_cancel_requested_at, + execution.cancellable AS execution_cancellable, ROW_NUMBER() OVER ( PARTITION BY run.job_id ORDER BY run.started_at DESC, run.id DESC ) AS row_number FROM scheduled_job_runs run + LEFT JOIN job_executions execution + ON execution.scheduled_run_id = run.id WHERE run.job_id IN (${placeholders}) ) WHERE row_number = 1 @@ -501,7 +579,15 @@ function mapJob( createdAt: row.created_at, updatedAt: row.updated_at, lastRun: latestRuns.get(row.id) ?? undefined, - isRunning: runningJobs.has(row.id), + resourceClass: isJobResourceClass(row.resource_class) + ? row.resource_class + : "light", + timeoutMs: + typeof row.timeout_ms === "number" && row.timeout_ms > 0 + ? row.timeout_ms + : defaultScheduledJobRunTimeoutMs, + isQueued: latestRuns.get(row.id)?.status === "queued", + isRunning: latestRuns.get(row.id)?.status === "running", }; } @@ -566,6 +652,12 @@ export function upsertScheduledJob(definition: ScheduledJobDefinition): Schedule new Date(timestamp) ) : existing.nextRunAt; + const resourceClass = definition.resourceClass ?? "light"; + const timeoutMs = + definition.timeoutMs ?? + actionHandlers.get(definition.actionKey)?.timeoutMs ?? + defaultScheduledJobRunTimeoutMs; + assertValidActionTimeoutMs(timeoutMs); database .prepare( `INSERT INTO scheduled_jobs ( @@ -600,20 +692,42 @@ export function upsertScheduledJob(definition: ScheduledJobDefinition): Schedule existing?.createdAt ?? timestamp, timestamp ); + database + .prepare( + `INSERT INTO scheduled_job_execution_policies ( + job_id, resource_class, timeout_ms, updated_at + ) VALUES (?, ?, ?, ?) + ON CONFLICT(job_id) DO UPDATE SET + resource_class = excluded.resource_class, + timeout_ms = excluded.timeout_ms, + updated_at = excluded.updated_at` + ) + .run(definition.id, resourceClass, timeoutMs, timestamp); return getScheduledJob(definition.id) as ScheduledJob; } export function listScheduledJobs(): ScheduledJob[] { const rows = database - .prepare("SELECT * FROM scheduled_jobs ORDER BY name COLLATE NOCASE, id") + .prepare( + `SELECT job.*, policy.resource_class, policy.timeout_ms + FROM scheduled_jobs job + LEFT JOIN scheduled_job_execution_policies policy ON policy.job_id = job.id + ORDER BY job.name COLLATE NOCASE, job.id` + ) .all() as unknown as ScheduledJobRow[]; const latestRuns = latestRunsByJobId(rows.map((row) => row.id)); return rows.map((row) => mapJob(row, latestRuns)); } export function getScheduledJob(id: string): ScheduledJob | undefined { - const row = database.prepare("SELECT * FROM scheduled_jobs WHERE id = ?").get(id) as - ScheduledJobRow | undefined; + const row = database + .prepare( + `SELECT job.*, policy.resource_class, policy.timeout_ms + FROM scheduled_jobs job + LEFT JOIN scheduled_job_execution_policies policy ON policy.job_id = job.id + WHERE job.id = ?` + ) + .get(id) as ScheduledJobRow | undefined; return row ? mapJob(row) : undefined; } @@ -624,10 +738,17 @@ export function listScheduledJobRuns(id: string, limit = 20): ScheduledJobRun[] return ( database .prepare( - `SELECT * - FROM scheduled_job_runs - WHERE job_id = ? - ORDER BY started_at DESC, id DESC + `SELECT run.*, + execution.id AS execution_id, + execution.queued_at AS execution_queued_at, + execution.resource_class AS execution_resource_class, + execution.cancel_requested_at AS execution_cancel_requested_at, + execution.cancellable AS execution_cancellable + FROM scheduled_job_runs run + LEFT JOIN job_executions execution + ON execution.scheduled_run_id = run.id + WHERE run.job_id = ? + ORDER BY run.started_at DESC, run.id DESC LIMIT ?` ) .all(id, normalizedLimit) as unknown as ScheduledJobRunRow[] @@ -724,231 +845,275 @@ export function updateScheduledJob( return getScheduledJob(id); } -function createRun(jobId: string, triggerType: ScheduledJobTriggerType): ScheduledJobRun { - const startedAt = nowIso(); +interface EnqueueScheduledJobOptions { + availableAt?: string; +} + +function scheduledRunById(id: number): ScheduledJobRun | undefined { + const row = database + .prepare( + `SELECT run.*, + execution.id AS execution_id, + execution.queued_at AS execution_queued_at, + execution.resource_class AS execution_resource_class, + execution.cancel_requested_at AS execution_cancel_requested_at, + execution.cancellable AS execution_cancellable + FROM scheduled_job_runs run + LEFT JOIN job_executions execution ON execution.scheduled_run_id = run.id + WHERE run.id = ?` + ) + .get(id) as ScheduledJobRunRow | undefined; + return mapRun(row); +} + +function insertScheduledRun( + jobId: string, + triggerType: ScheduledJobTriggerType, + status: "queued" | "running", + timestamp: string +): number { const result = database .prepare( `INSERT INTO scheduled_job_runs ( job_id, status, trigger_type, started_at, output_json - ) VALUES (?, 'running', ?, ?, '{}')` + ) VALUES (?, ?, ?, ?, '{}')` ) - .run(jobId, triggerType, startedAt); - return { - id: Number(result.lastInsertRowid), - jobId, - status: "running", - triggerType, - startedAt, - finishedAt: undefined, - message: undefined, - output: {}, - }; + .run(jobId, status, triggerType, timestamp); + return Number(result.lastInsertRowid); } -function finishRun( - run: ScheduledJobRun, - status: Exclude, - message: string | undefined, - output: Record -): ScheduledJobRun { - const finishedAt = nowIso(); - database - .prepare( - `UPDATE scheduled_job_runs - SET status = ?, finished_at = ?, message = ?, output_json = ? - WHERE id = ?` - ) - .run(status, finishedAt, sqlNullable(message), JSON.stringify(output), run.id); - return { ...run, status, finishedAt, message, output }; +function isActiveExecutionConflict(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("UNIQUE constraint failed: job_executions.scheduled_job_id") || + message.includes("idx_job_executions_active_scheduled_job") + ); } -function claimScheduledRun(job: ScheduledJob): ScheduledJobRun | undefined { - const currentJob = getScheduledJob(job.id); - const dueAt = nowIso(); - if (!currentJob?.enabled || !currentJob.nextRunAt || currentJob.nextRunAt > dueAt) { - return undefined; +function jobStatusError(message: string, statusCode: number): Error { + return Object.assign(new Error(message), { statusCode }); +} + +export function enqueueScheduledJob( + id: string, + triggerType: ScheduledJobTriggerType = "manual", + options: EnqueueScheduledJobOptions = {} +): ScheduledJobRun { + const job = getScheduledJob(id); + if (!job) throw jobStatusError("Scheduled job not found", 404); + if (triggerType === "startup" && !job.enabled) { + throw jobStatusError("Scheduled job is disabled", 409); } - const nextRunAt = calculateNextRunAt(currentJob); - const startedAt = nowIso(); + + const queuedAt = nowIso(); + database.run("BEGIN IMMEDIATE"); try { - database.run("BEGIN IMMEDIATE"); - const updateResult = database - .prepare( - `UPDATE scheduled_jobs - SET next_run_at = ?, updated_at = ? - WHERE id = ? AND enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ?` - ) - .run(sqlNullable(nextRunAt), startedAt, currentJob.id, dueAt); - if (updateResult.changes === 0) { - database.run("ROLLBACK"); - return undefined; + if (triggerType === "schedule") { + const nextRunAt = calculateNextRunAt(job, new Date(queuedAt)); + const update = database + .prepare( + `UPDATE scheduled_jobs + SET next_run_at = ?, updated_at = ? + WHERE id = ? AND enabled = 1 + AND next_run_at IS NOT NULL AND next_run_at <= ?` + ) + .run(sqlNullable(nextRunAt), queuedAt, job.id, queuedAt); + if (update.changes === 0) { + throw jobStatusError("Scheduled job is no longer due", 409); + } } - const insertResult = database - .prepare( - `INSERT INTO scheduled_job_runs ( - job_id, status, trigger_type, started_at, output_json - ) VALUES (?, 'running', 'schedule', ?, '{}')` - ) - .run(currentJob.id, startedAt); + + const runId = insertScheduledRun(job.id, triggerType, "queued", queuedAt); + insertJobExecution({ + actionKey: job.actionKey, + availableAt: options.availableAt, + displayName: job.name, + payload: job.actionPayload, + queuedAt, + resourceClass: job.resourceClass, + scheduledJobId: job.id, + scheduledRunId: runId, + timeoutMs: job.timeoutMs, + triggerType, + }); database.run("COMMIT"); - return { - id: Number(insertResult.lastInsertRowid), - jobId: currentJob.id, - status: "running", - triggerType: "schedule", - startedAt, - finishedAt: undefined, - message: undefined, - output: {}, - }; + return scheduledRunById(runId) as ScheduledJobRun; } catch (error) { try { database.run("ROLLBACK"); } catch { - // Ignore rollback failures after SQLite has already unwound the transaction. + // Preserve the enqueue error. + } + if (isActiveExecutionConflict(error)) { + throw jobStatusError("Scheduled job is already queued or running", 409); } throw error; } } -function markAbandonedRunningRuns(): void { - try { - database - .prepare( - `UPDATE scheduled_job_runs - SET status = 'failed', - finished_at = COALESCE(finished_at, ?), - message = COALESCE(message, ?) - WHERE status = 'running'` - ) - .run(nowIso(), "Scheduled job abandoned after backend restart"); - } catch (error) { - console.warn("[ScheduledJobs] Failed to mark abandoned scheduled runs:", error); - } -} - -function persistedRunFallback( - run: ScheduledJobRun, - status: Exclude, - message: string | undefined, - output: Record -): ScheduledJobRun { - return { ...run, status, finishedAt: nowIso(), message, output }; -} - -function finishRunOrReport( - run: ScheduledJobRun, - status: Exclude, - message: string | undefined, - output: Record -): ScheduledJobRun { - try { - return finishRun(run, status, message, output); - } catch (error) { - console.warn( - `[ScheduledJobs] Failed to persist ${status} scheduled job run:`, - error - ); - return persistedRunFallback( - run, - status, - errorMessage(error, `Scheduled job ${status} persistence failed`), - output - ); - } -} - -export function createManualScheduledJobRun(jobId: string): ScheduledJobRun { - if (runningJobs.has(jobId)) { - const error = new Error("Scheduled job is already running") as Error & { - statusCode?: number; - }; - error.statusCode = 409; - throw error; - } - runningJobs.add(jobId); - try { - return createRun(jobId, "manual"); - } catch (error) { - runningJobs.delete(jobId); - throw error; - } -} - -export function finishScheduledJobRun( - run: ScheduledJobRun, - status: Exclude, - message: string | undefined, - output: Record -): ScheduledJobRun { - try { - return finishRunOrReport(run, status, message, output); - } finally { - runningJobs.delete(run.jobId); - } -} - export async function runScheduledJob( id: string, triggerType: ScheduledJobTriggerType = "manual", signal?: AbortSignal ): Promise { const job = getScheduledJob(id); - if (!job) { - const error = new Error("Scheduled job not found") as Error & { - statusCode?: number; - }; - error.statusCode = 404; - throw error; - } - if (runningJobs.has(id)) { - const error = new Error("Scheduled job is already running") as Error & { - statusCode?: number; - }; - error.statusCode = 409; - throw error; - } + if (!job) throw jobStatusError("Scheduled job not found", 404); const action = actionHandlers.get(job.actionKey); if (!action && triggerType === "manual") { throw new ScheduledJobValidationError( `No scheduled job action registered for ${job.actionKey}` ); } - const timeoutMs = action?.timeoutMs ?? scheduledJobRunTimeoutMs; - - const run = - triggerType === "schedule" ? claimScheduledRun(job) : createRun(id, triggerType); - if (!run) { - const error = new Error("Scheduled job is no longer due") as Error & { - statusCode?: number; - }; - error.statusCode = 409; - throw error; + const run = enqueueScheduledJob(id, triggerType); + if (!run.executionId) throw jobStatusError("Scheduled job was not queued", 500); + await waitForJobExecution(run.executionId, { signal }); + return scheduledRunById(run.id) as ScheduledJobRun; +} + +async function executeClaimedJobExecution( + execution: JobExecution, + workerId: string, + signal?: AbortSignal +): Promise { + const currentJob = execution.scheduledJobId + ? getScheduledJob(execution.scheduledJobId) + : undefined; + if (!currentJob && execution.scheduledJobId) { + return finishJobExecution( + execution.id, + workerId, + "cancelled", + "Scheduled job was removed before execution", + {} + ); } - runningJobs.add(id); + if ( + currentJob && + !currentJob.enabled && + (execution.triggerType === "schedule" || execution.triggerType === "startup") + ) { + const finishedExecution = finishJobExecution( + execution.id, + workerId, + "cancelled", + "Scheduled job was disabled before execution", + {} + ); + return execution.scheduledRunId === undefined + ? finishedExecution + : (scheduledRunById(execution.scheduledRunId) as ScheduledJobRun); + } + const job: ScheduledJob = currentJob + ? { + ...currentJob, + actionKey: execution.actionKey, + actionPayload: execution.payload, + resourceClass: execution.resourceClass, + timeoutMs: execution.timeoutMs, + } + : { + actionKey: execution.actionKey, + actionPayload: execution.payload, + createdAt: execution.queuedAt, + cronExpression: undefined, + description: "", + disableIntent: undefined, + enabled: true, + id: execution.id, + intervalSeconds: 60, + isQueued: false, + isRunning: true, + lastRun: undefined, + name: execution.displayName, + nextRunAt: undefined, + resourceClass: execution.resourceClass, + scheduleType: "interval", + timeOfDay: undefined, + timeoutMs: execution.timeoutMs, + updatedAt: execution.startedAt ?? execution.queuedAt, + }; + const action = actionHandlers.get(execution.actionKey); + const controller = new AbortController(); + const abortFromSignal = () => controller.abort(); + signal?.addEventListener("abort", abortFromSignal, { once: true }); + if (signal?.aborted) controller.abort(); + const heartbeat = setInterval(() => { + try { + const lease = heartbeatJobExecution(execution.id, workerId); + if (!lease.hasLease || lease.cancelRequested) controller.abort(); + } catch (error) { + console.warn("[ScheduledJobs] Execution heartbeat failed:", error); + } + }, executorHeartbeatMs); + heartbeat.unref(); + + let status: "success" | "failed" = "success"; + let message: string | undefined; + let output: Record; try { - let output: Record; try { if (!action) { throw new ScheduledJobValidationError( - `No scheduled job action registered for ${job.actionKey}` + `No scheduled job action registered for ${execution.actionKey}` ); } - output = await runActionWithTimeout(timeoutMs, action, job, signal); + output = await withJobResourceClass(execution.resourceClass, () => + runActionWithTimeout( + execution.timeoutMs, + action, + job, + { + executionId: execution.id, + pauseWorkerClaims: pauseExecutorClaims, + protectFromCancellation: () => { + protectRunningJobExecutionFromCancellation(execution.id); + }, + updateOutput: (nextOutput) => { + updateJobExecutionOutput(execution.id, workerId, nextOutput); + }, + }, + controller.signal + ) + ); } catch (error) { - if (error instanceof ScheduledJobAbortError) { - await error.getHandlerSettled(); + if (error instanceof ScheduledJobInterruptionError) { + const didSettle = await waitForInterruptedHandler( + error.getHandlerSettled() + ); + if (!didSettle) { + console.warn( + "[ScheduledJobs] Interrupted action did not settle during cleanup grace", + { executionId: execution.id } + ); + } } - return finishRunOrReport( - run, - "failed", - errorMessage(error, "Scheduled job failed"), - error instanceof ScheduledJobActionError ? error.output : {} + status = "failed"; + message = errorMessage(error, "Scheduled job failed"); + const progressOutput = getJobExecution(execution.id)?.output ?? {}; + const statusCode = Number( + (error as { statusCode?: unknown } | undefined)?.statusCode ); + output = { + ...progressOutput, + ...(error instanceof ScheduledJobActionError && error.output), + ...(Number.isSafeInteger(statusCode) && + statusCode >= 400 && + statusCode < 600 && { statusCode }), + }; } - return finishRunOrReport(run, "success", undefined, output); + const finishedExecution = finishJobExecution( + execution.id, + workerId, + status, + message, + output + ); + return execution.scheduledRunId === undefined + ? finishedExecution + : (scheduledRunById(execution.scheduledRunId) as ScheduledJobRun); } finally { - runningJobs.delete(id); + clearInterval(heartbeat); + signal?.removeEventListener("abort", abortFromSignal); } } @@ -956,6 +1121,7 @@ async function runActionWithTimeout( timeoutMs: number, action: ScheduledJobActionRegistration, job: ScheduledJob, + context: ScheduledJobActionContext, signal?: AbortSignal ): Promise> { if (signal?.aborted) { @@ -964,40 +1130,27 @@ async function runActionWithTimeout( const controller = new AbortController(); const abortPromise = Promise.withResolvers(); let handlerSettled: Promise = Promise.resolve(); - const abortFromSignal = () => { + const interrupt = (message: string) => { + abortPromise.reject(new ScheduledJobInterruptionError(message, handlerSettled)); controller.abort(); - abortPromise.reject(new ScheduledJobAbortError(handlerSettled)); }; + const abortFromSignal = () => interrupt("Scheduled job aborted"); signal?.addEventListener("abort", abortFromSignal, { once: true }); - let isTimedOut = false; let timeout: NodeJS.Timeout | undefined; try { - const timeoutPromise = new Promise((_resolve, reject) => { - timeout = setTimeout(() => { - isTimedOut = true; - controller.abort(); - console.warn("[ScheduledJobs] Scheduled job exceeded timeout", { - timeoutMs, - }); - reject(new Error("Scheduled job timed out")); - }, timeoutMs); - }); + timeout = setTimeout(() => { + console.warn("[ScheduledJobs] Scheduled job exceeded timeout", { + timeoutMs, + }); + interrupt("Scheduled job timed out"); + }, timeoutMs); timeout?.unref(); - const handlerPromise = Promise.resolve(action.handler(job, controller.signal)); + const handlerPromise = Promise.resolve( + action.handler(job, controller.signal, context) + ); handlerSettled = suppressHandlerPromiseRejection(handlerPromise); - void suppressHandlerPromiseRejection(handlerPromise); - const output = - (await Promise.race([ - handlerPromise, - timeoutPromise, - abortPromise.promise, - ])) ?? {}; + const output = (await Promise.race([handlerPromise, abortPromise.promise])) ?? {}; return output; - } catch (error) { - if (isTimedOut) { - throw new Error("Scheduled job timed out", { cause: error }); - } - throw error; } finally { if (timeout) { clearTimeout(timeout); @@ -1006,11 +1159,6 @@ async function runActionWithTimeout( } } -function trackScheduledRun(run: Promise): void { - scheduledJobRuns.add(run); - void untrackScheduledRun(run); -} - async function suppressHandlerPromiseRejection( handlerPromise: Promise ): Promise { @@ -1021,11 +1169,20 @@ async function suppressHandlerPromiseRejection( } } -async function untrackScheduledRun(run: Promise): Promise { +async function waitForInterruptedHandler( + handlerSettled: Promise +): Promise { + const didSettle = async () => { + await handlerSettled; + return true; + }; + const timeout = Promise.withResolvers(); + const timer = setTimeout(() => timeout.resolve(false), interruptedHandlerGraceMs); + timer.unref(); try { - await run; + return await Promise.race([didSettle(), timeout.promise]); } finally { - scheduledJobRuns.delete(run); + clearTimeout(timer); } } @@ -1037,16 +1194,6 @@ function isStaleScheduledRunError(error: unknown): boolean { ); } -async function observeScheduledRun(id: string): Promise { - try { - await runScheduledJob(id, "schedule"); - } catch (error) { - if (!isStaleScheduledRunError(error)) { - console.warn("[ScheduledJobs] Scheduled run failed unexpectedly:", error); - } - } -} - async function runDueJobs(): Promise { const dueAt = nowIso(); const rows = database @@ -1057,26 +1204,80 @@ async function runDueJobs(): Promise { ) .all(dueAt) as Array<{ id: string }>; for (const row of rows) { - if (!runningJobs.has(row.id)) { - try { - const currentJob = getScheduledJob(row.id); - if ( - !currentJob?.enabled || - !currentJob.nextRunAt || - currentJob.nextRunAt > nowIso() - ) { - continue; - } - const run = observeScheduledRun(row.id); - trackScheduledRun(run); - } catch (error) { - console.warn( - "[ScheduledJobs] Failed to inspect due scheduled job:", - error - ); - // Keep later due jobs running even if a persisted row is stale. + try { + enqueueScheduledJob(row.id, "schedule"); + } catch (error) { + if (!isStaleScheduledRunError(error)) { + console.warn("[ScheduledJobs] Failed to queue due scheduled job:", error); } + // Keep later due jobs queueing even if a persisted row is stale. + } + } +} + +async function observeClaimedExecution( + execution: JobExecution, + controller: AbortController +): Promise { + try { + await executeClaimedJobExecution( + execution, + scheduledJobRuntimeState.workerId, + controller.signal + ); + } catch (error) { + console.warn("[ScheduledJobs] Queued execution failed unexpectedly:", error); + } finally { + activeExecutionControllers.delete(execution.id); + activeExecutionRuns.delete(execution.id); + queueMicrotask(executorTick); + } +} + +function pauseExecutorClaims(): () => void { + const generation = ++scheduledJobRuntimeState.executorClaimPauseGeneration; + scheduledJobRuntimeState.isExecutorClaimingPaused = true; + let isResumed = false; + return () => { + if (isResumed) return; + isResumed = true; + if (scheduledJobRuntimeState.executorClaimPauseGeneration !== generation) { + return; } + scheduledJobRuntimeState.isExecutorClaimingPaused = false; + queueMicrotask(executorTick); + }; +} + +function resetExecutorClaimPause(): void { + scheduledJobRuntimeState.executorClaimPauseGeneration += 1; + scheduledJobRuntimeState.isExecutorClaimingPaused = false; +} + +function executorTick(): void { + if ( + !scheduledJobRuntimeState.executor || + scheduledJobRuntimeState.isExecutorClaimingPaused || + scheduledJobRuntimeState.isExecutorTickRunning || + activeExecutionRuns.size >= executorCapacity + ) { + return; + } + scheduledJobRuntimeState.isExecutorTickRunning = true; + try { + const execution = claimNextJobExecution( + scheduledJobRuntimeState.workerId, + executorCapacity + ); + if (!execution) return; + const controller = new AbortController(); + activeExecutionControllers.set(execution.id, controller); + const run = observeClaimedExecution(execution, controller); + activeExecutionRuns.set(execution.id, run); + } catch (error) { + console.warn("[ScheduledJobs] Executor tick failed:", error); + } finally { + scheduledJobRuntimeState.isExecutorTickRunning = false; } } @@ -1100,12 +1301,58 @@ export function startScheduledJobScheduler(): void { if (scheduledJobRuntimeState.scheduler) { return; } - markAbandonedRunningRuns(); scheduledJobRuntimeState.scheduler = setInterval(scheduleTick, schedulerTickMs); scheduledJobRuntimeState.scheduler.unref(); scheduleTick(); } +export function startScheduledJobExecutor(): void { + if (scheduledJobRuntimeState.executor) return; + resetExecutorClaimPause(); + const timestamp = nowIso(); + const recoveredLegacyRuns = recoverOrphanedScheduledJobRuns(timestamp); + if (recoveredLegacyRuns > 0) { + console.warn("[ScheduledJobs] Recovered orphaned scheduled job runs", { + recovered: recoveredLegacyRuns, + }); + } + const recovered = recoverExpiredJobExecutions(timestamp); + if (recovered > 0) { + console.warn("[ScheduledJobs] Recovered expired job execution leases", { + recovered, + }); + } + registerJobWorker(scheduledJobRuntimeState.workerId, executorCapacity); + scheduledJobRuntimeState.workerHeartbeat = setInterval(() => { + try { + didHeartbeatJobWorker(scheduledJobRuntimeState.workerId); + } catch (error) { + console.warn("[ScheduledJobs] Worker heartbeat failed:", error); + } + }, executorHeartbeatMs); + scheduledJobRuntimeState.workerHeartbeat.unref(); + scheduledJobRuntimeState.executor = setInterval(executorTick, executorTickMs); + scheduledJobRuntimeState.executor.unref(); + executorTick(); +} + +export function recoverOrphanedScheduledJobRuns(timestamp = nowIso()): number { + return database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'failed', finished_at = ?, + message = 'Scheduled job interrupted before worker lease recovery' + WHERE status = 'running' + AND NOT EXISTS ( + SELECT 1 + FROM job_executions + WHERE scheduled_run_id = scheduled_job_runs.id + AND status IN ('queued', 'running') + )` + ) + .run(timestamp).changes; +} + export function stopScheduledJobScheduler(): void { if (!scheduledJobRuntimeState.scheduler) { return; @@ -1113,3 +1360,22 @@ export function stopScheduledJobScheduler(): void { clearInterval(scheduledJobRuntimeState.scheduler); scheduledJobRuntimeState.scheduler = undefined; } + +export async function stopScheduledJobExecutor(): Promise { + if (scheduledJobRuntimeState.executor) { + clearInterval(scheduledJobRuntimeState.executor); + scheduledJobRuntimeState.executor = undefined; + } + if (scheduledJobRuntimeState.workerHeartbeat) { + clearInterval(scheduledJobRuntimeState.workerHeartbeat); + scheduledJobRuntimeState.workerHeartbeat = undefined; + } + for (const controller of activeExecutionControllers.values()) { + controller.abort(); + } + await Promise.allSettled(activeExecutionRuns.values()); + activeExecutionControllers.clear(); + activeExecutionRuns.clear(); + unregisterJobWorker(scheduledJobRuntimeState.workerId); + resetExecutorClaimPause(); +} diff --git a/backend/src/workerStart.ts b/backend/src/workerStart.ts new file mode 100644 index 000000000..64c221488 --- /dev/null +++ b/backend/src/workerStart.ts @@ -0,0 +1,44 @@ +import { startDashboardJobWorker, stopDashboardJobWorker } from "./services/jobWorker.ts"; + +const WORKER_KEEP_ALIVE_INTERVAL_MS = 60_000; + +export function isDirectWorkerEntrypoint( + argvPath = process.argv[1], + moduleUrl = import.meta.url +): boolean { + return Boolean(argvPath && moduleUrl === Bun.pathToFileURL(argvPath).href); +} + +/** Keeps the dedicated worker process referenced while its runtime timers are idle. */ +export function createWorkerKeepAliveHandle(): NodeJS.Timeout { + return setInterval(() => 0, WORKER_KEEP_ALIVE_INTERVAL_MS); +} + +export async function runDashboardWorker(): Promise { + const shutdown = Promise.withResolvers(); + const stop = (signal: NodeJS.Signals) => shutdown.resolve(signal); + const keepAlive = createWorkerKeepAliveHandle(); + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + try { + startDashboardJobWorker(); + await shutdown.promise; + } finally { + process.removeListener("SIGINT", stop); + process.removeListener("SIGTERM", stop); + try { + await stopDashboardJobWorker(); + } finally { + clearInterval(keepAlive); + } + } +} + +if (isDirectWorkerEntrypoint()) { + try { + await runDashboardWorker(); + } catch (error) { + console.error("[Worker] Failed:", error); + process.exitCode = 1; + } +} diff --git a/backend/test/bunNativeServerBehavior.test.ts b/backend/test/bunNativeServerBehavior.test.ts index 356ad1cc4..5471d2f23 100644 --- a/backend/test/bunNativeServerBehavior.test.ts +++ b/backend/test/bunNativeServerBehavior.test.ts @@ -88,14 +88,28 @@ describe("Bun-native dashboard backend", () => { ); const serverScript = path.join(state.temporaryRoot, "native-server.ts"); const serverModulePath = path.resolve(import.meta.dirname, "../src/server.ts"); + const dockerActionsModulePath = path.resolve( + import.meta.dirname, + "../src/services/dockerActions.ts" + ); + const scheduledJobsModulePath = path.resolve( + import.meta.dirname, + "../src/services/scheduledJobs.ts" + ); const serverModuleUrl = pathToFileURL(serverModulePath).href; + const dockerActionsModuleUrl = pathToFileURL(dockerActionsModulePath).href; + const scheduledJobsModuleUrl = pathToFileURL(scheduledJobsModulePath).href; await fs.writeFile( serverScript, [ `import { createServer } from ${JSON.stringify(serverModuleUrl)};`, + `import { registerDockerExecutionActions } from ${JSON.stringify(dockerActionsModuleUrl)};`, + `import { startScheduledJobExecutor, stopScheduledJobExecutor } from ${JSON.stringify(scheduledJobsModuleUrl)};`, + "registerDockerExecutionActions();", + "startScheduledJobExecutor();", "const server = createServer(0);", "console.log(JSON.stringify({ port: server.port }));", - "process.on('SIGTERM', () => { server.stop(true).then(() => process.exit(0)); });", + "process.on('SIGTERM', () => { Promise.all([server.stop(true), stopScheduledJobExecutor()]).then(() => process.exit(0)).catch(() => process.exit(1)); });", ].join("\n") ); diff --git a/backend/test/dockerUpdater.test.ts b/backend/test/dockerUpdater.test.ts index 1197dbf3c..bc08e0774 100644 --- a/backend/test/dockerUpdater.test.ts +++ b/backend/test/dockerUpdater.test.ts @@ -272,6 +272,62 @@ describe("Docker updater tag patterns", () => { expect(runProcessSpy).not.toHaveBeenCalled(); }); + it("protects a pending updater git push when no service apply is needed", async () => { + rememberEnvironment("MIRA_DOCKER_APPS_ROOT"); + const appsRoot = createTemporaryRoot("mira-docker-updater-pending-sync-"); + process.env.MIRA_DOCKER_APPS_ROOT = appsRoot; + const mutationEvents: string[] = []; + const runProcessSpy = jest + .spyOn(processModule, "runProcess") + .mockImplementation((async (file, arguments_) => { + const command = arguments_.join(" "); + if (file === "git" && command === "rev-parse --show-toplevel") { + return { code: 0, stderr: "", stdout: `${appsRoot}\n` }; + } + if ( + file === "git" && + command === "rev-parse --abbrev-ref --symbolic-full-name @{u}" + ) { + return { code: 0, stderr: "", stdout: "origin/main\n" }; + } + if (file === "git" && command === "log --format=%s origin/main..HEAD") { + return { + code: 0, + stderr: "", + stdout: "chore: update managed app images\n", + }; + } + if (file === "git" && command === "push origin HEAD:refs/heads/main") { + mutationEvents.push("push"); + return { code: 0, stderr: "", stdout: "" }; + } + if (file === "git" && command === "rev-parse --short HEAD") { + return { code: 0, stderr: "", stdout: "abc1234\n" }; + } + return { code: 0, stderr: "", stdout: "" }; + }) as typeof processModule.runProcess); + cleanupCallbacks.push(() => runProcessSpy.mockRestore()); + const protectFromCancellation = jest.fn(() => { + mutationEvents.push("protect"); + }); + + const steps = await runDockerUpdaterService( + undefined, + undefined, + protectFromCancellation + ); + + expect(protectFromCancellation).toHaveBeenCalledTimes(1); + expect(mutationEvents).toEqual(["protect", "push"]); + expect(steps).toContainEqual( + expect.objectContaining({ + isOk: true, + step: "git-sync:docker", + stdout: expect.stringContaining('"pushed":true'), + }) + ); + }); + it("registers partial compose discoveries as nonblocking warnings", async () => { rememberEnvironment("MIRA_DOCKER_APPS_ROOT"); rememberEnvironment("MIRA_DOCKER_UPDATER_SKIP_REGISTRY"); @@ -340,6 +396,238 @@ describe("Docker updater tag patterns", () => { expect(runProcessSpy).not.toHaveBeenCalled(); }); + it("aborts an in-flight registry request through the worker signal", async () => { + rememberEnvironment("MIRA_DOCKER_APPS_ROOT"); + rememberEnvironment("MIRA_DOCKER_UPDATER_SKIP_REGISTRY"); + const appsRoot = createTemporaryRoot("mira-docker-updater-abort-"); + const appRoot = path.join(appsRoot, "unit-abort-app"); + mkdirSync(appRoot, { recursive: true }); + writeFileSync( + path.join(appRoot, "compose.yaml"), + [ + "services:", + " web:", + " image: ghcr.io/unit/abort:1.0.0", + " labels:", + " mira.updater.enabled: 'true'", + " mira.updater.autoUpdate: 'false'", + " mira.updater.track: tag", + String.raw` mira.updater.tagPattern: '^\d+\.\d+\.\d+$'`, + " mira.updater.tagPatternIsRegex: 'true'", + "", + ].join("\n") + ); + process.env.MIRA_DOCKER_APPS_ROOT = appsRoot; + delete process.env.MIRA_DOCKER_UPDATER_SKIP_REGISTRY; + + const registered = await registerDockerUpdaterServices(); + expect(registered.isOk).toBe(true); + const service = database + .prepare( + `SELECT id + FROM docker_managed_services + WHERE app_slug = 'unit-abort-app' AND service_name = 'web'` + ) + .get() as { id: number }; + + const requestStarted = Promise.withResolvers(); + const fetchSpy = jest.spyOn(globalThis, "fetch").mockImplementation((async ( + _input: Request | string | URL, + init?: RequestInit + ) => { + const signal = init?.signal; + if (!signal) throw new Error("Registry request signal was missing"); + requestStarted.resolve(); + return await new Promise((_resolve, reject) => { + const rejectForAbort = () => reject(signal.reason); + signal.addEventListener("abort", rejectForAbort, { once: true }); + if (signal.aborted) rejectForAbort(); + }); + }) as typeof fetch); + cleanupCallbacks.push(() => fetchSpy.mockRestore()); + + const controller = new AbortController(); + const run = runDockerUpdaterService(service.id, controller.signal); + await requestStarted.promise; + controller.abort(new DOMException("Updater cancelled", "AbortError")); + + await expect(run).rejects.toMatchObject({ + message: "Updater cancelled", + name: "AbortError", + }); + }); + + it("aborts an update waiting for another service in the same Compose project", async () => { + rememberEnvironment("MIRA_DOCKER_APPS_ROOT"); + rememberEnvironment("MIRA_DOCKER_BIN"); + rememberEnvironment("MIRA_DOCKER_COMPOSE_WRAPPER"); + rememberEnvironment("MIRA_DOCKER_UPDATER_SKIP_REGISTRY"); + rememberEnvironment("MIRA_DOCKER_UPDATER_PLATFORM"); + const appsRoot = createTemporaryRoot("mira-docker-updater-lock-abort-"); + const appRoot = path.join(appsRoot, "unit-lock-abort-app"); + const composePath = path.join(appRoot, "compose.yaml"); + const composeWrapper = path.join(appsRoot, "compose-wrapper"); + mkdirSync(appRoot, { recursive: true }); + writeFileSync( + composePath, + [ + "services:", + " first:", + " image: ghcr.io/unit/lock-first:1.0.0", + " labels:", + " mira.updater.enabled: 'true'", + " mira.updater.autoUpdate: 'false'", + " mira.updater.track: tag", + " mira.updater.tagPattern: 1.0.1", + " mira.updater.tagPatternIsRegex: 'false'", + " second:", + " image: ghcr.io/unit/lock-second:1.0.0", + " labels:", + " mira.updater.enabled: 'true'", + " mira.updater.autoUpdate: 'false'", + " mira.updater.track: tag", + " mira.updater.tagPattern: 1.0.1", + " mira.updater.tagPatternIsRegex: 'false'", + "", + ].join("\n") + ); + process.env.MIRA_DOCKER_APPS_ROOT = appsRoot; + process.env.MIRA_DOCKER_BIN = "docker"; + process.env.MIRA_DOCKER_COMPOSE_WRAPPER = composeWrapper; + delete process.env.MIRA_DOCKER_UPDATER_SKIP_REGISTRY; + process.env.MIRA_DOCKER_UPDATER_PLATFORM = "linux/amd64"; + + const fetchSpy = jest.spyOn(globalThis, "fetch").mockImplementation((async ( + input: Request | string | URL + ) => { + const url = String(input); + if (url.endsWith("/v2/unit/lock-first/manifests/1.0.1")) { + return Response.json( + { digest: "sha256:lock-first-101" }, + { + headers: { + "docker-content-digest": "sha256:lock-first-101", + }, + } + ); + } + if (url.endsWith("/v2/unit/lock-second/manifests/1.0.1")) { + return Response.json( + { digest: "sha256:lock-second-101" }, + { + headers: { + "docker-content-digest": "sha256:lock-second-101", + }, + } + ); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch); + cleanupCallbacks.push(() => fetchSpy.mockRestore()); + + const composeStarted = Promise.withResolvers(); + const composeRelease = + Promise.withResolvers>>(); + let composeCalls = 0; + const runProcessSpy = jest + .spyOn(processModule, "runProcess") + .mockImplementation(async (command) => { + if (command === composeWrapper) { + composeCalls += 1; + if (composeCalls > 1) { + throw new Error("The aborted waiter reached Docker Compose"); + } + composeStarted.resolve(); + return await composeRelease.promise; + } + return { code: 0, stderr: "", stdout: "" }; + }); + cleanupCallbacks.push(() => runProcessSpy.mockRestore()); + + const registered = await registerDockerUpdaterServices(); + expect(registered.isOk).toBe(true); + const services = database + .prepare( + `SELECT id, service_name + FROM docker_managed_services + WHERE app_slug = 'unit-lock-abort-app' + ORDER BY service_name` + ) + .all() as Array<{ id: number; service_name: string }>; + const firstService = services.find((service) => service.service_name === "first"); + const secondService = services.find( + (service) => service.service_name === "second" + ); + expect(firstService).toBeDefined(); + expect(secondService).toBeDefined(); + + const firstRun = runDockerUpdaterService(firstService?.id); + try { + await composeStarted.promise; + + const controller = new AbortController(); + const addAbortListenerSpy = jest.spyOn(controller.signal, "addEventListener"); + const secondRun = runDockerUpdaterService( + secondService?.id, + controller.signal + ); + async function observeSecondRun() { + try { + return { + status: "resolved" as const, + steps: await secondRun, + }; + } catch (error) { + return { error, status: "rejected" as const }; + } + } + async function observeTimeout() { + await Bun.sleep(500); + return { status: "timed-out" as const }; + } + const secondOutcome = observeSecondRun(); + + for (let attempt = 0; attempt < 100; attempt += 1) { + const abortListeners = addAbortListenerSpy.mock.calls.filter( + ([event]) => event === "abort" + ); + if (abortListeners.length >= 2) break; + await Bun.sleep(5); + } + expect( + addAbortListenerSpy.mock.calls.filter(([event]) => event === "abort") + ).toHaveLength(2); + expect(composeCalls).toBe(1); + + controller.abort(new DOMException("Lock waiter cancelled", "AbortError")); + const outcome = await Promise.race([secondOutcome, observeTimeout()]); + + expect(outcome).toMatchObject({ + error: { + message: "Lock waiter cancelled", + name: "AbortError", + }, + status: "rejected", + }); + expect(composeCalls).toBe(1); + + composeRelease.resolve({ code: 0, stderr: "", stdout: "compose ok" }); + await expect(firstRun).resolves.toContainEqual( + expect.objectContaining({ + isOk: true, + step: "manual-update:unit-lock-abort-app/first", + }) + ); + } finally { + composeRelease.resolve({ code: 0, stderr: "", stdout: "compose ok" }); + try { + await firstRun; + } catch { + // Preserve the original assertion failure after releasing the lock. + } + } + }); + it("applies a manual update to an isolated Compose file without invoking real Docker", async () => { rememberEnvironment("MIRA_DOCKER_APPS_ROOT"); rememberEnvironment("MIRA_DOCKER_BIN"); @@ -405,7 +693,13 @@ describe("Docker updater tag patterns", () => { ) .get() as { id: number }; - const steps = await runDockerUpdaterService(service.id); + const protectFromCancellation = jest.fn(); + const steps = await runDockerUpdaterService( + service.id, + undefined, + protectFromCancellation + ); + expect(protectFromCancellation).toHaveBeenCalledTimes(1); expect(steps).toContainEqual( expect.objectContaining({ isOk: true, diff --git a/backend/test/gitHygiene.test.ts b/backend/test/gitHygiene.test.ts index 430d03f30..ec3ef7a5e 100644 --- a/backend/test/gitHygiene.test.ts +++ b/backend/test/gitHygiene.test.ts @@ -47,11 +47,14 @@ describe("git hygiene automation", () => { rememberEnvironment("MIRA_OPENCLAW_ROOT"); process.env.MIRA_OPENCLAW_ROOT = createTemporaryRoot("mira-openclaw-sync-"); const calls: Array = []; + const mutationEvents: string[] = []; const runProcessSpy = jest .spyOn(processModule, "runProcess") .mockImplementation((async (_file, arguments_) => { calls.push(arguments_); const command = arguments_.join(" "); + if (command.startsWith("commit ")) mutationEvents.push("commit"); + if (command.startsWith("push ")) mutationEvents.push("push"); if (command === "status --porcelain=v1 -z -uall") { return { code: 0, @@ -84,12 +87,19 @@ describe("git hygiene automation", () => { return { code: 0, stderr: "", stdout: "" }; }) as typeof processModule.runProcess); cleanupCallbacks.push(() => runProcessSpy.mockRestore()); + const protectFromCancellation = jest.fn(() => { + mutationEvents.push("protect"); + }); - await expect(syncOpenClawWorkspaceSafePaths()).resolves.toEqual({ + await expect( + syncOpenClawWorkspaceSafePaths(undefined, protectFromCancellation) + ).resolves.toEqual({ changedPaths: ["workspace/MEMORY.md", "workspace/memory/2026-07-07.md"], commit: "abc1234", pushed: true, }); + expect(protectFromCancellation).toHaveBeenCalledTimes(1); + expect(mutationEvents).toEqual(["protect", "commit", "push"]); expect(calls).toEqual( expect.arrayContaining([ [ @@ -357,11 +367,13 @@ describe("git hygiene automation", () => { rememberEnvironment("MIRA_OPENCLAW_ROOT"); process.env.MIRA_OPENCLAW_ROOT = createTemporaryRoot("mira-openclaw-push-retry-"); const calls: Array = []; + const mutationEvents: string[] = []; const runProcessSpy = jest .spyOn(processModule, "runProcess") .mockImplementation((async (_file, arguments_) => { calls.push(arguments_); const command = arguments_.join(" "); + if (command.startsWith("push ")) mutationEvents.push("push"); if (command === "status --porcelain=v1 -z -uall") { return { code: 0, stderr: "", stdout: "" }; } @@ -389,12 +401,19 @@ describe("git hygiene automation", () => { return { code: 0, stderr: "", stdout: "" }; }) as typeof processModule.runProcess); cleanupCallbacks.push(() => runProcessSpy.mockRestore()); + const protectFromCancellation = jest.fn(() => { + mutationEvents.push("protect"); + }); - await expect(syncOpenClawWorkspaceSafePaths()).resolves.toEqual({ + await expect( + syncOpenClawWorkspaceSafePaths(undefined, protectFromCancellation) + ).resolves.toEqual({ changedPaths: [], commit: "abc1234", pushed: true, }); + expect(protectFromCancellation).toHaveBeenCalledTimes(1); + expect(mutationEvents).toEqual(["protect", "push"]); expect(calls).toEqual( expect.arrayContaining([ ["status", "--porcelain=v1", "-z", "-uall"], diff --git a/backend/test/httpApiBehavior.test.ts b/backend/test/httpApiBehavior.test.ts index 01470f253..9edced4ba 100644 --- a/backend/test/httpApiBehavior.test.ts +++ b/backend/test/httpApiBehavior.test.ts @@ -203,6 +203,21 @@ async function createTestServer( throw lastError; } +async function withTestScheduledExecutor( + registerActions: () => void, + operation: () => Promise +): Promise { + const { startScheduledJobExecutor, stopScheduledJobExecutor } = + await import("../src/services/scheduledJobs.ts"); + registerActions(); + startScheduledJobExecutor(); + try { + return await operation(); + } finally { + await stopScheduledJobExecutor(); + } +} + describe("Mira Dashboard backend integration", () => { beforeAll(async () => { testState.originalEnv = Object.fromEntries( @@ -282,10 +297,15 @@ describe("Mira Dashboard backend integration", () => { }); it("reports health and auth bootstrap state without production data", async () => { - const health = await api<{ status: string; sessionCount: number }>("/api/health"); + const health = await api<{ + status: string; + sessionCount: number; + workerOnline: boolean; + }>("/api/health"); expect(health.status).toBe(200); expect(health.body.status).toBe("isOk"); expect(health.body.sessionCount).toBe(0); + expect(health.body.workerOnline).toBe(false); const bootstrap = await api<{ isBootstrapRequired: boolean; @@ -1361,12 +1381,18 @@ describe("Mira Dashboard backend integration", () => { expect(directExec.status).toBe(400); expect(directExec.body.error).toBe("command executable is not approved"); - const shellExec = await api<{ code: number; stderr: string; stdout: string }>( - "/api/exec", - json("POST", { - command: "__mira_dashboard_shell_smoke_test__", - shell: true, - }) + const { registerExecExecutionActions } = + await import("../src/services/execJobs.ts"); + const shellExec = await withTestScheduledExecutor( + registerExecExecutionActions, + () => + api<{ code: number; stderr: string; stdout: string }>( + "/api/exec", + json("POST", { + command: "__mira_dashboard_shell_smoke_test__", + shell: true, + }) + ) ); expect(shellExec.status).toBe(200); expect(shellExec.body.code).not.toBe(0); @@ -1496,23 +1522,66 @@ describe("Mira Dashboard backend integration", () => { const run = await api<{ isOk: boolean; run: { + executionId: string; jobId: string; output: Record; status: string; triggerType: string; }; }>("/api/jobs/functional.test.job/run", { method: "POST" }); - expect(run.status).toBe(200); + expect(run.status).toBe(202); expect(run.body).toMatchObject({ isOk: true, run: { jobId: "functional.test.job", - output: { result: "ran from integration test" }, - status: "success", + output: {}, + status: "queued", triggerType: "manual", }, }); + const queue = await api<{ + executions: Array<{ id: string; scheduledJobId?: string; status: string }>; + summary: { queued: number; running: number }; + }>("/api/job-executions"); + expect(queue.status).toBe(200); + expect(queue.body.summary).toMatchObject({ queued: 1, running: 0 }); + expect(queue.body.executions).toContainEqual( + expect.objectContaining({ + id: run.body.run.executionId, + scheduledJobId: "functional.test.job", + status: "queued", + }) + ); + + const detail = await api<{ + execution: { id: string; output: Record; status: string }; + }>(`/api/job-executions/${run.body.run.executionId}`); + expect(detail.status).toBe(200); + expect(detail.body.execution).toMatchObject({ + id: run.body.run.executionId, + output: {}, + status: "queued", + }); + + const invalidExecutionId = await api<{ error: string }>( + "/api/job-executions/not-a-uuid" + ); + expect(invalidExecutionId.status).toBe(400); + expect(invalidExecutionId.body.error).toBe("Invalid job execution id"); + + const cancelled = await api<{ + execution: { id: string; status: string }; + isOk: boolean; + }>(`/api/job-executions/${run.body.run.executionId}/cancel`, { + method: "POST", + }); + expect(cancelled.status).toBe(200); + expect(cancelled.body).toMatchObject({ + execution: { id: run.body.run.executionId, status: "cancelled" }, + isOk: true, + }); + const runs = await api<{ runs: Array<{ jobId: string; status: string; triggerType: string }>; }>("/api/jobs/functional.test.job/runs"); @@ -1520,7 +1589,7 @@ describe("Mira Dashboard backend integration", () => { expect(runs.body.runs).toContainEqual( expect.objectContaining({ jobId: "functional.test.job", - status: "success", + status: "cancelled", triggerType: "manual", }) ); @@ -1709,9 +1778,14 @@ describe("Mira Dashboard backend integration", () => { expect(walg.status).toBe(200); expect(walg.body).toEqual({}); - const clearKopia = await api<{ error: string }>( - "/api/backups/kopia/clear-needs-attention", - { method: "POST" } + const { registerBackupScheduledJobs } = + await import("../src/services/backups.ts"); + const clearKopia = await withTestScheduledExecutor( + registerBackupScheduledJobs, + () => + api<{ error: string }>("/api/backups/kopia/clear-needs-attention", { + method: "POST", + }) ); expect(clearKopia.status).toBe(404); expect(clearKopia.body.error).toBe("KOPIA backup job not found"); @@ -2262,9 +2336,15 @@ describe("Mira Dashboard backend integration", () => { }); it("allows valid dotted Docker Compose service names", async () => { - const result = await api<{ output: string }>( - "/api/docker/stack/action", - json("POST", { action: "restart", service: "api.v1" }) + const { registerDockerExecutionActions } = + await import("../src/services/dockerActions.ts"); + const result = await withTestScheduledExecutor( + registerDockerExecutionActions, + () => + api<{ output: string }>( + "/api/docker/stack/action", + json("POST", { action: "restart", service: "api.v1" }) + ) ); expect(result.status).toBe(200); diff --git a/backend/test/jobExecutionQueue.test.ts b/backend/test/jobExecutionQueue.test.ts new file mode 100644 index 000000000..5ea5322f8 --- /dev/null +++ b/backend/test/jobExecutionQueue.test.ts @@ -0,0 +1,387 @@ +import { afterEach, describe, expect, it } from "bun:test"; + +import { database } from "../src/database.ts"; +import { + scopedJobProcessCommand, + withJobResourceClass, +} from "../src/lib/jobResources.ts"; +import { + cancelJobExecution, + claimNextJobExecution, + didHeartbeatJobWorker, + enqueueJobExecution, + finishJobExecution, + getJobExecution, + getJobExecutionSummary, + insertJobExecution, + protectRunningJobExecutionFromCancellation, + recoverExpiredJobExecutions, + registerJobWorker, + unregisterJobWorker, +} from "../src/services/jobExecutionQueue.ts"; +import { waitForJobExecution } from "../src/services/queuedJobExecution.ts"; +import { + enqueueScheduledJob, + recoverOrphanedScheduledJobRuns, + registerScheduledJobAction, + removeScheduledJobsNotInAction, + ScheduledJobActionError, + startScheduledJobExecutor, + stopScheduledJobExecutor, + updateScheduledJob, + upsertScheduledJob, +} from "../src/services/scheduledJobs.ts"; + +const testJobIds = new Set(); +const testExecutionIds = new Set(); + +afterEach(async () => { + await stopScheduledJobExecutor(); + for (const executionId of testExecutionIds) { + database.prepare("DELETE FROM job_executions WHERE id = ?").run(executionId); + } + for (const jobId of testJobIds) { + database.prepare("DELETE FROM scheduled_job_runs WHERE job_id = ?").run(jobId); + database.prepare("DELETE FROM scheduled_jobs WHERE id = ?").run(jobId); + } + testExecutionIds.clear(); + testJobIds.clear(); +}); + +function createScheduledTestJob( + resourceClass: "host-heavy" | "interactive", + name: string +): string { + const id = `test-queue-${Bun.randomUUIDv7()}`; + testJobIds.add(id); + upsertScheduledJob({ + actionKey: `test.queue-${Bun.randomUUIDv7()}`, + actionPayload: { name }, + id, + intervalSeconds: 3600, + name, + resourceClass, + scheduleType: "interval", + }); + return id; +} + +describe("persistent job execution queue", () => { + it("persists worker progress and structured action failures", async () => { + const actionKey = `test.worker-${Bun.randomUUIDv7()}`; + registerScheduledJobAction(actionKey, async (_job, _signal, context) => { + context.updateOutput({ phase: "streaming" }); + throw new ScheduledJobActionError("expected failure", { isOk: false }); + }); + const queued = enqueueJobExecution({ + actionKey, + displayName: "Structured worker failure", + resourceClass: "host-heavy", + timeoutMs: 60_000, + }); + testExecutionIds.add(queued.id); + startScheduledJobExecutor(); + const execution = await waitForJobExecution(queued.id, { + pollIntervalMs: 10, + timeoutMs: 5000, + }); + expect(execution).toMatchObject({ + message: "expected failure", + output: { isOk: false }, + status: "failed", + }); + }); + + it("cancels queued synchronous work when its observer times out", async () => { + const queued = enqueueJobExecution({ + actionKey: `test.wait-timeout-${Bun.randomUUIDv7()}`, + displayName: "Timed out synchronous wait", + resourceClass: "network", + timeoutMs: 60_000, + }); + testExecutionIds.add(queued.id); + + await expect( + waitForJobExecution(queued.id, { + pollIntervalMs: 10, + timeoutMs: 0, + }) + ).rejects.toMatchObject({ + executionId: queued.id, + statusCode: 504, + }); + expect(getJobExecution(queued.id)).toMatchObject({ + message: "Job cancelled before execution", + status: "cancelled", + }); + }); + + it("reports only fresh worker heartbeats as online", () => { + const workerId = `test-worker-${Bun.randomUUIDv7()}`; + registerJobWorker(workerId, 1, "2026-07-22T10:00:00.000Z"); + expect( + getJobExecutionSummary(Date.parse("2026-07-22T10:00:20.000Z")) + ).toMatchObject({ + workerCapacity: 1, + workerCount: 1, + workerOnline: true, + }); + expect( + getJobExecutionSummary(Date.parse("2026-07-22T10:01:00.000Z")) + ).toMatchObject({ + workerCapacity: 0, + workerCount: 0, + workerOnline: false, + }); + expect(didHeartbeatJobWorker(workerId, "2026-07-22T10:01:00.000Z")).toBe(true); + unregisterJobWorker(workerId); + }); + + it("wraps worker children in class-specific systemd scopes", () => { + const command = withJobResourceClass("host-heavy", () => + scopedJobProcessCommand( + "docker", + ["exec", "worker", "sh", "-c", 'printf "%s" "$JOB_COMMAND"'], + { + MIRA_DASHBOARD_ENABLE_JOB_SCOPES: "1", + MIRA_DASHBOARD_JOB_SCOPE_OWNER: "mira-dashboard-worker.service", + } + ) + ); + + expect(command.executable).toBe("systemd-run"); + expect(command.arguments).toEqual( + expect.arrayContaining([ + "--scope", + "--expand-environment=no", + "--nice=15", + "CPUWeight=15", + "IOWeight=15", + "MemoryHigh=2G", + "MemoryMax=4G", + "TasksMax=128", + "RuntimeMaxSec=7h", + "BindsTo=mira-dashboard-worker.service", + "docker", + "exec", + "worker", + "sh", + "-c", + 'printf "%s" "$JOB_COMMAND"', + ]) + ); + }); + + it("allows queued cancellation but protects a running mutation", () => { + const queued = enqueueJobExecution({ + actionKey: `test.protected-${Bun.randomUUIDv7()}`, + displayName: "Protected mutation", + resourceClass: "exclusive", + timeoutMs: 60_000, + }); + testExecutionIds.add(queued.id); + const workerId = `test-worker-${Bun.randomUUIDv7()}`; + const running = claimNextJobExecution(workerId, 1); + expect(running?.id).toBe(queued.id); + + expect(protectRunningJobExecutionFromCancellation(queued.id)).toMatchObject({ + cancellable: false, + status: "running", + }); + expect(() => cancelJobExecution(queued.id)).toThrow( + "This job execution cannot be cancelled here" + ); + finishJobExecution(queued.id, workerId, "success", undefined, {}); + + const cancellableWhileQueued = enqueueJobExecution({ + actionKey: `test.queued-cancellation-${Bun.randomUUIDv7()}`, + displayName: "Queued cancellation", + resourceClass: "exclusive", + timeoutMs: 60_000, + }); + testExecutionIds.add(cancellableWhileQueued.id); + expect(cancelJobExecution(cancellableWhileQueued.id)).toMatchObject({ + status: "cancelled", + }); + }); + + it("prioritizes interactive work and enforces global capacity", () => { + const heavyJobId = createScheduledTestJob("host-heavy", "Heavy test job"); + const interactiveJobId = createScheduledTestJob( + "interactive", + "Interactive test job" + ); + const heavyRun = enqueueScheduledJob(heavyJobId); + const interactiveRun = enqueueScheduledJob(interactiveJobId); + testExecutionIds.add(heavyRun.executionId as string); + testExecutionIds.add(interactiveRun.executionId as string); + + const workerId = `test-worker-${Bun.randomUUIDv7()}`; + const first = claimNextJobExecution(workerId, 1); + expect(first).toMatchObject({ + resourceClass: "interactive", + scheduledJobId: interactiveJobId, + status: "running", + }); + expect(claimNextJobExecution(workerId, 1)).toBeUndefined(); + finishJobExecution(first!.id, workerId, "success", undefined, { + completed: true, + }); + const second = claimNextJobExecution(workerId, 1); + expect(second).toMatchObject({ + resourceClass: "host-heavy", + scheduledJobId: heavyJobId, + status: "running", + }); + cancelJobExecution(second!.id); + expect( + finishJobExecution(second!.id, workerId, "success", undefined, {}) + ).toMatchObject({ status: "cancelled" }); + expect(getJobExecutionSummary()).toMatchObject({ queued: 0, running: 0 }); + }); + + it("prevents duplicate active runs and cancels queued work", () => { + const jobId = createScheduledTestJob("host-heavy", "Unique test job"); + const run = enqueueScheduledJob(jobId); + testExecutionIds.add(run.executionId as string); + + expect(() => enqueueScheduledJob(jobId)).toThrow( + "Scheduled job is already queued or running" + ); + expect(cancelJobExecution(run.executionId as string)).toMatchObject({ + status: "cancelled", + }); + expect( + database + .prepare("SELECT status FROM scheduled_job_runs WHERE id = ?") + .get(run.id) + ).toEqual({ status: "cancelled" }); + }); + + it("fails abandoned leases without replaying side effects", () => { + const queuedAt = "2026-01-01T00:00:00.000Z"; + const execution = insertJobExecution({ + actionKey: "test.expired", + cancellable: false, + displayName: "Expired execution", + leaseOwner: "missing-worker", + queuedAt, + resourceClass: "exclusive", + status: "running", + timeoutMs: 60_000, + triggerType: "system", + }); + testExecutionIds.add(execution.id); + + expect(recoverExpiredJobExecutions("2026-01-01T00:03:00.000Z")).toBe(1); + expect(getJobExecution(execution.id)).toMatchObject({ + message: "Job failed after its worker lease expired", + status: "failed", + }); + }); + + it("cancels queued startup work when its scheduled job is disabled", async () => { + const actionKey = `test.disabled-startup-${Bun.randomUUIDv7()}`; + const jobId = `test-queue-disabled-${Bun.randomUUIDv7()}`; + let actionCalls = 0; + testJobIds.add(jobId); + registerScheduledJobAction(actionKey, () => { + actionCalls += 1; + return { unexpected: true }; + }); + upsertScheduledJob({ + actionKey, + enabled: true, + id: jobId, + intervalSeconds: 3600, + name: "Disabled startup test job", + scheduleType: "interval", + }); + const run = enqueueScheduledJob(jobId, "startup"); + testExecutionIds.add(run.executionId as string); + expect(updateScheduledJob(jobId, { enabled: false })).toMatchObject({ + enabled: false, + }); + + startScheduledJobExecutor(); + const execution = await waitForJobExecution(run.executionId as string, { + pollIntervalMs: 10, + timeoutMs: 5000, + }); + + expect(execution).toMatchObject({ + message: "Scheduled job was disabled before execution", + status: "cancelled", + }); + expect(actionCalls).toBe(0); + expect( + database + .prepare("SELECT status, message FROM scheduled_job_runs WHERE id = ?") + .get(run.id) + ).toEqual({ + message: "Scheduled job was disabled before execution", + status: "cancelled", + }); + }); + + it("cancels queued work when its scheduled job is removed", async () => { + const actionKey = `test.removed-job-${Bun.randomUUIDv7()}`; + const jobId = `test-queue-removed-${Bun.randomUUIDv7()}`; + let actionCalls = 0; + testJobIds.add(jobId); + registerScheduledJobAction(actionKey, () => { + actionCalls += 1; + return { unexpected: true }; + }); + upsertScheduledJob({ + actionKey, + enabled: true, + id: jobId, + intervalSeconds: 3600, + name: "Removed scheduled test job", + scheduleType: "interval", + }); + const run = enqueueScheduledJob(jobId, "startup"); + testExecutionIds.add(run.executionId as string); + removeScheduledJobsNotInAction(actionKey, []); + + startScheduledJobExecutor(); + const execution = await waitForJobExecution(run.executionId as string, { + pollIntervalMs: 10, + timeoutMs: 5000, + }); + + expect(execution).toMatchObject({ + message: "Scheduled job was removed before execution", + status: "cancelled", + }); + expect(actionCalls).toBe(0); + expect( + database.prepare("SELECT id FROM scheduled_job_runs WHERE id = ?").get(run.id) + ).toBeNull(); + }); + + it("fails legacy running scheduled runs without an execution lease", () => { + const jobId = createScheduledTestJob("host-heavy", "Legacy running job"); + const run = database + .prepare( + `INSERT INTO scheduled_job_runs ( + job_id, status, trigger_type, started_at, output_json + ) VALUES (?, 'running', 'schedule', ?, '{}')` + ) + .run(jobId, "2026-01-01T00:00:00.000Z"); + + expect(recoverOrphanedScheduledJobRuns("2026-01-01T00:03:00.000Z")).toBe(1); + expect( + database + .prepare( + "SELECT status, finished_at AS finishedAt, message FROM scheduled_job_runs WHERE id = ?" + ) + .get(Number(run.lastInsertRowid)) + ).toEqual({ + finishedAt: "2026-01-01T00:03:00.000Z", + message: "Scheduled job interrupted before worker lease recovery", + status: "failed", + }); + }); +}); diff --git a/backend/test/routeAndServiceBehavior.test.ts b/backend/test/routeAndServiceBehavior.test.ts index 7adb70c8f..b22f6ef8e 100644 --- a/backend/test/routeAndServiceBehavior.test.ts +++ b/backend/test/routeAndServiceBehavior.test.ts @@ -25,7 +25,7 @@ import type { OpenClawGatewayClientOptions, } from "../src/lib/openclawGatewayClient.ts"; -const cleanupCallbacks: Array<() => void> = []; +const cleanupCallbacks: Array<() => Promise | void> = []; function rememberEnvironment(key: string): void { const originalValue = process.env[key]; @@ -184,7 +184,15 @@ async function responseJson(response: Response): Promise return (await response.json()) as Record; } -afterEach(() => { +async function startTestScheduledExecutor(): Promise { + const { startScheduledJobExecutor, stopScheduledJobExecutor } = + await import("../src/services/scheduledJobs.ts"); + startScheduledJobExecutor(); + cleanupCallbacks.push(stopScheduledJobExecutor); +} + +afterEach(async () => { + while (cleanupCallbacks.length > 0) await cleanupCallbacks.pop()?.(); database .prepare( "DELETE FROM task_updates WHERE task_id IN (SELECT id FROM tasks WHERE title LIKE 'Coverage %')" @@ -234,7 +242,6 @@ afterEach(() => { ) .run(); database.prepare("DELETE FROM users WHERE username LIKE 'coverage-%'").run(); - while (cleanupCallbacks.length > 0) cleanupCallbacks.pop()?.(); }); describe("backend route and service behavior", () => { @@ -2462,6 +2469,10 @@ describe("backend route and service behavior", () => { .prepare("DELETE FROM scheduled_jobs WHERE id = 'docker.updater'") .run(); }); + const { registerDockerUpdaterScheduledJobs } = + await import("../src/services/dockerUpdater.ts"); + registerDockerUpdaterScheduledJobs(); + await startTestScheduledExecutor(); const updaterRun = await dockerRoutes["/api/docker/updater/run"].POST(); expect(updaterRun.status).toBe(200); await expect(updaterRun.json()).resolves.toMatchObject({ @@ -3129,12 +3140,13 @@ describe("backend route and service behavior", () => { .spyOn(AbortSignal, "timeout") .mockReturnValue(gatewayHeadersTimeoutController.signal); cleanupCallbacks.push( - () => + () => { Object.defineProperty(globalThis, "fetch", { configurable: true, value: originalFetch, writable: true, - }), + }); + }, () => gatewayTimeoutSpy.mockRestore() ); @@ -3327,11 +3339,12 @@ describe("backend route and service behavior", () => { writeFakeBackupDocker(path.join(fakeBin, "docker")); process.env.PATH = `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`; const { backupRoutes } = await import("../src/routes/backupRoutes.ts"); - const { getCurrentBackupJob, registerBackupScheduledJobs } = + const { registerBackupScheduledJobs } = await import("../src/services/backups.ts"); try { registerBackupScheduledJobs(); + await startTestScheduledExecutor(); const response = await backupRoutes["/api/backups/walg/run"].POST(); expect(response.status).toBe(200); const body = (await response.json()) as { @@ -3343,20 +3356,24 @@ describe("backend route and service behavior", () => { job: { status: "running", type: "walg" }, }); - const completed = await getCurrentBackupJob("walg")?.completed; - expect(completed).toMatchObject({ - code: 0, - status: "done", - stdout: expect.stringContaining("backup ok"), - type: "walg", - }); - - const status = backupRoutes["/api/backups/walg"].GET(); - await expect(status.json()).resolves.toMatchObject({ - job: { code: 0, status: "done", type: "walg" }, + let statusBody: Record = {}; + const deadline = Date.now() + 3000; + while (Date.now() < deadline) { + statusBody = await responseJson(backupRoutes["/api/backups/walg"].GET()); + if ( + (statusBody.job as { status?: string } | undefined)?.status === "done" + ) + break; + await Bun.sleep(25); + } + expect(statusBody).toMatchObject({ + job: { + code: 0, + status: "done", + stdout: expect.stringContaining("backup ok"), + type: "walg", + }, }); - const clearedStatus = backupRoutes["/api/backups/walg"].GET(); - await expect(clearedStatus.json()).resolves.toEqual({ job: undefined }); } finally { database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") @@ -3380,6 +3397,29 @@ describe("backend route and service behavior", () => { process.env.MIRA_DOCKER_COMPOSE_WRAPPER = path.join(fakeBin, "compose"); process.env.MIRA_DOCKER_ROOT = dockerRoot; const { dockerRoutes } = await import("../src/routes/dockerRoutes.ts"); + const { registerDockerExecutionActions } = + await import("../src/services/dockerActions.ts"); + registerDockerExecutionActions(); + await startTestScheduledExecutor(); + const executionBaseline = database + .prepare("SELECT COALESCE(MAX(rowid), 0) AS rowId FROM job_executions") + .get() as { rowId: number }; + cleanupCallbacks.push(() => { + database + .prepare( + `DELETE FROM job_executions + WHERE rowid > ? + AND action_key IN ( + 'docker.stack.action', + 'docker.container.action', + 'docker.image.delete', + 'docker.prune.images', + 'docker.prune.volumes', + 'docker.volume.delete' + )` + ) + .run(executionBaseline.rowId); + }); const containers = await dockerRoutes["/api/docker/containers"].GET(); await expect(containers.json()).resolves.toMatchObject({ @@ -3582,6 +3622,41 @@ describe("backend route and service behavior", () => { await expect(stackServiceAction.json()).resolves.toEqual({ output: "compose:restart web", }); + const mutationExecutions = database + .prepare( + `SELECT action_key AS actionKey, cancellable, status + FROM job_executions + WHERE rowid > ? + AND action_key IN ( + 'docker.stack.action', + 'docker.container.action', + 'docker.image.delete', + 'docker.prune.images', + 'docker.prune.volumes', + 'docker.volume.delete' + ) + ORDER BY rowid` + ) + .all(executionBaseline.rowId) as Array<{ + actionKey: string; + cancellable: number; + status: string; + }>; + expect(mutationExecutions.map((execution) => execution.actionKey)).toEqual([ + "docker.container.action", + "docker.image.delete", + "docker.volume.delete", + "docker.prune.images", + "docker.prune.volumes", + "docker.stack.action", + "docker.stack.action", + ]); + expect( + mutationExecutions.every( + (execution) => + execution.cancellable === 0 && execution.status === "success" + ) + ).toBe(true); const invalidStackAction = await dockerRoutes["/api/docker/stack/action"].POST( jsonRequest("/api/docker/stack/action", { action: "reload" }) @@ -3653,6 +3728,16 @@ describe("backend route and service behavior", () => { ); const { jobId } = (await execStart.json()) as { jobId: string }; expect(jobId).toEqual(expect.any(String)); + expect( + database + .prepare( + "SELECT resource_class AS resourceClass, timeout_ms AS timeoutMs FROM job_executions WHERE id = ?" + ) + .get(jobId) + ).toEqual({ + resourceClass: "exclusive", + timeoutMs: 7 * 60 * 60 * 1000, + }); let execStatus = dockerRoutes["/api/docker/exec/:jobId"].GET( requestWithParameters(`/api/docker/exec/${jobId}`, { jobId }) @@ -3679,7 +3764,7 @@ describe("backend route and service behavior", () => { await expect(stopCompletedExec.json()).resolves.toEqual({ error: "Job is not running", }); - }); + }, 20_000); it("normalizes ops log-rotation status cache state", async () => { const { opsRoutes } = await import("../src/routes/opsRoutes.ts"); @@ -3750,8 +3835,14 @@ describe("backend route and service behavior", () => { }); it("exec service validation and error normalization branches", async () => { - const { execErrorResponse, getExecJob, runExecOnce, startExecJob, stopExecJob } = - await import("../src/services/execJobs.ts"); + const { + execErrorResponse, + getExecJob, + registerExecExecutionActions, + runExecOnce, + startExecJob, + stopExecJob, + } = await import("../src/services/execJobs.ts"); await expect(runExecOnce(undefined)).rejects.toThrow( "request body must be a JSON object" @@ -3814,6 +3905,8 @@ describe("backend route and service behavior", () => { }) ).rejects.toThrow("cwd does not exist"); + registerExecExecutionActions(); + await startTestScheduledExecutor(); const result = await runExecOnce({ command: "__mira_dashboard_shell_smoke_test__", cwd: process.cwd(), @@ -4748,6 +4841,7 @@ esac } registerCacheRefreshScheduledJobs(); + await startTestScheduledExecutor(); const rows = database .prepare( diff --git a/backend/test/serverStartupPolicy.test.ts b/backend/test/serverStartupPolicy.test.ts index 2975f9d97..420658097 100644 --- a/backend/test/serverStartupPolicy.test.ts +++ b/backend/test/serverStartupPolicy.test.ts @@ -20,6 +20,16 @@ describe("server start scheduler policy", () => { MIRA_DASHBOARD_DISABLE_SCHEDULER: "1", }) ).toBe(false); + expect( + shouldStartScheduledJobs({ + MIRA_DASHBOARD_EXECUTION_ROLE: "web", + }) + ).toBe(false); + expect( + shouldStartScheduledJobs({ + MIRA_DASHBOARD_EXECUTION_ROLE: "combined", + }) + ).toBe(true); }); it("resolves backend startup entrypoint and gateway token decisions without starting services", async () => { @@ -59,6 +69,155 @@ describe("server start scheduler policy", () => { expect(shouldStartOnImport("0", false)).toBe(false); }); + it("resolves the dedicated worker entrypoint and keeps its event loop referenced", async () => { + const { createWorkerKeepAliveHandle, isDirectWorkerEntrypoint } = + await import("../src/workerStart.ts"); + + expect( + isDirectWorkerEntrypoint("/tmp/workerStart.ts", "file:///tmp/workerStart.ts") + ).toBe(true); + expect( + isDirectWorkerEntrypoint("/tmp/serverStart.ts", "file:///tmp/workerStart.ts") + ).toBe(false); + + const keepAlive = createWorkerKeepAliveHandle(); + try { + expect(keepAlive.hasRef()).toBe(true); + } finally { + clearInterval(keepAlive); + } + }); + + it("cleans up dedicated worker state when startup fails", async () => { + const jobWorker = await import("../src/services/jobWorker.ts"); + const workerStart = await import("../src/workerStart.ts"); + const sigintListeners = process.listenerCount("SIGINT"); + const sigtermListeners = process.listenerCount("SIGTERM"); + const startSpy = jest + .spyOn(jobWorker, "startDashboardJobWorker") + .mockImplementation(() => { + throw new Error("worker startup failed"); + }); + const stopSpy = jest + .spyOn(jobWorker, "stopDashboardJobWorker") + .mockImplementation(async () => {}); + + try { + await expect(workerStart.runDashboardWorker()).rejects.toThrow( + "worker startup failed" + ); + expect(stopSpy).toHaveBeenCalledTimes(1); + expect(process.listenerCount("SIGINT")).toBe(sigintListeners); + expect(process.listenerCount("SIGTERM")).toBe(sigtermListeners); + } finally { + startSpy.mockRestore(); + stopSpy.mockRestore(); + } + }); + + it("keeps worker startup blocked until failed executor cleanup is retried", async () => { + const backups = await import("../src/services/backups.ts"); + const cacheRefresh = await import("../src/services/cacheRefresh.ts"); + const dockerActions = await import("../src/services/dockerActions.ts"); + const dockerUpdater = await import("../src/services/dockerUpdater.ts"); + const execJobs = await import("../src/services/execJobs.ts"); + const gitHygiene = await import("../src/services/gitHygiene.ts"); + const logRotation = await import("../src/services/logRotation.ts"); + const openclawActions = await import("../src/services/openclawActions.ts"); + const pullRequests = await import("../src/services/pullRequests.ts"); + const scheduledJobs = await import("../src/services/scheduledJobs.ts"); + const worker = await import("../src/services/jobWorker.ts"); + const registrationSpies = [ + jest + .spyOn(backups, "registerBackupScheduledJobs") + .mockImplementation(() => {}), + jest + .spyOn(cacheRefresh, "registerCacheRefreshScheduledJobs") + .mockImplementation(() => {}), + jest + .spyOn(dockerActions, "registerDockerExecutionActions") + .mockImplementation(() => {}), + jest + .spyOn(dockerUpdater, "registerDockerUpdaterScheduledJobs") + .mockImplementation(() => {}), + jest + .spyOn(execJobs, "registerExecExecutionActions") + .mockImplementation(() => {}), + jest + .spyOn(gitHygiene, "registerGitHygieneScheduledJobs") + .mockImplementation(() => {}), + jest + .spyOn(logRotation, "registerLogRotationScheduledJobs") + .mockImplementation(() => {}), + jest + .spyOn(openclawActions, "registerOpenClawExecutionActions") + .mockImplementation(() => {}), + jest + .spyOn(pullRequests, "registerPullRequestExecutionActions") + .mockImplementation(() => {}), + ]; + const startExecutorSpy = jest + .spyOn(scheduledJobs, "startScheduledJobExecutor") + .mockImplementation(() => {}); + const startSchedulerSpy = jest + .spyOn(scheduledJobs, "startScheduledJobScheduler") + .mockImplementation(() => {}); + startSchedulerSpy.mockImplementationOnce(() => { + throw new Error("scheduler startup failed"); + }); + const stopExecutorSpy = jest + .spyOn(scheduledJobs, "stopScheduledJobExecutor") + .mockImplementation(async () => {}); + stopExecutorSpy.mockImplementationOnce(async () => { + throw new Error("executor cleanup failed"); + }); + const stopSchedulerSpy = jest + .spyOn(scheduledJobs, "stopScheduledJobScheduler") + .mockImplementation(() => {}); + const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + + try { + expect(() => worker.startDashboardJobWorker()).toThrow( + "scheduler startup failed" + ); + for (const registrationSpy of registrationSpies) { + expect(registrationSpy).toHaveBeenCalledTimes(1); + } + await Bun.sleep(0); + expect(errorSpy).toHaveBeenCalledWith( + "[JobWorker] Failed to roll back executor startup:", + expect.objectContaining({ message: "executor cleanup failed" }) + ); + + worker.startDashboardJobWorker(); + expect(startExecutorSpy).toHaveBeenCalledTimes(1); + expect(startSchedulerSpy).toHaveBeenCalledTimes(1); + + await worker.stopDashboardJobWorker(); + expect(stopExecutorSpy).toHaveBeenCalledTimes(2); + + worker.startDashboardJobWorker(); + expect(startExecutorSpy).toHaveBeenCalledTimes(2); + expect(startSchedulerSpy).toHaveBeenCalledTimes(2); + await worker.stopDashboardJobWorker(); + expect(stopExecutorSpy).toHaveBeenCalledTimes(3); + } finally { + try { + await worker.stopDashboardJobWorker(); + } catch { + // Test cleanup is best-effort after assertions fail. + } + for (const registrationSpy of registrationSpies) { + registrationSpy.mockRestore(); + } + startExecutorSpy.mockRestore(); + startSchedulerSpy.mockRestore(); + stopExecutorSpy.mockRestore(); + stopSchedulerSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); + it("starts listening-time services with a configured gateway token", async () => { const originalGatewayToken = process.env.OPENCLAW_GATEWAY_TOKEN; const originalSchedulerDisabled = process.env.MIRA_DASHBOARD_DISABLE_SCHEDULER; diff --git a/backend/test/serviceBehavior.test.ts b/backend/test/serviceBehavior.test.ts index c421d2640..685492815 100644 --- a/backend/test/serviceBehavior.test.ts +++ b/backend/test/serviceBehavior.test.ts @@ -20,7 +20,7 @@ import type { DashboardSocket } from "../src/dashboardSocket.ts"; import { database, sqlNullable } from "../src/database.ts"; import * as processModule from "../src/lib/processes.ts"; -const cleanupCallbacks: Array<() => void> = []; +const cleanupCallbacks: Array<() => Promise | void> = []; function rememberEnvironment(key: string): void { const originalValue = process.env[key]; @@ -351,15 +351,29 @@ function waitFor(isReady: () => boolean, timeoutMilliseconds = 1000): Promise { +async function startTestScheduledExecutor(): Promise { + const { startScheduledJobExecutor, stopScheduledJobExecutor } = + await import("../src/services/scheduledJobs.ts"); + startScheduledJobExecutor(); + cleanupCallbacks.push(stopScheduledJobExecutor); +} + +afterEach(async () => { const errors: unknown[] = []; while (cleanupCallbacks.length > 0) { try { - cleanupCallbacks.pop()?.(); + await cleanupCallbacks.pop()?.(); } catch (error) { errors.push(error); } } + database + .prepare( + `DELETE FROM job_executions + WHERE scheduled_job_id LIKE 'backup.%' + OR action_key = 'backup.clear-attention'` + ) + .run(); if (errors.length > 0) { throw new AggregateError(errors, "Test cleanup failed"); } @@ -843,6 +857,10 @@ describe("backend service behavior", () => { }) as typeof fetch); cleanupCallbacks.push(() => fetchSpy.mockRestore()); + const { registerCacheRefreshScheduledJobs } = + await import("../src/services/cacheRefresh.ts"); + registerCacheRefreshScheduledJobs({ seedStrategy: "none" }); + await startTestScheduledExecutor(); const { cacheRoutes } = await import("../src/routes/cacheRoutes.ts"); const response = await cacheRoutes["/api/cache/:key/refresh"].POST( Object.assign( @@ -972,6 +990,189 @@ describe("backend service behavior", () => { } }); + it("keeps queued deployment locks active beyond the legacy stale window", async () => { + const { startDeployLatest } = await import("../src/services/pullRequests.ts"); + const { cancelJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); + const first = startDeployLatest(); + let replacementId: string | undefined; + try { + database + .prepare("UPDATE deployment_jobs SET updated_at = ? WHERE id = ?") + .run("2026-01-01T00:00:00.000Z", first.id); + database + .prepare("UPDATE deployment_lock SET updated_at = ? WHERE job_id = ?") + .run("2026-01-01T00:00:00.000Z", first.id); + + expect(() => startDeployLatest()).toThrow( + `Dashboard deploy already in progress (${first.id})` + ); + + const firstExecution = database + .prepare( + `SELECT id + FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .get(first.id) as { id: string }; + cancelJobExecution(firstExecution.id); + expect( + database + .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") + .get(first.id) + ).toEqual({ + note: "Deploy cancelled before execution", + status: "failed", + }); + expect( + database.prepare("SELECT job_id FROM deployment_lock WHERE id = 1").get() + ).toBeNull(); + + const replacement = startDeployLatest(); + replacementId = replacement.id; + expect(replacement.id).not.toBe(first.id); + } finally { + const deploymentIds = [first.id, replacementId].filter((id): id is string => + Boolean(id) + ); + database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); + for (const deploymentId of deploymentIds) { + database + .prepare( + `DELETE FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .run(deploymentId); + database + .prepare("DELETE FROM deployment_jobs WHERE id = ?") + .run(deploymentId); + } + } + }); + + it("fails deployments and releases their locks when worker leases expire", async () => { + const { startDeployLatest } = await import("../src/services/pullRequests.ts"); + const { getJobExecution, recoverExpiredJobExecutions } = + await import("../src/services/jobExecutionQueue.ts"); + const deployment = startDeployLatest(); + let replacementId: string | undefined; + try { + const execution = database + .prepare( + `SELECT id + FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .get(deployment.id) as { id: string }; + database + .prepare( + `UPDATE job_executions + SET status = 'running', started_at = ?, heartbeat_at = ?, + lease_owner = ?, lease_expires_at = ?, attempt = 1 + WHERE id = ?` + ) + .run( + "2100-01-01T00:00:00.000Z", + "2100-01-01T00:00:00.000Z", + "missing-deploy-worker", + "2100-01-01T00:02:00.000Z", + execution.id + ); + + expect(recoverExpiredJobExecutions("2100-01-01T00:03:00.000Z")).toBe(1); + expect(getJobExecution(execution.id)).toMatchObject({ + message: "Job failed after its worker lease expired", + status: "failed", + }); + expect( + database + .prepare("SELECT status, note FROM deployment_jobs WHERE id = ?") + .get(deployment.id) + ).toEqual({ + note: "Deploy failed after its worker lease expired", + status: "failed", + }); + expect( + database.prepare("SELECT job_id FROM deployment_lock WHERE id = 1").get() + ).toBeNull(); + + const replacement = startDeployLatest(); + replacementId = replacement.id; + expect(replacement.status).toBe("building"); + } finally { + const deploymentIds = [deployment.id, replacementId].filter( + (id): id is string => Boolean(id) + ); + database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); + for (const deploymentId of deploymentIds) { + database + .prepare( + `DELETE FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .run(deploymentId); + database + .prepare("DELETE FROM deployment_jobs WHERE id = ?") + .run(deploymentId); + } + } + }); + + it("reserves the deployment lock while a pull request approval is queued", async () => { + const { runPullRequestApproval, startDeployLatest } = + await import("../src/services/pullRequests.ts"); + const { cancelJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); + const approval = runPullRequestApproval(11, false); + let approvalExecutionId: string | undefined; + let deploymentId: string | undefined; + try { + const execution = database + .prepare( + `SELECT id + FROM job_executions + WHERE action_key = 'github.merge' + ORDER BY queued_at DESC, id DESC + LIMIT 1` + ) + .get() as { id: string }; + approvalExecutionId = execution.id; + expect(() => startDeployLatest()).toThrow( + "Dashboard deploy already in progress" + ); + + cancelJobExecution(execution.id); + await expect(approval).rejects.toThrow("Job cancelled before execution"); + + const deployment = startDeployLatest(); + deploymentId = deployment.id; + expect(deployment.status).toBe("building"); + } finally { + database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); + if (approvalExecutionId) { + database + .prepare("DELETE FROM job_executions WHERE id = ?") + .run(approvalExecutionId); + } + if (deploymentId) { + database + .prepare( + `DELETE FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .run(deploymentId); + database + .prepare("DELETE FROM deployment_jobs WHERE id = ?") + .run(deploymentId); + } + } + }); + it("runs deploy latest build flow against an isolated checkout", async () => { rememberEnvironment("PATH"); rememberEnvironment("MIRA_DASHBOARD_ROOT"); @@ -1036,8 +1237,31 @@ printf 'scheduled\n' process.env.MIRA_DASHBOARD_ROOT = fakeRoot; process.env.MIRA_DASHBOARD_WORKTREE_ROOT = path.join(fakeRoot, "worktrees"); - const { startDeployLatest } = await import("../src/services/pullRequests.ts"); + const { registerPullRequestExecutionActions, startDeployLatest } = + await import("../src/services/pullRequests.ts"); + const { enqueueJobExecution, getJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); + const { registerScheduledJobAction } = + await import("../src/services/scheduledJobs.ts"); + registerPullRequestExecutionActions(); + registerScheduledJobAction("test.after-deploy", async () => ({})); + await startTestScheduledExecutor(); const job = startDeployLatest(); + const deploymentExecution = database + .prepare( + `SELECT id + FROM job_executions + WHERE action_key = 'dashboard.deploy' + AND json_extract(payload_json, '$.deploymentId') = ?` + ) + .get(job.id) as { id: string }; + const followUpExecution = enqueueJobExecution({ + actionKey: "test.after-deploy", + displayName: "Must wait for restarted worker", + priority: 0, + resourceClass: "light", + timeoutMs: 1000, + }); try { await waitFor(() => { @@ -1054,7 +1278,12 @@ printf 'scheduled\n' } | undefined; return row?.status === "restart-scheduled" && existsSync(systemdLog); - }); + }, 5000); + await waitFor( + () => getJobExecution(deploymentExecution.id)?.status === "success", + 5000 + ); + await Bun.sleep(25); const row = database .prepare( @@ -1084,13 +1313,48 @@ printf 'scheduled\n' await expect(Bun.file(systemdLog).text()).resolves.toContain( `mira-dashboard-deploy-${job.id}` ); + const restartCommand = await Bun.file(systemdLog).text(); + expect(restartCommand).toContain("/api/health"); + expect(restartCommand).toContain('"workerOnline":true'); + expect(restartCommand).not.toContain("/api/job-executions"); expect(existsSync(path.join(fakeRoot, "node_modules"))).toBe(false); expect(existsSync(path.join(fakeRoot, "backend", "node_modules"))).toBe( false ); + expect(getJobExecution(deploymentExecution.id)).toMatchObject({ + cancellable: false, + status: "success", + }); + expect(getJobExecution(followUpExecution.id)).toMatchObject({ + status: "queued", + }); + database + .prepare( + `UPDATE deployment_jobs + SET status = 'failed', + updated_at = ?, + note = 'Detached restart failed' + WHERE id = ?` + ) + .run(new Date().toISOString(), job.id); + await waitFor( + () => getJobExecution(followUpExecution.id)?.status === "success", + 5000 + ); + expect(getJobExecution(followUpExecution.id)).toMatchObject({ + status: "success", + }); } finally { + database + .prepare("DELETE FROM job_executions WHERE id = ?") + .run(followUpExecution.id); database.prepare("DELETE FROM deployment_lock WHERE job_id = ?").run(job.id); database.prepare("DELETE FROM deployment_jobs WHERE id = ?").run(job.id); + database + .prepare( + "DELETE FROM job_executions WHERE action_key = 'dashboard.deploy' AND payload_json LIKE ?" + ) + .run(`%${job.id}%`); } }); @@ -1245,8 +1509,26 @@ fi process.env.RAJOHAN_GITHUB_TOKEN = "review-token"; process.env.RAJOHAN_GITHUB_USERNAME = "rajohan"; - const { approvePullRequestReview, rejectPullRequest, updatePullRequestBranch } = - await import("../src/services/pullRequests.ts"); + const { + approvePullRequestReview, + registerPullRequestExecutionActions, + rejectPullRequest, + updatePullRequestBranch, + } = await import("../src/services/pullRequests.ts"); + registerPullRequestExecutionActions(); + cleanupCallbacks.push(() => { + database + .prepare( + `DELETE FROM job_executions + WHERE action_key IN ( + 'github.review-approval', + 'github.update-branch', + 'github.reject' + )` + ) + .run(); + }); + await startTestScheduledExecutor(); const { pullRequestRoutes } = await import("../src/routes/pullRequestRoutes.ts"); await expect(approvePullRequestReview(3)).resolves.toMatchObject({ @@ -1313,6 +1595,34 @@ fi isOk: true, message: "PR #5 closed", }); + const queuedMutations = database + .prepare( + `SELECT action_key AS actionKey, cancellable, status + FROM job_executions + WHERE action_key IN ( + 'github.review-approval', + 'github.update-branch', + 'github.reject' + ) + ORDER BY action_key, id` + ) + .all() as Array<{ + actionKey: string; + cancellable: number; + status: string; + }>; + expect(queuedMutations.map((execution) => execution.actionKey)).toEqual([ + "github.reject", + "github.reject", + "github.review-approval", + "github.update-branch", + ]); + expect( + queuedMutations.every( + (execution) => + execution.cancellable === 0 && execution.status === "success" + ) + ).toBe(true); const malformedApproveRoute = await pullRequestRoutes[ "/api/pull-requests/:number/approve" @@ -1391,9 +1701,11 @@ fi process.env.RAJOHAN_GITHUB_USERNAME = "rajohan"; try { - const { approvePullRequest } = + const { registerPullRequestExecutionActions, runPullRequestApproval } = await import("../src/services/pullRequests.ts"); - const result = await approvePullRequest(11, false); + registerPullRequestExecutionActions(); + await startTestScheduledExecutor(); + const result = await runPullRequestApproval(11, false); expect(result).toMatchObject({ cleanup: { @@ -1403,13 +1715,31 @@ fi }, isOk: true, message: "PR #11 merged", - syncError: undefined, }); await expect(Bun.file(ghLog).text()).resolves.toContain("pr merge 11"); await expect(Bun.file(gitLog).text()).resolves.toContain("worktree remove"); expect(existsSync(localWorktree)).toBe(false); + expect( + database + .prepare( + `SELECT cancellable, status + FROM job_executions + WHERE action_key = 'github.merge' + AND json_extract(payload_json, '$.number') = 11 + ORDER BY queued_at DESC, id DESC + LIMIT 1` + ) + .get() + ).toEqual({ cancellable: 0, status: "success" }); } finally { database.prepare("DELETE FROM deployment_lock WHERE id = 1").run(); + database + .prepare( + `DELETE FROM job_executions + WHERE action_key = 'github.merge' + AND json_extract(payload_json, '$.number') = 11` + ) + .run(); database .prepare("DELETE FROM deployment_jobs WHERE id LIKE 'approve-%'") .run(); @@ -2196,10 +2526,28 @@ fi await waitFor(() => weatherResponses === 1); const secondRefresh = refreshCacheProducer("weather.spydeberg"); const abortController = new AbortController(); + const abortedRefresh = refreshCacheProducer( + "weather.spydeberg", + abortController.signal + ); abortController.abort(); - await expect( - refreshCacheProducer("weather.spydeberg", abortController.signal) - ).rejects.toThrow("Cache refresh aborted"); + const abortedRefreshState = (async () => { + try { + await abortedRefresh; + return "resolved" as const; + } catch (error) { + return error instanceof Error ? error.message : "rejected without error"; + } + })(); + expect( + await Promise.race([ + abortedRefreshState, + (async () => { + await Bun.sleep(10); + return "pending" as const; + })(), + ]) + ).toBe("pending"); releaseWeather?.(); await expect(firstRefresh).resolves.toEqual({ @@ -2208,6 +2556,7 @@ fi await expect(secondRefresh).resolves.toEqual({ refreshed: ["weather.spydeberg"], }); + await expect(abortedRefreshState).resolves.toBe("Cache refresh aborted"); expect(weatherResponses).toBe(1); }); @@ -2509,13 +2858,18 @@ fi const fakeBin = createTemporaryRoot("mira-backup-docker-bin-"); writeFakeDocker(path.join(fakeBin, "docker")); process.env.PATH = `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`; - const { getCurrentBackupJob, registerBackupScheduledJobs, startManualBackup } = - await import("../src/services/backups.ts"); + const { + clearNeedsAttentionBackupJob, + getCurrentBackupJob, + registerBackupScheduledJobs, + startManualBackup, + } = await import("../src/services/backups.ts"); const { getScheduledJob, runScheduledJob, upsertScheduledJob } = await import("../src/services/scheduledJobs.ts"); try { registerBackupScheduledJobs(); + await startTestScheduledExecutor(); expect(getScheduledJob("backup.walg")).toMatchObject({ actionKey: "backup.run", enabled: true, @@ -2546,23 +2900,10 @@ fi stdout: expect.stringContaining("backup ok"), type: "walg", }); - expect(getCurrentBackupJob("walg")).toMatchObject({ status: "done" }); - expect(getCurrentBackupJob("walg")).toBeUndefined(); - await waitFor(() => { - const row = database - .prepare( - "SELECT status FROM scheduled_job_runs WHERE job_id = 'backup.walg' ORDER BY id DESC LIMIT 1" - ) - .get() as { status?: string } | undefined; - return row?.status === "success"; + await expect(clearNeedsAttentionBackupJob("walg")).rejects.toMatchObject({ + statusCode: 404, }); - expect( - database - .prepare( - "SELECT status FROM scheduled_job_runs WHERE job_id = 'backup.walg' ORDER BY id DESC LIMIT 1" - ) - .get() - ).toEqual({ status: "success" }); + expect(getCurrentBackupJob("walg")).toBeUndefined(); } finally { database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") @@ -2585,17 +2926,6 @@ fi statusCode: 503, }); expect(getCurrentBackupJob("walg")).toBeUndefined(); - await waitFor(() => { - const row = database - .prepare( - "SELECT status, message FROM scheduled_job_runs WHERE job_id = 'backup.walg' ORDER BY id DESC LIMIT 1" - ) - .get() as { message?: string; status?: string } | undefined; - return ( - row?.status === "failed" && - row.message?.includes("pgrep failed") === true - ); - }); } finally { database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") @@ -2695,7 +3025,7 @@ fi } }); - it("runs a manual backup without scheduled metadata and trims oversized output", async () => { + it("trims oversized output in the worker backup primitive", async () => { const { getCurrentBackupJob, startManualBackup } = await import("../src/services/backups.ts"); const largeOutput = `${"x".repeat(100_200)}tail-marker\n`; @@ -2739,7 +3069,6 @@ fi }); expect(completed.stdout.length).toBeLessThanOrEqual(100_000); expect(completed.stdout).toEndWith("tail-marker\n"); - expect(completed.manualScheduledRun).toBeUndefined(); expect( database .prepare( @@ -2804,17 +3133,6 @@ fi expect(completed.stdout).toContain("stdout before failure"); expect(completed.stderr).toContain("stderr before failure"); expect(completed.stderr).toContain("child process promise failed"); - await waitFor(() => { - const row = database - .prepare( - "SELECT status, message FROM scheduled_job_runs WHERE job_id = 'backup.walg' ORDER BY id DESC LIMIT 1" - ) - .get() as { message: string; status: string } | undefined; - return ( - row?.status === "failed" && - row.message.includes("child process promise failed") - ); - }); expect(getCurrentBackupJob("walg")).toMatchObject({ status: "done" }); expect(getCurrentBackupJob("walg")).toBeUndefined(); } finally { @@ -2827,20 +3145,26 @@ fi } }); - it("marks aborted scheduled backups failed before preflight starts", async () => { + it("cancels queued backups before worker preflight starts", async () => { const { getCurrentBackupJob, registerBackupScheduledJobs } = await import("../src/services/backups.ts"); - const { runScheduledJob } = await import("../src/services/scheduledJobs.ts"); - const controller = new AbortController(); - controller.abort(); + const { cancelJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); + const { enqueueScheduledJob } = await import("../src/services/scheduledJobs.ts"); try { registerBackupScheduledJobs(); - const run = await runScheduledJob("backup.walg", "manual", controller.signal); - expect(run).toMatchObject({ + const run = enqueueScheduledJob("backup.walg", "manual"); + cancelJobExecution(run.executionId as string); + expect( + database + .prepare( + "SELECT job_id AS jobId, status FROM scheduled_job_runs WHERE id = ?" + ) + .get(run.id) + ).toMatchObject({ jobId: "backup.walg", - message: "Scheduled job aborted", - status: "failed", + status: "cancelled", }); expect(getCurrentBackupJob("walg")).toBeUndefined(); } finally { @@ -2888,6 +3212,7 @@ fi try { registerBackupScheduledJobs(); + await startTestScheduledExecutor(); const run = await runScheduledJob("backup.walg"); expect(run).toMatchObject({ @@ -2938,6 +3263,7 @@ fi try { registerBackupScheduledJobs(); + await startTestScheduledExecutor(); const run = await runScheduledJob("backup.kopia"); expect(run).toMatchObject({ @@ -2966,7 +3292,11 @@ fi it("terminates running WAL-G backups when a scheduled run is aborted", async () => { const { getCurrentBackupJob, registerBackupScheduledJobs } = await import("../src/services/backups.ts"); - const { runScheduledJob } = await import("../src/services/scheduledJobs.ts"); + const { cancelJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); + const { waitForJobExecution } = + await import("../src/services/queuedJobExecution.ts"); + const { enqueueScheduledJob } = await import("../src/services/scheduledJobs.ts"); const exit = Promise.withResolvers(); const runProcessCalls: string[] = []; const runProcessSpy = jest @@ -3002,22 +3332,20 @@ fi stdout: readableUtf8Stream(""), }) as unknown as processModule.BunProcess ); - const controller = new AbortController(); - try { registerBackupScheduledJobs(); - const runPromise = runScheduledJob( - "backup.walg", - "manual", - controller.signal - ); - await waitFor(() => spawnSpy.mock.calls.length === 1); - controller.abort(); + await startTestScheduledExecutor(); + const run = enqueueScheduledJob("backup.walg", "manual"); + await waitFor(() => spawnSpy.mock.calls.length === 1, 3000); + cancelJobExecution(run.executionId as string); + await waitFor(() => killSignals.includes("SIGTERM"), 3000); exit.resolve(143); - const run = await runPromise; - expect(run.status).toBe("failed"); - expect(run.message).toBe("Scheduled job aborted"); + const execution = await waitForJobExecution(run.executionId as string, { + timeoutMs: 3000, + }); + expect(execution.status).toBe("cancelled"); + expect(execution.message).toBe("Job cancelled"); expect(killSignals).toContain("SIGTERM"); expect(runProcessCalls).toEqual( expect.arrayContaining([ @@ -3046,17 +3374,30 @@ fi writeRunningWalgPreflightDocker(path.join(fakeBin, "docker")); process.env.PATH = `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`; const { - clearNeedsAttentionBackupJob, + clearPersistedBackupAttention, getCurrentBackupJob, + getPersistedBackupJob, mapBackupJob, registerBackupScheduledJobs, startManualBackup, } = await import("../src/services/backups.ts"); + const { runScheduledJob } = await import("../src/services/scheduledJobs.ts"); try { registerBackupScheduledJobs(); - await expect(startManualBackup("walg")).rejects.toMatchObject({ - statusCode: 409, + await startTestScheduledExecutor(); + await expect(runScheduledJob("backup.walg")).resolves.toMatchObject({ + output: { + backup: { + code: 130, + status: "needs_attention", + stderr: expect.stringContaining( + "backup process is still running" + ), + type: "walg", + }, + }, + status: "failed", }); expect(mapBackupJob(getCurrentBackupJob("walg"))).toMatchObject({ code: 130, @@ -3067,20 +3408,157 @@ fi await expect(startManualBackup("walg")).rejects.toThrow( "WALG backup needs attention" ); + expect(getPersistedBackupJob("walg")).toMatchObject({ + code: 130, + status: "needs_attention", + stderr: expect.stringContaining("backup process is still running"), + type: "walg", + }); - const clearedJob = await clearNeedsAttentionBackupJob("walg"); - expect(mapBackupJob(clearedJob)).toMatchObject({ + const clearedJob = await clearPersistedBackupAttention("walg"); + expect(clearedJob).toMatchObject({ status: "needs_attention", type: "walg", }); expect(getCurrentBackupJob("walg")).toBeUndefined(); - await waitFor(() => { - const row = database + expect(getPersistedBackupJob("walg")).toBeUndefined(); + } finally { + database + .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") + .run(); + database.prepare("DELETE FROM scheduled_jobs WHERE id LIKE 'backup.%'").run(); + } + }); + + it("clears persisted backup attention without in-memory worker state", async () => { + rememberEnvironment("PATH"); + const fakeBin = createTemporaryRoot("mira-backup-persisted-clear-bin-"); + writeFakeDocker(path.join(fakeBin, "docker")); + process.env.PATH = `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`; + const { + clearPersistedBackupAttention, + getCurrentBackupJob, + getPersistedBackupJob, + queueManualBackup, + registerBackupScheduledJobs, + } = await import("../src/services/backups.ts"); + const { enqueueScheduledJob, runScheduledJob } = + await import("../src/services/scheduledJobs.ts"); + + try { + registerBackupScheduledJobs(); + const run = enqueueScheduledJob("backup.walg", "manual"); + const executionId = run.executionId; + if (!executionId) throw new Error("Backup execution id was missing"); + const completedAt = "2026-07-22T02:00:00.000Z"; + const backup = { + code: 130, + endedAt: Date.parse(completedAt), + id: Bun.randomUUIDv7(), + startedAt: Date.parse(completedAt), + status: "needs_attention", + stderr: "Worker restarted before attention was cleared", + stdout: "", + type: "walg", + }; + database + .prepare( + `UPDATE job_executions + SET status = 'failed', started_at = ?, finished_at = ?, + message = 'WALG backup needs attention', output_json = ? + WHERE id = ?` + ) + .run(completedAt, completedAt, JSON.stringify({ backup }), executionId); + database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'failed', started_at = ?, finished_at = ?, + message = 'WALG backup needs attention', output_json = ? + WHERE id = ?` + ) + .run(completedAt, completedAt, JSON.stringify({ backup }), run.id); + + expect(getCurrentBackupJob("walg")).toBeUndefined(); + expect(getPersistedBackupJob("walg")).toMatchObject(backup); + expect(() => queueManualBackup("walg")).toThrow( + "WALG backup needs attention" + ); + + await startTestScheduledExecutor(); + await expect(runScheduledJob("backup.walg")).resolves.toMatchObject({ + output: { backup }, + status: "failed", + }); + expect(getPersistedBackupJob("walg")).toMatchObject(backup); + await expect(clearPersistedBackupAttention("walg")).resolves.toMatchObject({ + code: backup.code, + endedAt: backup.endedAt, + id: backup.id, + startedAt: backup.startedAt, + status: backup.status, + stdout: backup.stdout, + type: backup.type, + }); + expect( + database .prepare( - "SELECT status FROM scheduled_job_runs WHERE job_id = 'backup.walg' ORDER BY id DESC LIMIT 1" + `SELECT cancellable, status + FROM job_executions + WHERE action_key = 'backup.clear-attention' + ORDER BY rowid DESC + LIMIT 1` ) - .get() as { status?: string } | undefined; - return row?.status === "failed"; + .get() + ).toEqual({ cancellable: 0, status: "success" }); + expect(getPersistedBackupJob("walg")).toBeUndefined(); + } finally { + database + .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") + .run(); + database.prepare("DELETE FROM scheduled_jobs WHERE id LIKE 'backup.%'").run(); + } + }); + + it("reports recovered backup failures instead of stale running snapshots", async () => { + const { getPersistedBackupJob, registerBackupScheduledJobs } = + await import("../src/services/backups.ts"); + const { + claimNextJobExecution, + recoverExpiredJobExecutions, + updateJobExecutionOutput, + } = await import("../src/services/jobExecutionQueue.ts"); + const { enqueueScheduledJob } = await import("../src/services/scheduledJobs.ts"); + + try { + registerBackupScheduledJobs(); + const run = enqueueScheduledJob("backup.walg", "manual"); + const executionId = run.executionId; + if (!executionId) throw new Error("Backup execution id was missing"); + const workerId = `backup-recovery-${Bun.randomUUIDv7()}`; + const startedAt = new Date(Date.now() + 1000).toISOString(); + const claimed = claimNextJobExecution(workerId, 1, startedAt, 1000); + expect(claimed?.id).toBe(executionId); + const backup = { + code: undefined, + endedAt: undefined, + id: Bun.randomUUIDv7(), + startedAt: Date.parse(startedAt), + status: "running", + stderr: "", + stdout: "backup started", + type: "walg", + }; + updateJobExecutionOutput(executionId, workerId, { backup }); + const recoveredAt = new Date(Date.parse(startedAt) + 2000).toISOString(); + + expect(recoverExpiredJobExecutions(recoveredAt)).toBe(1); + expect(getPersistedBackupJob("walg")).toMatchObject({ + id: backup.id, + endedAt: Date.parse(recoveredAt), + status: "failed", + stderr: "Job failed after its worker lease expired", + stdout: backup.stdout, + type: "walg", }); } finally { database @@ -3090,6 +3568,112 @@ fi } }); + it("does not clear a newer persisted backup than the requested execution", async () => { + const { + clearPersistedBackupAttention, + getPersistedBackupJob, + registerBackupScheduledJobs, + } = await import("../src/services/backups.ts"); + const { enqueueScheduledJob } = await import("../src/services/scheduledJobs.ts"); + + try { + registerBackupScheduledJobs(); + const oldRun = enqueueScheduledJob("backup.walg", "manual"); + const oldExecutionId = oldRun.executionId; + if (!oldExecutionId) throw new Error("Old backup execution id was missing"); + const oldBackup = { + code: 130, + endedAt: Date.parse("2026-07-22T02:00:00.000Z"), + id: Bun.randomUUIDv7(), + startedAt: Date.parse("2026-07-22T02:00:00.000Z"), + status: "needs_attention", + stderr: "Old backup needs attention", + stdout: "", + type: "walg", + }; + database + .prepare( + `UPDATE job_executions + SET status = 'failed', started_at = ?, finished_at = ?, + message = 'Old WALG backup needs attention', output_json = ? + WHERE id = ?` + ) + .run( + "2026-07-22T02:00:00.000Z", + "2026-07-22T02:00:00.000Z", + JSON.stringify({ backup: oldBackup }), + oldExecutionId + ); + database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'failed', started_at = ?, finished_at = ?, + message = 'Old WALG backup needs attention', output_json = ? + WHERE id = ?` + ) + .run( + "2026-07-22T02:00:00.000Z", + "2026-07-22T02:00:00.000Z", + JSON.stringify({ backup: oldBackup }), + oldRun.id + ); + + const clearPromise = clearPersistedBackupAttention("walg"); + const newerRun = enqueueScheduledJob("backup.walg", "manual"); + const newerExecutionId = newerRun.executionId; + if (!newerExecutionId) { + throw new Error("Newer backup execution id was missing"); + } + const newerBackup = { + ...oldBackup, + endedAt: Date.parse("2026-07-22T03:00:00.000Z"), + id: Bun.randomUUIDv7(), + startedAt: Date.parse("2026-07-22T03:00:00.000Z"), + stderr: "Newer backup needs attention", + }; + database + .prepare( + `UPDATE job_executions + SET status = 'failed', queued_at = ?, started_at = ?, + finished_at = ?, message = 'Newer WALG backup needs attention', + output_json = ? + WHERE id = ?` + ) + .run( + "2999-01-01T03:00:00.000Z", + "2999-01-01T03:00:00.000Z", + "2999-01-01T03:00:00.000Z", + JSON.stringify({ backup: newerBackup }), + newerExecutionId + ); + database + .prepare( + `UPDATE scheduled_job_runs + SET status = 'failed', started_at = ?, finished_at = ?, + message = 'Newer WALG backup needs attention', output_json = ? + WHERE id = ?` + ) + .run( + "2999-01-01T03:00:00.000Z", + "2999-01-01T03:00:00.000Z", + JSON.stringify({ backup: newerBackup }), + newerRun.id + ); + + await startTestScheduledExecutor(); + await expect(clearPromise).rejects.toMatchObject({ + message: "WALG backup attention changed before clearing", + statusCode: 409, + }); + expect(getPersistedBackupJob("walg")).toMatchObject(newerBackup); + } finally { + database + .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") + .run(); + database.prepare("DELETE FROM scheduled_jobs WHERE id LIKE 'backup.%'").run(); + } + }); + it("reports Kopia host pgrep failures without recording needs-attention state", async () => { rememberEnvironment("PATH"); const fakeBin = createTemporaryRoot("mira-backup-host-pgrep-error-bin-"); @@ -3110,17 +3694,6 @@ fi statusCode: 503, }); expect(getCurrentBackupJob("kopia")).toBeUndefined(); - await waitFor(() => { - const row = database - .prepare( - "SELECT status, message FROM scheduled_job_runs WHERE job_id = 'backup.kopia' ORDER BY id DESC LIMIT 1" - ) - .get() as { message?: string; status?: string } | undefined; - return ( - row?.status === "failed" && - row.message?.includes("pgrep unavailable") === true - ); - }); } finally { database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") @@ -3168,14 +3741,6 @@ fi expect(readFileSync(pgrepLog, "utf8")).toContain( "-f /opt/docker/apps/kopia/backup.sh" ); - await waitFor(() => { - const row = database - .prepare( - "SELECT status FROM scheduled_job_runs WHERE job_id = 'backup.kopia' ORDER BY id DESC LIMIT 1" - ) - .get() as { status?: string } | undefined; - return row?.status === "failed"; - }); } finally { database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'backup.%'") @@ -3773,6 +4338,7 @@ fi try { registerLogRotationScheduledJobs(); + await startTestScheduledExecutor(); const run = await runScheduledJob("ops.log-rotation"); expect(run.status).toBe("failed"); @@ -3830,6 +4396,7 @@ fi try { registerLogRotationScheduledJobs(); + await startTestScheduledExecutor(); const run = await runScheduledJob("ops.log-rotation"); expect(run.status).toBe("failed"); @@ -3895,9 +4462,11 @@ fi try { registerLogRotationScheduledJobs(); + await startTestScheduledExecutor(); const run = await runScheduledJob("ops.log-rotation"); expect(run.status).toBe("success"); + expect(run.cancellable).toBe(false); expect(run.message).toBeUndefined(); expect(run.output).toMatchObject({ logRotation: { @@ -4507,7 +5076,7 @@ fi }); it("starts, stops, and reports exec jobs through the service lifecycle", async () => { - const { getExecJob, startExecJob, stopExecJob } = + const { getExecJob, registerExecExecutionActions, startExecJob, stopExecJob } = await import("../src/services/execJobs.ts"); const exit = Promise.withResolvers(); const spawnSpy = jest.spyOn(processModule, "spawnProcess").mockImplementation( @@ -4523,6 +5092,8 @@ fi }) as unknown as processModule.BunProcess ); try { + registerExecExecutionActions(); + await startTestScheduledExecutor(); const { jobId } = startExecJob({ command: "__mira_dashboard_shell_smoke_test__", shell: true, @@ -4531,14 +5102,12 @@ fi jobId, status: "running", }); + await waitFor(() => spawnSpy.mock.calls.length === 1, 3000); expect(stopExecJob(jobId)).toEqual({ isSuccess: true, message: "Stop signal sent", }); - expect(getExecJob(jobId)).toMatchObject({ - jobId, - status: "signaled", - }); + await waitFor(() => getExecJob(jobId).status === "done", 3000); expect(() => stopExecJob(jobId)).toThrow("Job is not running"); } finally { exit.resolve(0); @@ -4547,8 +5116,8 @@ fi } }); - it("rejects new exec jobs while the active job cap is full", async () => { - const { execErrorResponse, startExecJob } = + it("serializes concurrent exec jobs through global execution capacity", async () => { + const { registerExecExecutionActions, startExecJob } = await import("../src/services/execJobs.ts"); const exits: Array>> = []; const spawnSpy = jest @@ -4564,37 +5133,33 @@ fi stdout: readableUtf8Stream(""), } as unknown as processModule.BunProcess; }); - const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); - try { - let capError: unknown; - for (let index = 0; !capError && index < 120; index += 1) { - try { - startExecJob({ - command: "__mira_dashboard_shell_smoke_test__", - shell: true, - }); - } catch (error) { - capError = error; - } - } - expect(execErrorResponse(capError)).toEqual({ - error: "Too many exec jobs", - status: 429, + registerExecExecutionActions(); + await startTestScheduledExecutor(); + const first = startExecJob({ + command: "__mira_dashboard_shell_smoke_test__", + shell: true, + }); + const second = startExecJob({ + command: "__mira_dashboard_shell_smoke_test__", + shell: true, }); - expect(warnSpy).toHaveBeenCalled(); + expect(first.jobId).not.toBe(second.jobId); + await waitFor(() => exits.length === 1, 3000); + exits[0]?.resolve(0); + await waitFor(() => exits.length === 2, 3000); + exits[1]?.resolve(0); } finally { for (const exit of exits) { exit.resolve(0); } await Bun.sleep(0); spawnSpy.mockRestore(); - warnSpy.mockRestore(); } }); it("records exec process failures and trims oversized output", async () => { - const { getExecJob, runExecOnce, startExecJob } = + const { getExecJob, registerExecExecutionActions, runExecOnce, startExecJob } = await import("../src/services/execJobs.ts"); const longOutput = `${"x".repeat(101_000)}tail`; const spawnSpy = jest.spyOn(processModule, "spawnProcess").mockImplementation( @@ -4609,6 +5174,8 @@ fi ); try { + registerExecExecutionActions(); + await startTestScheduledExecutor(); const once = await runExecOnce({ command: "__mira_dashboard_shell_smoke_test__", shell: true, @@ -4890,9 +5457,31 @@ fi hash: "hash-1", }); + const { registerOpenClawExecutionActions } = + await import("../src/services/openclawActions.ts"); + registerOpenClawExecutionActions(); + cleanupCallbacks.push(() => { + database + .prepare( + "DELETE FROM job_executions WHERE action_key = 'openclaw.gateway.restart'" + ) + .run(); + }); + await startTestScheduledExecutor(); const restartResponse = await openclawConfigRoutes["/api/restart"].POST(); expect(restartResponse.status).toBe(200); await expect(restartResponse.json()).resolves.toEqual({ isOk: true }); + expect( + database + .prepare( + `SELECT cancellable, status + FROM job_executions + WHERE action_key = 'openclaw.gateway.restart' + ORDER BY rowid DESC + LIMIT 1` + ) + .get() + ).toEqual({ cancellable: 0, status: "success" }); }); it("normalizes cron and session route contracts through a patched gateway", async () => { @@ -5260,6 +5849,10 @@ fi throw new Error("docker exec spawn failed"); }); try { + const { registerDockerExecutionActions } = + await import("../src/services/dockerActions.ts"); + registerDockerExecutionActions(); + await startTestScheduledExecutor(); const execStart = await dockerRoutes["/api/docker/exec/start"].POST( new Request("https://dashboard.test/api/docker/exec/start", { body: JSON.stringify({ @@ -5270,6 +5863,12 @@ fi }) ); const execStartBody = (await execStart.json()) as { jobId: string }; + await waitFor(() => { + const row = database + .prepare("SELECT status FROM job_executions WHERE id = ?") + .get(execStartBody.jobId) as { status?: string } | undefined; + return row?.status === "failed"; + }, 3000); const failedExec = dockerRoutes["/api/docker/exec/:jobId"].GET( Object.assign( new Request( @@ -5435,8 +6034,9 @@ fi ].join("\n") ); process.env.MIRA_DOCKER_APPS_ROOT = appsRoot; - const { registerDockerUpdaterServices } = + const { registerDockerUpdaterScheduledJobs, registerDockerUpdaterServices } = await import("../src/services/dockerUpdater.ts"); + registerDockerUpdaterScheduledJobs(); await expect(registerDockerUpdaterServices()).resolves.toMatchObject({ isOk: true, }); @@ -5479,7 +6079,7 @@ fi const scheduledDisabledId = `test-job-scheduled-disabled-${Bun.randomUUIDv7()}`; const { calculateNextRunAt, - finishScheduledJobRun, + enqueueScheduledJob, getScheduledJob, isScheduledJobValidationError, listScheduledJobs, @@ -5487,12 +6087,16 @@ fi registerScheduledJobAction, removeScheduledJobsNotInAction, runScheduledJob, - createManualScheduledJobRun, + startScheduledJobExecutor, + stopScheduledJobExecutor, updateScheduledJob, upsertScheduledJob, } = await import("../src/services/scheduledJobs.ts"); + const { cancelJobExecution } = + await import("../src/services/jobExecutionQueue.ts"); try { + startScheduledJobExecutor(); expect( calculateNextRunAt( { @@ -5638,11 +6242,11 @@ fi nextRunAt: undefined, }); - const manualRun = createManualScheduledJobRun(keepId); - expect(() => createManualScheduledJobRun(keepId)).toThrow( - "Scheduled job is already running" + const manualRun = enqueueScheduledJob(keepId); + expect(() => enqueueScheduledJob(keepId)).toThrow( + "Scheduled job is already queued or running" ); - finishScheduledJobRun(manualRun, "success", undefined, { manual: true }); + cancelJobExecution(manualRun.executionId as string); const result = await runScheduledJob(keepId); expect(result).toMatchObject({ @@ -5742,11 +6346,16 @@ fi const timeoutActionKey = `test-timeout-action-${Bun.randomUUIDv7()}`; const timeoutJobId = `test-job-timeout-${Bun.randomUUIDv7()}`; + let isTimeoutHandlerSettled = false; registerScheduledJobAction( timeoutActionKey, async () => { - await Bun.sleep(50); - return { late: true }; + try { + await Bun.sleep(50); + return { late: true }; + } finally { + isTimeoutHandlerSettled = true; + } }, { timeoutMs: 1 } ); @@ -5765,6 +6374,7 @@ fi output: {}, status: "failed", }); + expect(isTimeoutHandlerSettled).toBe(true); } finally { warnSpy.mockRestore(); } @@ -5783,13 +6393,9 @@ fi controller.abort(); await expect( runScheduledJob(abortJobId, "manual", controller.signal) - ).resolves.toMatchObject({ - jobId: abortJobId, - message: "Scheduled job aborted", - output: {}, - status: "failed", - }); + ).rejects.toHaveProperty("name", "AbortError"); } finally { + await stopScheduledJobExecutor(); database .prepare("DELETE FROM scheduled_job_runs WHERE job_id LIKE 'test-job-%'") .run(); diff --git a/backend/test/utilityBehavior.test.ts b/backend/test/utilityBehavior.test.ts index 624aaac99..dae6b20b7 100644 --- a/backend/test/utilityBehavior.test.ts +++ b/backend/test/utilityBehavior.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import type { Server } from "bun"; -import { describe, expect, it } from "bun:test"; +import { describe, expect, it, jest } from "bun:test"; import { isAllowedDashboardOrigin, @@ -30,9 +30,11 @@ import { stringFallback, } from "../src/lib/values.ts"; import { resetRequestPolicyForTests, withRequestPolicy } from "../src/requestPolicy.ts"; +import { routes as appRoutes } from "../src/routes.ts"; import { compactHeartbeatData } from "../src/routes/cacheRoutes.ts"; import { isValidAgentId } from "../src/services/agents.ts"; import { mapBackupJob } from "../src/services/backups.ts"; +import * as jobExecutionQueueModule from "../src/services/jobExecutionQueue.ts"; import { getResolvedRoots, validatePrNumber } from "../src/services/pullRequests.ts"; function serverWithAddress(address: string): Server { @@ -46,16 +48,19 @@ function canonicalPath(value: string): string { } async function callTestRoute( - routes: Record< - string, - (request: Request, server: Server) => Response | Promise - >, + routes: Record, path: string, server: Server ): Promise { - const handler = routes[path]; - if (!handler) { - throw new Error(`Missing test route: ${path}`); + const entry = routes[path]; + const handler = + typeof entry === "function" + ? entry + : typeof entry === "object" && entry !== null && "GET" in entry + ? entry.GET + : undefined; + if (typeof handler !== "function") { + throw new TypeError(`Missing test route: ${path}`); } return handler(new Request(`http://localhost${path}`), server); } @@ -621,6 +626,35 @@ describe("backend service utilities", () => { ); }); + it("keeps health available when worker telemetry cannot be read", async () => { + const summarySpy = jest + .spyOn(jobExecutionQueueModule, "getJobExecutionSummary") + .mockImplementation(() => { + throw new Error("queue telemetry unavailable"); + }); + const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); + try { + const response = await callTestRoute( + appRoutes, + "/api/health", + serverWithAddress("127.0.0.1") + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + status: "isOk", + workerOnline: false, + }); + expect(warnSpy).toHaveBeenCalledWith( + "[Health] Failed to read job worker telemetry:", + expect.objectContaining({ message: "queue telemetry unavailable" }) + ); + } finally { + summarySpy.mockRestore(); + warnSpy.mockRestore(); + } + }); + it("applies request policy auth, rate limit, and handler error behavior", async () => { resetRequestPolicyForTests(); try { @@ -637,6 +671,11 @@ describe("backend service utilities", () => { "/generic-error": () => { throw new Error("boom"); }, + "/status-error": () => { + throw Object.assign(new Error("Job capacity is full"), { + statusCode: 409, + }); + }, }; const routes = withRequestPolicy(routeEntries); const server = serverWithAddress("203.0.113.10"); @@ -657,6 +696,12 @@ describe("backend service utilities", () => { error: "Invalid JSON", }); + const statusResponse = await callTestRoute(routes, "/status-error", server); + expect(statusResponse.status).toBe(409); + await expect(statusResponse.json()).resolves.toEqual({ + error: "Job capacity is full", + }); + const authRequest = new Request("http://localhost/api/auth/login"); const authLogin = routes["/api/auth/login"]; if (!authLogin) { diff --git a/bun.lock b/bun.lock index 8c89a3181..ef92c2348 100644 --- a/bun.lock +++ b/bun.lock @@ -10,23 +10,23 @@ "@dnd-kit/react": "^0.5.0", "@dnd-kit/sortable": "^10.0.0", "@headlessui/react": "^2.2.10", - "@microlink/react-json-view": "^1.31.22", + "@microlink/react-json-view": "^1.31.23", "@tailwindcss/typography": "^0.5.20", - "@tanstack/query-core": "5.101.2", - "@tanstack/query-db-collection": "^1.1.0", + "@tanstack/query-core": "5.101.4", + "@tanstack/query-db-collection": "^1.2.0", "@tanstack/react-db": "^0.1.94", "@tanstack/react-form": "^1.33.2", - "@tanstack/react-query": "^5.101.2", + "@tanstack/react-query": "^5.101.4", "@tanstack/react-router": "^1.170.18", "@tanstack/react-store": "^0.11.0", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.14.6", + "@tanstack/react-virtual": "^3.14.8", "clsx": "^2.1.1", "date-fns": "^4.4.0", "json5": "^2.2.3", "lucide-react": "^1.25.0", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", "react-error-boundary": "^6.1.2", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", @@ -39,13 +39,13 @@ "devDependencies": { "@babel/core": "^8.0.1", "@eslint/js": "^10.0.1", - "@happy-dom/global-registrator": "^20.11.0", - "@tanstack/react-devtools": "^0.10.8", + "@happy-dom/global-registrator": "^20.11.1", + "@tanstack/react-devtools": "^0.10.9", "@tanstack/react-form-devtools": "^0.2.31", - "@tanstack/react-query-devtools": "^5.101.2", + "@tanstack/react-query-devtools": "^5.101.4", "@tanstack/react-router-devtools": "1.167.0", "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/babel__core": "^7.20.5", @@ -68,11 +68,11 @@ "eslint-plugin-tailwindcss": "^4.2.0", "eslint-plugin-unicorn": "^72.0.0", "globals": "^17.7.0", - "happy-dom": "^20.11.0", - "prettier": "^3.9.5", + "happy-dom": "^20.11.1", + "prettier": "^3.9.6", "tailwindcss": "^4.3.3", "typescript": "^6.0.3", - "typescript-eslint": "^8.64.0", + "typescript-eslint": "^8.65.0", }, }, }, @@ -165,7 +165,7 @@ "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], - "@happy-dom/global-registrator": ["@happy-dom/global-registrator@20.11.0", "", { "dependencies": { "@types/node": ">=20.0.0", "happy-dom": "^20.11.0" } }, "sha512-rHb/2dOy3APuHKNU4aqPQt+HbEt6BkulTTyCks89z5a+EipJecnFFd+3cz3meaFfCOXWsoxplPTLrGjFIEhd0A=="], + "@happy-dom/global-registrator": ["@happy-dom/global-registrator@20.11.1", "", { "dependencies": { "@types/node": ">=20.0.0", "happy-dom": "^20.11.1" } }, "sha512-1C0wwHiyMlEdsbrJqff5ORE2PwaXBOam5rMqkZVOReAezd5nt6Tl/WGg28UAo75Ziuwchs0KaF8lvnn2oAtvJA=="], "@headlessui/react": ["@headlessui/react@2.2.10", "", { "dependencies": { "@floating-ui/react": "^0.26.16", "@react-aria/focus": "^3.20.2", "@react-aria/interactions": "^3.25.0", "@tanstack/react-virtual": "^3.13.9", "use-sync-external-store": "^1.5.0" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, "sha512-5pVLNK9wlpxTUTy9GpgbX/SdcRh+HBnPktjM2wbiLTH4p+2EPHBO1aoSryUCuKUIItdDWO9ITlhUL8UnUN/oIA=="], @@ -187,7 +187,7 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], - "@microlink/react-json-view": ["@microlink/react-json-view@1.31.22", "", { "dependencies": { "react-base16-styling": "~0.10.0", "react-lifecycles-compat": "~3.0.4", "react-textarea-autosize": "~8.5.9" }, "peerDependencies": { "react": ">= 15", "react-dom": ">= 15" } }, "sha512-cfszidqClRqjnF5YQ5PSGeORo9MnMZvDv+csgq92G1LRZduWEBjeHXKLnulvhE7bX9+ORvjRCWcMQfwRlhPxmA=="], + "@microlink/react-json-view": ["@microlink/react-json-view@1.31.23", "", { "dependencies": { "react-base16-styling": "~0.10.0", "react-lifecycles-compat": "~3.0.4", "react-textarea-autosize": "~8.5.9" }, "peerDependencies": { "react": ">= 15", "react-dom": ">= 15" } }, "sha512-Qjt7uUpYPDAQiu0LgxyRSuwAgMBwIXtJZ6HSbjTAx6TXu0DdZ/BVIFD5A27z/NcXS2q2zVQgAIxut3WbUdDwRg=="], "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.14", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Omj20SuiHBOUjUBIyqtkNjSUIjOtEOJwmbix/ZyFH4BaQ6OZTaaRWIR4TjHVz0yadHgli6lLTiAh1uarnvD49A=="], @@ -261,7 +261,7 @@ "@tanstack/db-ivm": ["@tanstack/db-ivm@0.1.18", "", { "dependencies": { "fractional-indexing": "^3.2.0", "sorted-btree": "^1.8.1" }, "peerDependencies": { "typescript": ">=4.7" } }, "sha512-+pZJiRKdoKRM5Epq9T7otD9ZJl82pRFauo7LKuJGrarjVKQ7r+QQlPe3kGdN9LEKSnuNGIWjX9OOY4M8kH4eLw=="], - "@tanstack/devtools": ["@tanstack/devtools@0.12.5", "", { "dependencies": { "@solid-primitives/event-listener": "^2.4.3", "@solid-primitives/keyboard": "^1.3.3", "@solid-primitives/resize-observer": "^2.1.3", "@tanstack/devtools-client": "0.0.8", "@tanstack/devtools-event-bus": "0.4.2", "@tanstack/devtools-ui": "0.6.0", "clsx": "^2.1.1", "goober": "^2.1.16", "solid-js": "^1.9.9" }, "bin": { "intent": "./bin/intent.js" } }, "sha512-JdxTSeVdjJheycgz4c7qbldNKDCEDWlWr1l9dZBhd9sOmRBT5Z70ka9Eb8mb+FUnalcOIB62IDSR/iSxAIUD8Q=="], + "@tanstack/devtools": ["@tanstack/devtools@0.13.0", "", { "dependencies": { "@solid-primitives/event-listener": "^2.4.3", "@solid-primitives/keyboard": "^1.3.3", "@solid-primitives/resize-observer": "^2.1.3", "@tanstack/devtools-client": "0.0.8", "@tanstack/devtools-event-bus": "0.4.2", "@tanstack/devtools-ui": "0.6.0", "clsx": "^2.1.1", "goober": "^2.1.16", "solid-js": "^1.9.9" }, "bin": { "intent": "./bin/intent.js" } }, "sha512-p/nOH9bS/OO/u3402zPjoGu+Mz6Fzi/iRqJuYghuuYRUY32kZt+C0/d+pP/bi6/2JTi1FdT6oEXI2lWlA5tXxw=="], "@tanstack/devtools-client": ["@tanstack/devtools-client@0.0.8", "", { "dependencies": { "@tanstack/devtools-event-client": "^0.5.0" } }, "sha512-cG3iZkGWCwN330bLBKa8+9r4Of2AXNoz2zUqcsy/4XsD3105ghVBx78cGyvJj9fSclNomPxoqAnDGXXhg1WLvA=="], @@ -281,23 +281,23 @@ "@tanstack/pacer-lite": ["@tanstack/pacer-lite@0.2.1", "", {}, "sha512-3PouiFjR4B6x1c969/Pl4ZIJleof1M0n6fNX8NRiC9Sqv1g06CVDlEaXUR4212ycGFyfq4q+t8Gi37Xy+z34iQ=="], - "@tanstack/query-core": ["@tanstack/query-core@5.101.2", "", {}, "sha512-hH5MLoJhF7KaIGd7q3xTXGXvslI+GYlM1Z/35aSHHWaCJWB7XvTSHYuV3eM7tw+aE0mT/xMro4M4Q9rCGHT0lw=="], + "@tanstack/query-core": ["@tanstack/query-core@5.101.4", "", {}, "sha512-gNwcvOJcRbLWPOLG/2OBm+zM+Yv+MKsXKEOWC57USuZDEsI71hEErQsiEGx5wX9rzWWkfwM0fVSPoiIFSsxfiw=="], - "@tanstack/query-db-collection": ["@tanstack/query-db-collection@1.1.0", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@tanstack/db": "0.6.16" }, "peerDependencies": { "@tanstack/query-core": "^5.0.0", "typescript": ">=4.7" } }, "sha512-duggStGqINUX9dqq7+1JDsq4utNn/XExgtna1ACTO+WueqOmDyh83Lu2XUf+9457m9ozlbZvIFq2bcxdTXhx4Q=="], + "@tanstack/query-db-collection": ["@tanstack/query-db-collection@1.2.0", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@tanstack/db": "0.6.16" }, "peerDependencies": { "@tanstack/query-core": "^5.0.0", "typescript": ">=4.7" } }, "sha512-WPVj8TW5/qtGWkvBz/ru/odYm+QnD6ZuWmk59xD2biq2T+/cIssRgHuNIOvVw4texGrUiROzF8NUcuNICijXkg=="], - "@tanstack/query-devtools": ["@tanstack/query-devtools@5.101.2", "", {}, "sha512-o+wHcqgN7Pp0s8v1i0UGq/ZrrEKrxdIiMQmKRdYb2w7NPtylYSJ4+wg/tIn71m9DLstwUwdEGAvROdly6HXP6w=="], + "@tanstack/query-devtools": ["@tanstack/query-devtools@5.101.4", "", {}, "sha512-z5IPHnDX3aUWeTWlRKLyooBQekaCAw4xRpZqPQ390RiWTDBcTynjpPT221BArw0u2+pnQMdGvPQI9YNNubBcmA=="], "@tanstack/react-db": ["@tanstack/react-db@0.1.94", "", { "dependencies": { "@tanstack/db": "0.6.16", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "react": ">=16.8.0" } }, "sha512-ZtSMBOCdR/ElcBxg84uboXrqQ+kqiOwlq6kTy+o54UDywYr/by02jmWKHqayrnoiPVlrx83WNHDaomfe141f4A=="], - "@tanstack/react-devtools": ["@tanstack/react-devtools@0.10.8", "", { "dependencies": { "@tanstack/devtools": "0.12.5" }, "peerDependencies": { "@types/react": ">=16.8", "@types/react-dom": ">=16.8", "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-YJV6YttQf9lhhPbPBLULgy1eScEvJUMsCS26mjg9hfBKgAJQA5sF9zvzorDzW9Ob6o/asoXikO81JnRUVuFX0Q=="], + "@tanstack/react-devtools": ["@tanstack/react-devtools@0.10.9", "", { "dependencies": { "@tanstack/devtools": "0.13.0" }, "peerDependencies": { "@types/react": ">=16.8", "@types/react-dom": ">=16.8", "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-lS6mtccEmUaodsWiRORGM/MGKT0jgzcy5v+eY6pzOPxEgzTHUDhca+WGxShFqKxmF4oneRxXjww1gkvMrWq6uw=="], "@tanstack/react-form": ["@tanstack/react-form@1.33.2", "", { "dependencies": { "@tanstack/form-core": "1.33.2", "@tanstack/react-store": "^0.11.0" }, "peerDependencies": { "react": "^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-nEfayOu+27q5cZ5E0G5dmnddqLcLjdFCatbL/LCs/iLD469a1o1yYJlr8RISV3GfnqsBpm0hf+8kM4okh5fPCw=="], "@tanstack/react-form-devtools": ["@tanstack/react-form-devtools@0.2.31", "", { "dependencies": { "@tanstack/devtools-utils": "^0.4.0", "@tanstack/form-devtools": "0.2.31" }, "peerDependencies": { "react": "^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-p++EPC01hmadxfycUO+WyKWWeVgdAjKed7/58T4up4d58taGLIum2aSCpGyouSnnl3dMWWurs3LHCYorIOUDBA=="], - "@tanstack/react-query": ["@tanstack/react-query@5.101.2", "", { "dependencies": { "@tanstack/query-core": "5.101.2" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-seDkr6kzGzX1okaaTtZPtgA688CDPlXUz1C6xSg0ESqn04Vuc8tlrYms1s3de+znBqhPVxFRfpAfUf+6XvfPWg=="], + "@tanstack/react-query": ["@tanstack/react-query@5.101.4", "", { "dependencies": { "@tanstack/query-core": "5.101.4" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-yRg2pfOCxIs4ZJW3XYYHU/WgtD04FHSnfHlpRT7h7pR77hwkdRG4wxbKe4aq6P0RvXUTBSQpQeadS1SUYUe+KA=="], - "@tanstack/react-query-devtools": ["@tanstack/react-query-devtools@5.101.2", "", { "dependencies": { "@tanstack/query-devtools": "5.101.2" }, "peerDependencies": { "@tanstack/react-query": "^5.101.2", "react": "^18 || ^19" } }, "sha512-eU7HctdA9gDjqoERoEdzLbw9DiqnBDfh5+Hu0u26gjqoHJezOpQAuiesDL2VvkU+2cPV76zgv0tMZsOrI4LjnQ=="], + "@tanstack/react-query-devtools": ["@tanstack/react-query-devtools@5.101.4", "", { "dependencies": { "@tanstack/query-devtools": "5.101.4" }, "peerDependencies": { "@tanstack/react-query": "^5.101.4", "react": "^18 || ^19" } }, "sha512-VeK2gtmfj7kvRBjtxS7TKxt/6qKhn8VzabY4UiYMr7NV9CddjSRYRgeYyld+NpjAkgMV9dd+2Qdr8ah5I03NeA=="], "@tanstack/react-router": ["@tanstack/react-router@1.170.18", "", { "dependencies": { "@tanstack/history": "1.162.0", "@tanstack/react-store": "^0.9.3", "@tanstack/router-core": "1.171.15", "isbot": "^5.1.22" }, "peerDependencies": { "react": ">=18.0.0 || >=19.0.0", "react-dom": ">=18.0.0 || >=19.0.0" } }, "sha512-wpbGYZEp/fmz1q4bn7BD8VZ+/VZ7GBqSJv5V969pU+chP8y7dquWDmKTFMohvUegb9lg12m1uPVvD6kB2wORvQ=="], @@ -307,7 +307,7 @@ "@tanstack/react-table": ["@tanstack/react-table@8.21.3", "", { "dependencies": { "@tanstack/table-core": "8.21.3" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-5nNMTSETP4ykGegmVkhjcS8tTLW6Vl4axfEGQN3v0zdHYbK4UfoqfPChclTrJ4EoK9QynqAu9oUf8VEmrpZ5Ww=="], - "@tanstack/react-virtual": ["@tanstack/react-virtual@3.14.6", "", { "dependencies": { "@tanstack/virtual-core": "3.17.4" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-4+Uq8m0/gzO4kMCHUEpTtGX1RnONK0C+g88b2ltwPMWUBiaVarBuWKoPJaz7gj1cKCVRAdyu+U8GcKhwCc2beA=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.14.8", "", { "dependencies": { "@tanstack/virtual-core": "3.17.6" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-O39GJQpAYEJcIu3uN1//YtmhjSEOyw75vg9CKCatBDPiD5hKtZQoJHfferyrB/LdOD3UWaoMLWtdEjarwIwdDw=="], "@tanstack/router-core": ["@tanstack/router-core@1.171.13", "", { "dependencies": { "@tanstack/history": "1.162.0", "cookie-es": "^3.0.0", "seroval": "^1.5.4", "seroval-plugins": "^1.5.4" } }, "sha512-+NOwEj1kO/6IGmpHRIZHasYxYWpyBQGNIZAST9aNrk9Q3YlU9SgqVnl1pbLa9qAKfeNdXQIRve0RQb/0kyDeDA=="], @@ -317,11 +317,11 @@ "@tanstack/table-core": ["@tanstack/table-core@8.21.3", "", {}, "sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg=="], - "@tanstack/virtual-core": ["@tanstack/virtual-core@3.17.4", "", {}, "sha512-nGm5KteqxasUdThLc2izl6dHUqLv0LQj7Nuyo5gYalTPf/U8a9ermvsl7reT+6ioBW1l8WfpP/mcU338nLXpqw=="], + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.17.6", "", {}, "sha512-h0/Ebo18CkOrChlQIhNtQkM5ySUnh/GumQ/D1st3hG2HWUPEF+ILUc2k29UtivCi/9G7w7G3/f7Xyd5cCFbKBw=="], "@testing-library/dom": ["@testing-library/dom@10.4.1", "", { "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", "picocolors": "1.1.1", "pretty-format": "^27.0.2" } }, "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg=="], - "@testing-library/jest-dom": ["@testing-library/jest-dom@6.9.1", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" } }, "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA=="], + "@testing-library/jest-dom": ["@testing-library/jest-dom@7.0.0", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" }, "peerDependencies": { "@testing-library/dom": ">=10 <11" } }, "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg=="], "@testing-library/react": ["@testing-library/react@16.3.2", "", { "dependencies": { "@babel/runtime": "^7.12.5" }, "peerDependencies": { "@testing-library/dom": "^10.0.0", "@types/react": "^18.0.0 || ^19.0.0", "@types/react-dom": "^18.0.0 || ^19.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g=="], @@ -377,25 +377,25 @@ "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], - "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.64.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/type-utils": "8.64.0", "@typescript-eslint/utils": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.64.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-CGvQPBxN3wZLu6Rz2kFUpZeoCm78xUic92ck39KPePkO1NPOwjCqdQnm5Q87tpWw9vcBvW8XLrDXjH9PWYtJ3Q=="], + "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.65.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/type-utils": "8.65.0", "@typescript-eslint/utils": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.65.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-IEgob78X12rHpUmtcwFsXhZdVGJtwTVP8FiCLZkR6GlYVrl2PcuB+KhCE5BlVC/eQpQnu8WXRtkHZuPar+gCRA=="], - "@typescript-eslint/parser": ["@typescript-eslint/parser@8.64.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-KA0OshtlcCCXmbfqyZkM5pV3/WNraJf7DkJRLpyrmwPtud57H5BDX7C3k0LPSPxpprfRL+cJDGabF10mvNCoCw=="], + "@typescript-eslint/parser": ["@typescript-eslint/parser@8.65.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.64.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.64.0", "@typescript-eslint/types": "^8.64.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-tk4WpOJ6IEbGrVHaNmM0YRrwAD3exZlIK3iadQNAxh4YKk6jvUQ4ecq18n+v7+meh+cJ3j+D8nbk8sRKhlwLQg=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.65.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.65.0", "@typescript-eslint/types": "^8.65.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q=="], "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "@typescript-eslint/visitor-keys": "8.63.0" } }, "sha512-uUyfMWCnDSN8bCpcrY8nGP2BLkQ9Xn0GsipcONcpIDWhwhO4ZSyHvyS14U3X75mzxWxL3I2UZIrenTzdzcJO8A=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.64.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-2yo8rRNKuzbVWQp5kslhANqZ2uDAeROQHBRZNPu8JDsHmeFNj/XJJhX/FhNUWmkHHvoNsKa6+tHJiig87EzsQw=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.65.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg=="], - "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/utils": "8.64.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-XWG4Fmmv/6SvyS9nH8jWrKs6terwJvE8cyRt1CzYYqzp9OrPhCT4cMc/f7C6RZCwG+qMmiffJS1/qJP8G1URtg=="], + "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/utils": "8.65.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-YjaZ7PRI5qY7ax2L3PbvX0rRyGtipAReCWs0mhhDBHjH/vl0g0BonaGXrKdKpMbIIsMIwDgbk/xzkBTyAltS5g=="], "@typescript-eslint/types": ["@typescript-eslint/types@8.63.0", "", {}, "sha512-xyLtl9DUBBFrcJS4x2pIqGLH68/tC2uOa4Z7pUteW09D3bXnnXUom4dyPikzWgB7llmIc1zoeI3aoUdC4rPK/Q=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.64.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.64.0", "@typescript-eslint/tsconfig-utils": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-Pztpsn1aCE1oWDvDEfUk31nngvvF7vUB5SwHFEaZIFpvw7WJtqUHHL4plBZDA9HfWJJjL13BdG0YrJInTUvoVA=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.65.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.65.0", "@typescript-eslint/tsconfig-utils": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg=="], "@typescript-eslint/utils": ["@typescript-eslint/utils@8.63.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.63.0", "@typescript-eslint/types": "8.63.0", "@typescript-eslint/typescript-estree": "8.63.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-fUKaeAvrTuQg/Tgt3nliAUSZHJM6DlCcfyEmxCvlX8kieWSStBX+5O5Fnidtc3i2JrH+9c/GL4RY2iasd/GPTA=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-mrtuL8Nsn6gi2H4mo5KMTp823M+3Q19Ew/i+Zlikq20tIMm99C3Ez0dCmkWWnxut20esQvTg8aUSEhMcAOXhEw=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-8C71BQkGjiMmXtop7pHVJu1l2NNShFdkCyD6a2ezzs5vU/L3LRtb69EtcteFwz0mYMPzIgOw0n6OV4VBUWZd7A=="], "@typescript/native": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], @@ -485,7 +485,7 @@ "baseline-browser-mapping": ["baseline-browser-mapping@2.10.42", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-c/jurFrDLyui7o1J86yLkRu4LMsTYcBohveus7/I2Hzdn9KIP2bdJPTue/lR1KH46enoPbD77GKeSYNdyPoD3Q=="], - "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], + "brace-expansion": ["brace-expansion@5.0.7", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA=="], "browserslist": ["browserslist@4.28.4", "", { "dependencies": { "baseline-browser-mapping": "^2.10.38", "caniuse-lite": "^1.0.30001799", "electron-to-chromium": "^1.5.376", "node-releases": "^2.0.48", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw=="], @@ -715,7 +715,7 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "happy-dom": ["happy-dom@20.11.0", "", { "dependencies": { "@types/node": ">=20.0.0", "@types/whatwg-mimetype": "^3.0.2", "@types/ws": "^8.18.1", "buffer-image-size": "^0.6.4", "entities": "^7.0.1", "whatwg-mimetype": "^3.0.0", "ws": "^8.21.0" } }, "sha512-XogN4asPd1a56di9prVG6bZxteNcXsZxxKmAvcEfc5Px5Ca2hMyMgk8wvqK2K1V8zXg40j9VANXsDaJYh9DeNA=="], + "happy-dom": ["happy-dom@20.11.1", "", { "dependencies": { "@types/node": ">=20.0.0", "@types/whatwg-mimetype": "^3.0.2", "@types/ws": "^8.18.1", "buffer-image-size": "^0.6.4", "entities": "^7.0.1", "whatwg-mimetype": "^3.0.0", "ws": "^8.21.0" } }, "sha512-XSt8tMzbW9ymE7687xztkO1ckR7qJNQ3LywY9vlYGhGi3zXrGBHuUo2Cl1ztZaICW+1eAGdkLbj6iwVqDT33kg=="], "has-bigints": ["has-bigints@1.1.0", "", {}, "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg=="], @@ -1057,7 +1057,7 @@ "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], - "prettier": ["prettier@3.9.5", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg=="], + "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], "prettier-linter-helpers": ["prettier-linter-helpers@1.0.1", "", { "dependencies": { "fast-diff": "^1.1.2" } }, "sha512-SxToR7P8Y2lWmv/kTzVLC1t/GDI2WGjMwNhLLE9qtH8Q13C+aEmuRlzDst4Up4s0Wc8sF2M+J57iB3cMLqftfg=="], @@ -1075,13 +1075,13 @@ "quote-js-string": ["quote-js-string@0.1.0", "", {}, "sha512-Y3NoRtprEEZQD8RfxMCfS0ZTqc4e+i18OrXEXAvpM6TfC/3y+0L5rNbZiSnbBBEkDfFzbpd8o+cE8q3/anjMGA=="], - "react": ["react@19.2.7", "", {}, "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ=="], + "react": ["react@19.2.8", "", {}, "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw=="], "react-base16-styling": ["react-base16-styling@0.10.0", "", { "dependencies": { "@types/lodash": "^4.17.0", "color": "^4.2.3", "csstype": "^3.1.3", "lodash-es": "^4.17.21" } }, "sha512-H1k2eFB6M45OaiRru3PBXkuCcn2qNmx+gzLb4a9IPMR7tMH8oBRXU5jGbPDYG1Hz+82d88ED0vjR8BmqU3pQdg=="], "react-day-picker": ["react-day-picker@10.0.1", "", { "dependencies": { "@date-fns/tz": "^1.4.1", "date-fns": "^4.1.0" }, "peerDependencies": { "@types/react": ">=16.8.0", "react": ">=16.8.0" }, "optionalPeers": ["@types/react"] }, "sha512-eNh6BlwcYInWaJtRv18mXQ06Ys/H6rdTZAnTaSdOYJuTpwP1JMCHNd1FDRadA+gbeinq+psdULN5Xnowy9mV8w=="], - "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + "react-dom": ["react-dom@19.2.8", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.8" } }, "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ=="], "react-error-boundary": ["react-error-boundary@6.1.2", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-3DpCr5HVdZ0caUjYE/kIHBEJN0mNP3ZCgf16c48uJ5TbWjorKVp+YG8W3XqlJ7vJAVNw6wNIImyPXmFydwmyng=="], @@ -1229,7 +1229,7 @@ "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], - "typescript-eslint": ["typescript-eslint@8.64.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.64.0", "@typescript-eslint/parser": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0", "@typescript-eslint/utils": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-0qg+pDNMnqYzqH9AnNK+39tejHvsShUOUUoRUgtnTGE7QuMZhiFDnozq8nHJVq+Wae6NMLKNWLg5WmkcC/ndyQ=="], + "typescript-eslint": ["typescript-eslint@8.65.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.65.0", "@typescript-eslint/parser": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0", "@typescript-eslint/utils": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-/ggrHAwyjENDusvyxbuqxAC2dTnZg/Z8F+fgQtYIz+L6n/9HfSlEZcFGV/NsMNa6CkGk0xUjUAFwC0vHOflvIA=="], "ufo": ["ufo@1.6.4", "", {}, "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA=="], @@ -1345,31 +1345,31 @@ "@types/ws/@types/node": ["@types/node@25.9.3", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], - "@typescript-eslint/eslint-plugin/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0" } }, "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w=="], + "@typescript-eslint/eslint-plugin/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0" } }, "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg=="], - "@typescript-eslint/eslint-plugin/@typescript-eslint/utils": ["@typescript-eslint/utils@8.64.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ=="], + "@typescript-eslint/eslint-plugin/@typescript-eslint/utils": ["@typescript-eslint/utils@8.65.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA=="], "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], - "@typescript-eslint/parser/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0" } }, "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w=="], + "@typescript-eslint/parser/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0" } }, "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg=="], - "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], - "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], "@typescript-eslint/scope-manager/@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-UexrHGnGTpbuQHct2ExOc2ZcFbGUS9FOesCxxqdBGcpI1BxYu/LZ6U8Aq6/72XtF/qRBk9nhuGHFJIXXMhPMdw=="], - "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], - "@typescript-eslint/type-utils/@typescript-eslint/utils": ["@typescript-eslint/utils@8.64.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ=="], + "@typescript-eslint/type-utils/@typescript-eslint/utils": ["@typescript-eslint/utils@8.65.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA=="], - "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], "@typescript-eslint/utils/@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.63.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.63.0", "@typescript-eslint/tsconfig-utils": "8.63.0", "@typescript-eslint/types": "8.63.0", "@typescript-eslint/visitor-keys": "8.63.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-ygBkU+B7ex5UI/gKhaqexWev79uISfIv7XQCRNYO/jmD8rGLPyWLAb3KMRT6nd8Gt9bmUBi9+iX6tBdYfOY81Q=="], - "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], "babel-plugin-react-compiler/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], @@ -1419,7 +1419,7 @@ "redent/strip-indent": ["strip-indent@3.0.0", "", { "dependencies": { "min-indent": "^1.0.0" } }, "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ=="], - "typescript-eslint/@typescript-eslint/utils": ["@typescript-eslint/utils@8.64.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.64.0", "@typescript-eslint/types": "8.64.0", "@typescript-eslint/typescript-estree": "8.64.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ=="], + "typescript-eslint/@typescript-eslint/utils": ["@typescript-eslint/utils@8.65.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/typescript-estree": "8.65.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA=="], "@babel/helper-compilation-targets/browserslist/baseline-browser-mapping": ["baseline-browser-mapping@2.10.27", "", { "bin": "dist/cli.cjs" }, "sha512-zEs/ufmZoUd7WftKpKyXaT6RFxpQ5Qm9xytKRHvJfxFV9DFJkZph9RvJ1LcOUi0Z1ZVijMte65JbILeV+8QQEA=="], @@ -1481,11 +1481,11 @@ "@types/ws/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], - "@typescript-eslint/eslint-plugin/@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/eslint-plugin/@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], - "@typescript-eslint/eslint-plugin/@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "@typescript-eslint/eslint-plugin/@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], - "@typescript-eslint/type-utils/@typescript-eslint/utils/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0" } }, "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w=="], + "@typescript-eslint/type-utils/@typescript-eslint/utils/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0" } }, "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg=="], "@typescript-eslint/utils/@typescript-eslint/typescript-estree/@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.63.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.63.0", "@typescript-eslint/types": "^8.63.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-e5dh0/UI0ok53AlZ5wRkXCB32z/f2jUZqPR/ygAw5WYaSw8j9EoJWlS7wQjr/dmOaqWjnPIn2m+HhVPCMWGZVQ=="], @@ -1511,7 +1511,7 @@ "core-js-compat/browserslist/node-releases": ["node-releases@2.0.38", "", {}, "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw=="], - "eslint-plugin-jsx-a11y/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], + "eslint-plugin-jsx-a11y/minimatch/brace-expansion": ["brace-expansion@1.1.16", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw=="], "eslint-plugin-react-hooks/@babel/core/@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], @@ -1531,13 +1531,13 @@ "eslint-plugin-react-hooks/@babel/parser/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], - "eslint-plugin-react/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], + "eslint-plugin-react/minimatch/brace-expansion": ["brace-expansion@1.1.16", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw=="], "mlly/pkg-types/confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], - "typescript-eslint/@typescript-eslint/utils/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.64.0", "", { "dependencies": { "@typescript-eslint/types": "8.64.0", "@typescript-eslint/visitor-keys": "8.64.0" } }, "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w=="], + "typescript-eslint/@typescript-eslint/utils/@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0" } }, "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg=="], - "typescript-eslint/@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.64.0", "", {}, "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA=="], + "typescript-eslint/@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], "@babel/helper-module-imports/@babel/traverse/@babel/code-frame/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 2065dacdd..0071f0281 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -7,8 +7,8 @@ route files under `backend/src/routes/` for exact validation details. | Method | Path | Purpose | | ------ | --------------- | ------------------------------------- | -| `GET` | `/health` | Public health check. | -| `GET` | `/api/health` | Public API health check. | +| `GET` | `/health` | Public health and worker check. | +| `GET` | `/api/health` | Public API health and worker check. | | `GET` | `/api/sessions` | Legacy/session snapshot from Gateway. | ## Auth @@ -102,18 +102,21 @@ Create body: ## Jobs And Cron -| Method | Path | Purpose | -| ------- | --------------------------- | -------------------------------------- | -| `GET` | `/api/jobs` | Lists Dashboard scheduled jobs. | -| `GET` | `/api/jobs/:id` | Reads a scheduled job. | -| `PATCH` | `/api/jobs/:id` | Updates scheduled job settings and intentional-disable metadata. | -| `POST` | `/api/jobs/:id/run` | Runs a scheduled job now. | -| `GET` | `/api/jobs/:id/runs` | Lists job run history. | -| `GET` | `/api/cron/jobs` | Lists OpenClaw cron jobs and open linked tasks. | -| `POST` | `/api/cron/jobs/:id/run` | Runs an OpenClaw cron job. | -| `POST` | `/api/cron/jobs/:id/toggle` | Enables/disables an OpenClaw cron job and updates its Dashboard-owned disable intent. | -| `POST` | `/api/cron/jobs/:id/update` | Updates an OpenClaw cron job patch. | -| `POST` | `/api/cron/jobs/:id/delete` | Deletes an OpenClaw cron job. | +| Method | Path | Purpose | +| ------- | -------------------------------- | ------------------------------------------------------------------------------------- | +| `GET` | `/api/jobs` | Lists Dashboard scheduled jobs. | +| `GET` | `/api/jobs/:id` | Reads a scheduled job. | +| `PATCH` | `/api/jobs/:id` | Updates scheduled job settings and intentional-disable metadata. | +| `POST` | `/api/jobs/:id/run` | Queues a scheduled job and returns `202`. | +| `GET` | `/api/job-executions` | Lists recent executions plus queue/worker summary. | +| `GET` | `/api/job-executions/:id` | Reads one execution, including its persisted progress/result output snapshot. | +| `POST` | `/api/job-executions/:id/cancel` | Cancels queued work or requests cooperative cancellation of a running execution. | +| `GET` | `/api/jobs/:id/runs` | Lists job run history. | +| `GET` | `/api/cron/jobs` | Lists OpenClaw cron jobs and open linked tasks. | +| `POST` | `/api/cron/jobs/:id/run` | Runs an OpenClaw cron job. | +| `POST` | `/api/cron/jobs/:id/toggle` | Enables/disables an OpenClaw cron job and updates its Dashboard-owned disable intent. | +| `POST` | `/api/cron/jobs/:id/update` | Updates an OpenClaw cron job patch. | +| `POST` | `/api/cron/jobs/:id/delete` | Deletes an OpenClaw cron job. | When a Dashboard job or OpenClaw cron job is intentionally disabled, its update body may include `disableIntent: { mode, comment, until? }`. `mode` is `until` @@ -123,16 +126,24 @@ the annotation in `scheduled_jobs.disable_intent_json`; OpenClaw cron jobs use the Dashboard-owned `openclaw_cron_job_metadata` table rather than modifying the OpenClaw payload or linked tasks. +Dashboard-owned execution-plane routes enqueue work in `job_executions`; the +dedicated worker owns the action and every child process. Scheduled jobs return +the queued execution immediately. Routes that retain an older synchronous API +shape wait by observing the persisted row, but an HTTP disconnect or web +restart does not cancel the action. Poll the execution detail endpoint for its +bounded progress/output snapshot, and use the explicit cancel endpoint when a +queued or running action should stop. + ## OpenClaw Config -| Method | Path | Purpose | -| ------ | ------------------- | ----------------------------------------- | -| `GET` | `/api/config` | Reads OpenClaw config snapshot plus hash. | -| `PUT` | `/api/config` | Writes OpenClaw config with hash check. | -| `GET` | `/api/skills` | Lists OpenClaw skills. | -| `POST` | `/api/skills/:name` | Toggles a skill. | -| `POST` | `/api/backup` | Creates config backup. | -| `POST` | `/api/restart` | Restarts OpenClaw Gateway. | +| Method | Path | Purpose | +| ------ | ------------------- | ---------------------------------------------------------------------- | +| `GET` | `/api/config` | Reads OpenClaw config snapshot plus hash. | +| `PUT` | `/api/config` | Writes OpenClaw config with hash check. | +| `GET` | `/api/skills` | Lists OpenClaw skills. | +| `POST` | `/api/skills/:name` | Toggles a skill. | +| `POST` | `/api/backup` | Creates config backup. | +| `POST` | `/api/restart` | Queues an OpenClaw Gateway restart and waits for its persisted result. | ## Files, Config Files, Logs, Media @@ -162,58 +173,58 @@ upstream download metadata. ## Docker -| Method | Path | Purpose | -| -------- | ------------------------------------------------ | -------------------------------- | -| `GET` | `/api/docker/containers` | Lists containers. | -| `GET` | `/api/docker/containers/:containerId` | Reads container details. | -| `POST` | `/api/docker/containers/:containerId/action` | Starts/stops/restarts container. | -| `GET` | `/api/docker/containers/:containerId/logs` | Reads container logs. | -| `POST` | `/api/docker/exec/start` | Starts a container exec job. | -| `GET` | `/api/docker/exec/:jobId` | Reads exec job output/state. | -| `POST` | `/api/docker/exec/:jobId/stop` | Stops exec job. | -| `GET` | `/api/docker/images` | Lists images. | -| `DELETE` | `/api/docker/images/:imageId` | Deletes an image. | -| `GET` | `/api/docker/volumes` | Lists volumes. | -| `DELETE` | `/api/docker/volumes/:volumeName` | Deletes a volume. | -| `POST` | `/api/docker/prune` | Runs Docker prune target. | -| `POST` | `/api/docker/stack/action` | Runs a compose stack action. | -| `GET` | `/api/docker/updater/services` | Lists managed update services. | -| `GET` | `/api/docker/updater/events` | Lists update events. | -| `POST` | `/api/docker/updater/run` | Runs updater scan. | -| `POST` | `/api/docker/updater/services/:serviceId/update` | Updates one managed service. | +| Method | Path | Purpose | +| -------- | ------------------------------------------------ | ----------------------------------------- | +| `GET` | `/api/docker/containers` | Lists containers. | +| `GET` | `/api/docker/containers/:containerId` | Reads container details. | +| `POST` | `/api/docker/containers/:containerId/action` | Queues a container start/stop/restart. | +| `GET` | `/api/docker/containers/:containerId/logs` | Reads container logs. | +| `POST` | `/api/docker/exec/start` | Queues a worker-owned container exec job. | +| `GET` | `/api/docker/exec/:jobId` | Reads persisted exec output/state. | +| `POST` | `/api/docker/exec/:jobId/stop` | Requests exec cancellation. | +| `GET` | `/api/docker/images` | Lists images. | +| `DELETE` | `/api/docker/images/:imageId` | Queues image deletion. | +| `GET` | `/api/docker/volumes` | Lists volumes. | +| `DELETE` | `/api/docker/volumes/:volumeName` | Queues volume deletion. | +| `POST` | `/api/docker/prune` | Queues a Docker prune target. | +| `POST` | `/api/docker/stack/action` | Queues a Compose stack action. | +| `GET` | `/api/docker/updater/services` | Lists managed update services. | +| `GET` | `/api/docker/updater/events` | Lists update events. | +| `POST` | `/api/docker/updater/run` | Queues an updater scan. | +| `POST` | `/api/docker/updater/services/:serviceId/update` | Queues one managed service update. | ## Pull Requests And Deployments -| Method | Path | Purpose | -| ------ | -------------------------------------------- | ------------------------------------ | -| `GET` | `/api/pull-requests` | Lists Dashboard PRs. | -| `POST` | `/api/pull-requests/:number/approve` | Approves/merges, optionally deploys. | -| `POST` | `/api/pull-requests/:number/reject` | Rejects/closes. | -| `POST` | `/api/pull-requests/:number/review-approval` | Approves review. | -| `POST` | `/api/pull-requests/:number/update-branch` | Updates branch. | -| `POST` | `/api/pull-requests/deploy` | Starts deploy latest job. | -| `GET` | `/api/pull-requests/deployments` | Lists deploy jobs. | -| `GET` | `/api/pull-requests/production-checkout` | Reads production checkout status. | +| Method | Path | Purpose | +| ------ | -------------------------------------------- | -------------------------------------------- | +| `GET` | `/api/pull-requests` | Lists Dashboard PRs. | +| `POST` | `/api/pull-requests/:number/approve` | Queues merge, optionally followed by deploy. | +| `POST` | `/api/pull-requests/:number/reject` | Queues reject/close. | +| `POST` | `/api/pull-requests/:number/review-approval` | Queues review approval. | +| `POST` | `/api/pull-requests/:number/update-branch` | Queues branch update. | +| `POST` | `/api/pull-requests/deploy` | Queues deploy latest. | +| `GET` | `/api/pull-requests/deployments` | Lists deploy jobs. | +| `GET` | `/api/pull-requests/production-checkout` | Reads production checkout status. | ## Backups, Cache, Metrics, Ops -| Method | Path | Purpose | -| ------ | ------------------------------------------ | ---------------------------------------------------------------------- | -| `GET` | `/api/backups/kopia` | Reads Kopia backup state. | -| `POST` | `/api/backups/kopia/run` | Starts Kopia backup. | -| `POST` | `/api/backups/kopia/clear-needs-attention` | Clears Kopia attention state. | -| `GET` | `/api/backups/walg` | Reads WAL-G backup state. | -| `POST` | `/api/backups/walg/run` | Starts WAL-G backup. | -| `POST` | `/api/backups/walg/clear-needs-attention` | Clears WAL-G attention state. | +| Method | Path | Purpose | +| ------ | ------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| `GET` | `/api/backups/kopia` | Reads Kopia backup state. | +| `POST` | `/api/backups/kopia/run` | Queues Kopia backup. | +| `POST` | `/api/backups/kopia/clear-needs-attention` | Queues clearing Kopia attention state. | +| `GET` | `/api/backups/walg` | Reads WAL-G backup state. | +| `POST` | `/api/backups/walg/run` | Queues WAL-G backup. | +| `POST` | `/api/backups/walg/clear-needs-attention` | Queues clearing WAL-G attention state. | | `GET` | `/api/cache/heartbeat` | Reads schema v3 cache envelopes plus compact task, OpenClaw cron, and Dashboard-job projections. | -| `GET` | `/api/cache/status` | Reads cache envelopes without payload data for lightweight UI polling. | -| `GET` | `/api/cache/:key` | Reads one cache entry. | -| `POST` | `/api/cache/:key/refresh` | Refreshes one cache entry. | -| `GET` | `/api/metrics` | Reads host metrics. | -| `GET` | `/api/database/overview` | Reads Postgres/PgBouncer overview. | -| `GET` | `/api/ops/log-rotation/status` | Reads log rotation status. | -| `POST` | `/api/ops/log-rotation/dry-run` | Runs log rotation dry-run. | -| `POST` | `/api/ops/log-rotation/run` | Runs log rotation. | +| `GET` | `/api/cache/status` | Reads cache envelopes without payload data for lightweight UI polling. | +| `GET` | `/api/cache/:key` | Reads one cache entry. | +| `POST` | `/api/cache/:key/refresh` | Queues and observes one cache refresh. | +| `GET` | `/api/metrics` | Reads host metrics. | +| `GET` | `/api/database/overview` | Reads Postgres/PgBouncer overview. | +| `GET` | `/api/ops/log-rotation/status` | Reads log rotation status. | +| `POST` | `/api/ops/log-rotation/dry-run` | Queues and observes log rotation dry-run. | +| `POST` | `/api/ops/log-rotation/run` | Queues and observes log rotation. | `/api/database/overview` includes Comet/Bitmagnet torrent counts and a conservative Postgres maintenance assessment. Bloat is marked for review at an @@ -240,14 +251,14 @@ their consumer contracts. ## Exec And Terminal -| Method | Path | Purpose | -| ------ | ------------------------ | ------------------------------------- | -| `POST` | `/api/exec` | Runs one command. | -| `POST` | `/api/exec/start` | Starts long-running exec job. | -| `GET` | `/api/exec/:jobId` | Reads exec job output/state. | -| `POST` | `/api/exec/:jobId/stop` | Stops exec job. | -| `POST` | `/api/terminal/complete` | Returns shell/path completions. | -| `POST` | `/api/terminal/cd` | Resolves validated directory changes. | +| Method | Path | Purpose | +| ------ | ------------------------ | ----------------------------------------------------- | +| `POST` | `/api/exec` | Queues one command and observes its persisted result. | +| `POST` | `/api/exec/start` | Queues a worker-owned long-running exec job. | +| `GET` | `/api/exec/:jobId` | Reads persisted exec output/state. | +| `POST` | `/api/exec/:jobId/stop` | Requests exec cancellation. | +| `POST` | `/api/terminal/complete` | Returns shell/path completions. | +| `POST` | `/api/terminal/cd` | Resolves validated directory changes. | The Terminal page executes commands through `/api/exec/start` and polls the result by job ID. The backend validates the argv contract. The Terminal UI diff --git a/docs/architecture/database.md b/docs/architecture/database.md index c40f9c8ad..4e9205eb4 100644 --- a/docs/architecture/database.md +++ b/docs/architecture/database.md @@ -29,26 +29,30 @@ non-temporary paths and symlinked temp paths. ## Tables -| Table | Purpose | -| ------------------------- | --------------------------------------------------------------- | -| `users` | Dashboard auth users. | -| `auth_sessions` | Cookie-backed Dashboard sessions. | -| `app_config` | Small persistent config, currently including `gateway_token`. | -| `tasks` | Local task records. | -| `task_events` | Audit/event records for task changes. | -| `task_updates` | Markdown progress updates on tasks. | -| `notifications` | Notification bell items, including report links and ops alerts. | -| `reports` | Daily briefs, daily summaries, heartbeats, and custom reports. | -| `cache_entries` | Cache refresh state and cached provider data. | -| `quota_alert_state` | Notification arming state for quota thresholds. | -| `openclaw_alert_state` | Notification arming state for OpenClaw update alerts. | -| `agent_task_history` | Agent current/completed task history. | -| `deployment_jobs` | Dashboard deploy job state/output. | -| `deployment_lock` | Single active deployment lock. | -| `scheduled_jobs` | Dashboard-local scheduled job definitions. | -| `scheduled_job_runs` | Scheduled job run history. | -| `docker_managed_services` | Docker updater managed service inventory. | -| `docker_update_events` | Docker updater event history. | +| Table | Purpose | +| ---------------------------------- | --------------------------------------------------------------------- | +| `users` | Dashboard auth users. | +| `auth_sessions` | Cookie-backed Dashboard sessions. | +| `app_config` | Small persistent config, currently including `gateway_token`. | +| `tasks` | Local task records. | +| `task_events` | Audit/event records for task changes. | +| `task_updates` | Markdown progress updates on tasks. | +| `notifications` | Notification bell items, including report links and ops alerts. | +| `reports` | Daily briefs, daily summaries, heartbeats, and custom reports. | +| `cache_entries` | Cache refresh state and cached provider data. | +| `quota_alert_state` | Notification arming state for quota thresholds. | +| `openclaw_alert_state` | Notification arming state for OpenClaw update alerts. | +| `agent_task_history` | Agent current/completed task history. | +| `deployment_jobs` | Dashboard deploy job state/output. | +| `deployment_lock` | Single active deployment lock. | +| `scheduled_jobs` | Dashboard-local scheduled job definitions. | +| `scheduled_job_runs` | Scheduled job run history. | +| `scheduled_job_execution_policies` | Resource class and timeout for each Dashboard job. | +| `openclaw_cron_job_metadata` | Disable intent and Dashboard metadata for OpenClaw cron jobs. | +| `job_executions` | Persistent execution queue with leases, heartbeats, and cancellation. | +| `job_workers` | Worker capacity and liveness heartbeats. | +| `docker_managed_services` | Docker updater managed service inventory. | +| `docker_update_events` | Docker updater event history. | ## Backup Before Manual DB Work diff --git a/docs/architecture/gateway-and-chat.md b/docs/architecture/gateway-and-chat.md index b95be589d..932a94c47 100644 --- a/docs/architecture/gateway-and-chat.md +++ b/docs/architecture/gateway-and-chat.md @@ -55,6 +55,11 @@ server may need manual inspection. | `gatewayConnected` | Whether the backend Gateway client is authenticated and connected. | | `sessionCount` | Gateway session count known to Dashboard. | | `backendCommit` | Git commit served by the backend when available. | +| `workerOnline` | Whether the persistent job worker has a fresh heartbeat. | + +If queue telemetry cannot be read, health remains available and reports +`workerOnline:false`. Treat that degraded value as either an offline worker or +unavailable queue telemetry, and inspect both backend and worker logs. If `gatewayConnected:false`, check: diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index c03fee8ad..6ef9c4667 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -11,16 +11,18 @@ Browser | | HTTP + /ws v -Bun backend on :3100 +Mira Dashboard web on :3100 |-- static frontend from dist/ |-- /api/* route table |-- authenticated Dashboard WebSocket - |-- SQLite via bun:sqlite - |-- background scheduled jobs - | - | OpenClaw Gateway WebSocket - v -OpenClaw Gateway on :18789 + |-- OpenClaw Gateway WebSocket --> OpenClaw Gateway on :18789 + `-- SQLite via bun:sqlite <------------------. + | +Mira Dashboard worker | + |-- scheduler | + |-- bounded action executor | + |-- resource-scoped child processes | + `-- persistent execution queue (SQLite) ----' ``` ## Frontend @@ -73,7 +75,8 @@ Key files: | `backend/src/http.ts` | JSON helpers, cookies, origin/proxy/IP helpers. | | `backend/src/gateway.ts` | OpenClaw Gateway client lifecycle and Dashboard WebSocket fanout. | | `backend/src/database.ts` | SQLite path, schema, PRAGMAs, database proxy. | -| `backend/src/serverStart.ts` | Production startup and background scheduler registration. | +| `backend/src/serverStart.ts` | HTTP/WebSocket production startup (or combined fallback role). | +| `backend/src/workerStart.ts` | Dedicated scheduler/executor production startup. | ## Authentication @@ -115,8 +118,14 @@ token recovery, browser WebSocket behavior, and chat event handling. ## Background Jobs -When `MIRA_DASHBOARD_DISABLE_SCHEDULER` is not set for development/testing, -startup registers jobs for: +The dedicated worker registers scheduled jobs and every long-running or +mutating execution-plane adapter. The web process validates requests, enqueues +work in SQLite, and observes persisted result/progress snapshots when an API +must retain a synchronous response shape. Restarting the web process therefore +does not terminate worker-owned deploy, exec, Docker, backup, cache refresh, +log rotation, GitHub, or OpenClaw restart actions. + +Worker startup registers scheduled jobs for: - backups; - cache refresh; @@ -126,8 +135,10 @@ startup registers jobs for: - OpenClaw update notifications; - scheduled job runner. -Local development disables scheduler by default through the backend `dev` -script. +Production uses `MIRA_DASHBOARD_EXECUTION_ROLE=web` and `worker` in separate +systemd services. The default `combined` role exists for compatibility and +tests; local backend development disables it through +`MIRA_DASHBOARD_DISABLE_SCHEDULER=1`. See [Scheduler, cache, and backups](../operations/scheduler-cache-backups.md) for job tables, cache entries, backup scripts, and inspection commands. diff --git a/docs/operations/scheduler-cache-backups.md b/docs/operations/scheduler-cache-backups.md index 3b0026896..822541b0e 100644 --- a/docs/operations/scheduler-cache-backups.md +++ b/docs/operations/scheduler-cache-backups.md @@ -1,19 +1,23 @@ # Scheduler, Cache, And Backups -Dashboard starts background jobs from `backend/src/serverStart.ts` unless -`MIRA_DASHBOARD_DISABLE_SCHEDULER=1` is set. Other values, such as `0`, -`false`, or `true`, do not disable schedulers. The backend development script -disables schedulers by default. +Dashboard runs background jobs from `backend/src/workerStart.ts`. Production +sets `MIRA_DASHBOARD_EXECUTION_ROLE=web` on the web unit and +`MIRA_DASHBOARD_EXECUTION_ROLE=worker` on the worker unit. The backward- +compatible default is `combined`; `MIRA_DASHBOARD_DISABLE_SCHEDULER=1` still +disables the in-process worker during local development. ## Scheduled Jobs Dashboard-local scheduled jobs are stored in SQLite: -| Table | Purpose | -| -------------------- | ---------------------------------------- | -| `scheduled_jobs` | Job definitions. | -| `scheduled_job_runs` | Run history, status, output, and errors. | -| `openclaw_cron_job_metadata` | Dashboard-owned metadata for external OpenClaw cron jobs. | +| Table | Purpose | +| ---------------------------------- | --------------------------------------------------------- | +| `scheduled_jobs` | Job definitions. | +| `scheduled_job_runs` | Run history, status, output, and errors. | +| `scheduled_job_execution_policies` | Resource class and timeout per job. | +| `job_executions` | Persistent queue, lease, heartbeat, and cancellation. | +| `job_workers` | Worker capacity and liveness heartbeat. | +| `openclaw_cron_job_metadata` | Dashboard-owned metadata for external OpenClaw cron jobs. | Supported schedule shapes: @@ -26,6 +30,22 @@ Operational defaults: - scheduler tick: 30 seconds; - minimum interval schedule: 60 seconds; - default run timeout: 5 minutes. +- global worker concurrency: 1; +- worker lease: 2 minutes, refreshed every second while an action runs; +- startup cache seeds: persistent, single-concurrency queue with five-second stagger + plus bounded jitter. + +Queue statuses are `queued`, `running`, `success`, `failed`, and `cancelled`. +An expired running lease is marked failed instead of automatically replayed; +this avoids repeating backup, update, or other non-idempotent side effects after +a worker crash. + +`GET /api/job-executions` exposes queue depth, oldest wait, resource classes, +and worker liveness. `GET /api/job-executions/:id` includes the bounded +persisted output snapshot used for stdout/stderr and incremental progress. +HTTP waiters are observers only: disconnecting a request or restarting the web +service never writes a cancellation request. Cancellation is explicit through +`POST /api/job-executions/:id/cancel`. Use the Jobs page to inspect definitions and run history before editing the database manually. @@ -48,7 +68,7 @@ There is deliberately no runtime migration or fallback for these fields. ## Built-In Startup Jobs -When enabled, startup registers jobs for: +When enabled, worker startup registers jobs for: - backup refresh/status; - cache refresh; @@ -57,7 +77,23 @@ When enabled, startup registers jobs for: - log rotation; - quota notifications; - OpenClaw update notifications; -- scheduled job runner. +- persistent scheduled job executor. + +The same executor owns manual cache refresh, backups, log rotation, OpenClaw +Gateway restart, tracked shell/exec, Docker mutations and container exec, +Docker updater, GitHub pull-request mutations, and Dashboard deploy/build. +These adapters are registered only in the worker process; the web process only +validates, enqueues, and reads persisted state. + +Resource classes are `interactive`, `light`, `network`, `host-heavy`, and +`exclusive`. The queue prioritizes interactive/light data ahead of maintenance +work, while the initial global concurrency of one prevents classes from +overlapping. Child commands launched by worker actions use transient systemd +scopes with class-specific `Nice`, `CPUWeight`, `IOWeight`, `MemoryHigh`, +`MemoryMax`, `TasksMax`, and maximum runtime limits. Each action scope is bound +to `mira-dashboard-worker.service`; restarting only +`mira-dashboard.service` leaves it running, while stopping/restarting the worker +cooperatively aborts the active execution and terminates its scoped child cgroup. Do not assume a job is running just because the code exists. Check the Jobs page or `scheduled_jobs` table for enabled state and recent run history. @@ -100,9 +136,9 @@ latest scheduled job run before debugging the frontend. Dashboard exposes two intentionally different aggregate cache endpoints: -| Endpoint | Consumer | Payload contract | -| ---------------------- | ---------------------- | ---------------------------------------------------------------------- | -| `/api/cache/status` | Dashboard UI polling | Cache envelopes only; `data` is `null`. | +| Endpoint | Consumer | Payload contract | +| ---------------------- | ---------------------- | ------------------------------------------------------------------------------------------ | +| `/api/cache/status` | Dashboard UI polling | Cache envelopes only; `data` is `null`. | | `/api/cache/heartbeat` | OpenClaw ops heartbeat | `schemaVersion: 3` plus compact cache, task, OpenClaw cron, and Dashboard-job projections. | Both responses retain every cache envelope so consumers can assess freshness, @@ -161,8 +197,9 @@ host backups. Relevant scripts in production include: /usr/local/bin/backup-push.sh ``` -Long-running backup jobs can run for hours. Do not restart Dashboard just -because a backup action is active; first inspect the active job and host logs. +Long-running backup jobs can run for hours. Restarting the web service does not +interrupt them. Do not restart the worker while a backup is active unless the +intent is to abort it; first inspect the execution row and host logs. ## Operational Checks diff --git a/docs/setup/new-vps.md b/docs/setup/new-vps.md index 736e5bee0..7469145cd 100644 --- a/docs/setup/new-vps.md +++ b/docs/setup/new-vps.md @@ -77,42 +77,42 @@ minimum production setup normally needs: - optional provider keys for Moltbook, ElevenLabs, OpenRouter, and Synthetic health checks depending on enabled Dashboard features. -## Create The Systemd User Service +## Create The Systemd User Services -Service path: +Run this section from an interactive shell as the `ubuntu` user. Use `sudo` only +for the explicit `loginctl` command; the install and `systemctl --user` commands +must target `ubuntu`'s user manager. -```text -/home/ubuntu/.config/systemd/user/mira-dashboard.service -``` - -Current production shape: - -```ini -[Unit] -Description=Mira Dashboard -After=network-online.target openclaw-gateway.service -Wants=network-online.target - -[Service] -Type=simple -WorkingDirectory=/home/ubuntu/projects/mira-dashboard/backend -Environment=NODE_ENV=production -ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/serverStart.js -Restart=on-failure -RestartSec=5 +Install the tracked web and worker units: -[Install] -WantedBy=default.target +```bash +cd /home/ubuntu/projects/mira-dashboard +install -d -m 0755 /home/ubuntu/.config/systemd/user +install -m 0644 systemd/mira-dashboard.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service +install -m 0644 systemd/mira-dashboard-worker.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service ``` -Enable and start: +The web role owns HTTP, WebSocket, and the Gateway bridge. The worker role owns +scheduled-job registration, queue claims, cache startup seeds, and action +execution. Both units have explicit CPU, IO, memory, and task guardrails. Heavy +worker children are additionally placed in transient resource-class scopes. + +Enable and start both: ```bash +sudo loginctl enable-linger ubuntu +loginctl show-user ubuntu -p Linger systemctl --user daemon-reload -systemctl --user enable --now mira-dashboard.service +systemctl --user enable --now mira-dashboard.service mira-dashboard-worker.service systemctl --user status mira-dashboard.service --no-pager +systemctl --user status mira-dashboard-worker.service --no-pager ``` +Lingering keeps the user manager and both services running after the SSH/login +session ends and across reboots. + ## Bootstrap The First User If the database has no users, the UI shows first-user setup. The bootstrap flow: @@ -147,6 +147,7 @@ Expected after setup: curl http://127.0.0.1:3100/api/health systemctl --user status mira-dashboard.service --no-pager journalctl --user -u mira-dashboard.service -n 100 --no-pager +journalctl --user -u mira-dashboard-worker.service -n 100 --no-pager ``` Healthy response shape: @@ -156,9 +157,13 @@ Healthy response shape: "status": "isOk", "gatewayConnected": true, "sessionCount": 9, - "backendCommit": "abc1234" + "backendCommit": "abc1234", + "workerOnline": true } ``` If `gatewayConnected` is false, check the Gateway token, OpenClaw Gateway service, and `/api/auth/bootstrap` state before debugging the frontend. +If `workerOnline` is false, inspect both Dashboard and +`mira-dashboard-worker.service`; the worker heartbeat may be stale or queue +telemetry may be unavailable. diff --git a/docs/setup/production-deploy.md b/docs/setup/production-deploy.md index af0537108..8f7970dbd 100644 --- a/docs/setup/production-deploy.md +++ b/docs/setup/production-deploy.md @@ -18,9 +18,17 @@ This is a single-host service: - frontend assets are built into `dist/`; - backend TypeScript is built into `backend/dist/`; -- `mira-dashboard.service` runs `bun dist/serverStart.js` through Doppler; +- `mira-dashboard.service` runs the HTTP/WebSocket process from + `bun dist/serverStart.js` through Doppler; +- `mira-dashboard-worker.service` runs the persistent scheduler/executor from + `bun dist/workerStart.js` through Doppler; - SQLite state lives under `backend/data/` unless `MIRA_DASHBOARD_DB_PATH` is set. +Both tracked units preserve the production environment contract by launching +through Doppler project/config `rajohan/prd`. Auth and origin settings such as +`MIRA_DASHBOARD_ENABLE_LOOPBACK_AUTH` and `MIRA_DASHBOARD_ALLOWED_ORIGINS` +remain owned by Doppler; do not duplicate their values in unit files. + There is no container image for the Dashboard service today. ## Build @@ -42,20 +50,47 @@ bun install --frozen-lockfile bun run build ``` +## Install Or Refresh Units + +After building, install the tracked resource-limited units: + +```bash +cd /home/ubuntu/projects/mira-dashboard +install -m 0644 systemd/mira-dashboard.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service +install -m 0644 systemd/mira-dashboard-worker.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service +systemctl --user daemon-reload +``` + +For the first split-process rollout, restart the web unit with its explicit +`web` role before starting the worker. This avoids overlapping the legacy +combined scheduler with the dedicated worker: + +```bash +systemctl --user restart mira-dashboard.service +systemctl --user enable --now mira-dashboard-worker.service +``` + ## Restart Always tell Raymond before restarting OpenClaw Gateway. Dashboard restart is -safe after a merged/deployed Dashboard change: +safe after a merged/deployed Dashboard change. A web-only restart does not +interrupt queued/running actions. Before restarting the worker, verify the Jobs +queue is idle or explicitly accept that its active action will be cancelled: ```bash systemctl --user restart mira-dashboard.service +systemctl --user restart mira-dashboard-worker.service systemctl --user status mira-dashboard.service --no-pager +systemctl --user status mira-dashboard-worker.service --no-pager ``` Logs: ```bash journalctl --user -u mira-dashboard.service -n 120 --no-pager +journalctl --user -u mira-dashboard-worker.service -n 120 --no-pager ``` ## Smoke Test @@ -65,9 +100,15 @@ curl http://127.0.0.1:3100/api/health curl http://127.0.0.1:3100/api/auth/bootstrap ``` +The queue endpoint requires a valid Dashboard session unless the explicitly +configured direct-loopback bypass is enabled. Production currently sources +`MIRA_DASHBOARD_ENABLE_LOOPBACK_AUTH=1` from Doppler, but the portable smoke test +does not depend on that host-specific bypass. + For an authenticated browser session, also verify: - header/WebSocket status is connected; +- Jobs shows the execution queue and the worker becomes idle after startup seeds; - Dashboard page cards load; - Reports page loads recent reports; - Notifications bell loads without global chat/tool errors. @@ -82,12 +123,28 @@ git log --oneline -n 10 git switch main git reset --hard bun run build -cd backend -bun run build -systemctl --user restart mira-dashboard.service +(cd backend && bun run build) +install -m 0644 systemd/mira-dashboard.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard.service +if test -f backend/dist/workerStart.js; then + install -m 0644 systemd/mira-dashboard-worker.service \ + /home/ubuntu/.config/systemd/user/mira-dashboard-worker.service + systemctl --user daemon-reload + systemctl --user restart mira-dashboard.service + systemctl --user restart mira-dashboard-worker.service +else + systemctl --user disable --now mira-dashboard-worker.service + systemctl --user daemon-reload + systemctl --user restart mira-dashboard.service +fi curl http://127.0.0.1:3100/api/health ``` +If the known-good target predates `workerStart.js`, the branch above stops the +worker before starting that version and reinstalls the checked-out legacy +combined web unit. Do not repeatedly restart a worker unit whose target +entrypoint does not exist. + Do not use `git reset --hard` casually in normal work. It is a rollback procedure for production incidents after an explicit decision. @@ -115,14 +172,19 @@ Healthy `/api/health`: "status": "isOk", "gatewayConnected": true, "sessionCount": 9, - "backendCommit": "abc1234" + "backendCommit": "abc1234", + "workerOnline": true } ``` Important failures: - `gatewayConnected: false`: check OpenClaw Gateway service and Gateway token. +- `workerOnline: false`: the worker heartbeat is stale or queue telemetry is + unavailable; check both Dashboard and worker service logs. - HTTP `503 Frontend Not Built`: build root frontend with `bun run build`. - `Unauthorized` on API routes: auth/session or cookie issue. - `database is locked`: another process is holding SQLite; retry after - background jobs settle, then inspect service logs. + background jobs settle, then inspect both service logs. Dashboard uses a + five-second SQLite busy timeout; WAL remains a separate storage-lifecycle + decision that must be paired with a tested backup/checkpoint plan. diff --git a/docs/setup/secrets-and-env.md b/docs/setup/secrets-and-env.md index 8f7c4c298..95e1153fa 100644 --- a/docs/setup/secrets-and-env.md +++ b/docs/setup/secrets-and-env.md @@ -43,6 +43,21 @@ token in `app_config.gateway_token`. Environment token precedence is: See [Auth and trust boundaries](../security/auth-and-trust-boundaries.md) for route auth, loopback bypass, proxy trust, bootstrap, and token handling. +## Execution Roles And Resource Scopes + +| Variable | Production value | Purpose | +| ---------------------------------- | ------------------- | ------------------------------------------------------------------------------------------ | +| `MIRA_DASHBOARD_EXECUTION_ROLE` | `web` / `worker` | Keeps HTTP/WebSocket handling in the web unit and scheduler/executor work in the worker. | +| `MIRA_DASHBOARD_ENABLE_JOB_SCOPES` | `1` in both units | Runs classified job children in constrained transient user scopes. | +| `MIRA_DASHBOARD_JOB_SCOPE_OWNER` | owning service unit | Binds transient scopes to their service lifecycle so restarts terminate orphaned children. | +| `MIRA_DASHBOARD_DISABLE_SCHEDULER` | unset in production | Development/test escape hatch; `1` disables scheduler/executor startup. | + +The tracked systemd units set these orchestration values directly. Doppler +remains the source of auth, origin, provider, and credential values. Production +actions run in the worker, so their child scopes bind to +`mira-dashboard-worker.service`; restarting only the web unit leaves them +untouched. + ## GitHub And PR Operations | Variable | Required for | Purpose | diff --git a/package.json b/package.json index 695f6d77f..9e85f99b4 100644 --- a/package.json +++ b/package.json @@ -26,23 +26,23 @@ "@dnd-kit/react": "^0.5.0", "@dnd-kit/sortable": "^10.0.0", "@headlessui/react": "^2.2.10", - "@microlink/react-json-view": "^1.31.22", + "@microlink/react-json-view": "^1.31.23", "@tailwindcss/typography": "^0.5.20", - "@tanstack/query-core": "5.101.2", - "@tanstack/query-db-collection": "^1.1.0", + "@tanstack/query-core": "5.101.4", + "@tanstack/query-db-collection": "^1.2.0", "@tanstack/react-db": "^0.1.94", "@tanstack/react-form": "^1.33.2", - "@tanstack/react-query": "^5.101.2", + "@tanstack/react-query": "^5.101.4", "@tanstack/react-router": "^1.170.18", "@tanstack/react-store": "^0.11.0", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.14.6", + "@tanstack/react-virtual": "^3.14.8", "clsx": "^2.1.1", "date-fns": "^4.4.0", "json5": "^2.2.3", "lucide-react": "^1.25.0", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", "react-error-boundary": "^6.1.2", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", @@ -55,13 +55,13 @@ "devDependencies": { "@babel/core": "^8.0.1", "@eslint/js": "^10.0.1", - "@happy-dom/global-registrator": "^20.11.0", - "@tanstack/react-devtools": "^0.10.8", + "@happy-dom/global-registrator": "^20.11.1", + "@tanstack/react-devtools": "^0.10.9", "@tanstack/react-form-devtools": "^0.2.31", - "@tanstack/react-query-devtools": "^5.101.2", + "@tanstack/react-query-devtools": "^5.101.4", "@tanstack/react-router-devtools": "1.167.0", "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/babel__core": "^7.20.5", @@ -84,10 +84,10 @@ "eslint-plugin-tailwindcss": "^4.2.0", "eslint-plugin-unicorn": "^72.0.0", "globals": "^17.7.0", - "happy-dom": "^20.11.0", - "prettier": "^3.9.5", + "happy-dom": "^20.11.1", + "prettier": "^3.9.6", "tailwindcss": "^4.3.3", "typescript": "^6.0.3", - "typescript-eslint": "^8.64.0" + "typescript-eslint": "^8.65.0" } } diff --git a/src/components/features/dashboard/JobsOverviewCard.tsx b/src/components/features/dashboard/JobsOverviewCard.tsx index d33874136..f7b749ea9 100644 --- a/src/components/features/dashboard/JobsOverviewCard.tsx +++ b/src/components/features/dashboard/JobsOverviewCard.tsx @@ -24,7 +24,8 @@ function jobTimestamp(job: ScheduledJob, key: "nextRunAt" | "lastRun") { function lastRunVariant(job: ScheduledJob | undefined) { const status = job?.lastRun?.status; if (status === "failed") return "error" as const; - if (status === "running") return "info" as const; + if (status === "queued" || status === "running") return "info" as const; + if (status === "cancelled") return "warning" as const; if (status === "success") return "success" as const; return "default" as const; } @@ -53,7 +54,7 @@ export function JobsOverviewCard() { const cronEnabledCount = cronJobs.filter((job) => job.enabled !== false).length; const disabledCount = jobs.filter((job) => !job.enabled).length; const cronDisabledCount = cronJobs.filter((job) => job.enabled === false).length; - const runningCount = jobs.filter((job) => job.isRunning).length; + const activeCount = jobs.filter((job) => job.isQueued || job.isRunning).length; const latestRunJob = [...jobs] .filter((job) => jobTimestamp(job, "lastRun") !== undefined) @@ -153,13 +154,13 @@ export function JobsOverviewCard() {
- Running + Queued/running 0 ? "text-blue-300" : "text-primary-300" + activeCount > 0 ? "text-blue-300" : "text-primary-300" } > - {runningCount} + {activeCount}
diff --git a/src/components/features/dashboard/LogRotationCard.tsx b/src/components/features/dashboard/LogRotationCard.tsx index 2e739d297..9b63a7b82 100644 --- a/src/components/features/dashboard/LogRotationCard.tsx +++ b/src/components/features/dashboard/LogRotationCard.tsx @@ -179,12 +179,19 @@ export function LogRotationCard() { isDryRun.isPending || realRun.isPending || !logRotationJob || + logRotationJob.isQueued || logRotationJob.isRunning } className="w-full justify-center" > - {realRun.isPending ? "Running..." : "Run real now"} + {realRun.isPending + ? "Queueing..." + : logRotationJob?.isQueued + ? "Queued" + : logRotationJob?.isRunning + ? "Running..." + : "Run real now"}
diff --git a/src/components/features/jobs/JobExecutionQueueCard.tsx b/src/components/features/jobs/JobExecutionQueueCard.tsx new file mode 100644 index 000000000..db3cfc73c --- /dev/null +++ b/src/components/features/jobs/JobExecutionQueueCard.tsx @@ -0,0 +1,172 @@ +import { Activity, Clock3, Layers3, XCircle } from "lucide-react"; + +import { + type JobExecution, + useCancelJobExecution, + useJobExecutions, +} from "../../../hooks"; +import { formatDate, formatDuration } from "../../../utils/format"; +import { Alert } from "../../ui/Alert"; +import { Badge } from "../../ui/Badge"; +import { Button } from "../../ui/Button"; +import { Card, CardTitle } from "../../ui/Card"; + +function statusVariant(execution: JobExecution) { + if (execution.status === "failed") return "error" as const; + if (execution.status === "success") return "success" as const; + if (execution.status === "cancelled") return "warning" as const; + return "info" as const; +} + +function resourceLabel(resourceClass: JobExecution["resourceClass"]): string { + return resourceClass.replace("-", " "); +} + +/** Shows global queue pressure and cancellation controls for active executions. */ +export function JobExecutionQueueCard() { + const queue = useJobExecutions(); + const cancelExecution = useCancelJobExecution(); + const summary = queue.data?.summary; + const activeExecutions = (queue.data?.executions ?? []).filter( + (execution) => execution.status === "queued" || execution.status === "running" + ); + const activeClasses = summary?.activeResourceClasses + .map((resourceClass) => resourceLabel(resourceClass)) + .join(", "); + const oldestQueuedTimestamp = summary?.oldestQueuedAt + ? Date.parse(summary.oldestQueuedAt) + : NaN; + const oldestWait = Number.isFinite(oldestQueuedTimestamp) + ? formatDuration(oldestQueuedTimestamp, { includeSeconds: true }) + : "None"; + + return ( + +
+
+ Execution queue +

+ Persistent worker queue · global concurrency 1 +

+
+ + {queue.isLoading + ? "Loading worker" + : summary?.workerOnline + ? summary.running + ? "Worker active" + : "Worker idle" + : "Worker offline"} + +
+ + {queue.error ? ( + + Queue refresh failed. {queue.error.message} + + ) : undefined} + {cancelExecution.error ? ( + {cancelExecution.error.message} + ) : undefined} + +
+
+
+ Queue depth +
+
+ {summary?.queued ?? 0} +
+
+
+
+ Running +
+
+ {summary?.running ?? 0}/{summary?.workerCapacity ?? 0} +
+
+
+
+ Oldest queued +
+
+ {oldestWait} +
+
+
+
Active class
+
+ {activeClasses || "None"} +
+
+
+ + {activeExecutions.length > 0 ? ( +
+ {activeExecutions.map((execution) => { + const isCancelling = + cancelExecution.isPending && + cancelExecution.variables === execution.id; + return ( +
+
+
+ + {execution.displayName} + + + {execution.cancelRequestedAt + ? "cancel requested" + : execution.status} + + + {resourceLabel(execution.resourceClass)} + +
+
+ Queued {formatDate(execution.queuedAt)} + {execution.startedAt + ? ` · started ${formatDate(execution.startedAt)}` + : ""} +
+
+ {execution.cancellable && !execution.cancelRequestedAt ? ( + + ) : undefined} +
+ ); + })} +
+ ) : ( +

No queued or running jobs.

+ )} + + ); +} diff --git a/src/hooks/index.ts b/src/hooks/index.ts index c267e768d..bc13ca818 100644 --- a/src/hooks/index.ts +++ b/src/hooks/index.ts @@ -41,6 +41,17 @@ export { } from "./useCron"; export { fileKeys, useFileContent, useFiles, useSaveFile } from "./useFiles"; export { useHealth } from "./useHealth"; +export type { + JobExecution, + JobExecutionStatus, + JobExecutionSummary, + JobResourceClass, +} from "./useJobExecutions"; +export { + jobExecutionKeys, + useCancelJobExecution, + useJobExecutions, +} from "./useJobExecutions"; export { logKeys, useLogContent, useLogFiles } from "./useLogs"; export { useMetrics } from "./useMetrics"; export { diff --git a/src/hooks/useBackups.ts b/src/hooks/useBackups.ts index 1c360e4f5..7b8d6d8d7 100644 --- a/src/hooks/useBackups.ts +++ b/src/hooks/useBackups.ts @@ -7,7 +7,7 @@ import { cacheKeys } from "./useCache"; export interface BackupJob { id: string; type: "kopia" | "walg"; - status: "running" | "done" | "needs_attention"; + status: "running" | "done" | "needs_attention" | "failed" | "cancelled"; code: number | undefined; stdout: string; stderr: string; diff --git a/src/hooks/useCache.ts b/src/hooks/useCache.ts index 74a1b4bce..8e3824b05 100644 --- a/src/hooks/useCache.ts +++ b/src/hooks/useCache.ts @@ -108,31 +108,43 @@ export function useRefreshCacheEntry() { .map((key) => key.trim()) .filter(Boolean); - const results = await Promise.all( - keys.map((key) => - apiPostRequired<{ + const results: Array<{ + isOk: boolean; + entry: CacheEnvelope; + }> = []; + const errors: unknown[] = []; + for (const key of keys) { + try { + const response = await apiPostRequired<{ isOk: boolean; entry: CacheEnvelope; - }>(`/cache/${encodeURIComponent(key)}/refresh`) - ) - ); + }>(`/cache/${encodeURIComponent(key)}/refresh`); + results.push(response); + if (response.entry?.key) { + queryClient.setQueryData( + cacheKeys.entry(response.entry.key), + response.entry + ); + } + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError( + errors, + `${errors.length} cache refresh requests failed` + ); + } return { keys, results }; }, - onSuccess: async (result, keysToken) => { + onSettled: async (_result, _error, keysToken) => { const keys = keysToken .split(",") .map((key) => key.trim()) .filter(Boolean); - for (const response of result.results) { - if (response.entry?.key) { - queryClient.setQueryData( - cacheKeys.entry(response.entry.key), - response.entry - ); - } - } - await Promise.all([ queryClient.invalidateQueries({ queryKey: cacheKeys.heartbeat() }), queryClient.invalidateQueries({ queryKey: cacheKeys.status() }), diff --git a/src/hooks/useJobExecutions.ts b/src/hooks/useJobExecutions.ts new file mode 100644 index 000000000..b3f8db57b --- /dev/null +++ b/src/hooks/useJobExecutions.ts @@ -0,0 +1,81 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiFetchRequired, apiPostRequired } from "./useApi"; +import { scheduledJobKeys } from "./useScheduledJobs"; + +export type JobResourceClass = + "interactive" | "light" | "network" | "host-heavy" | "exclusive"; +export type JobExecutionStatus = + "queued" | "running" | "success" | "failed" | "cancelled"; + +export interface JobExecution { + id: string; + scheduledJobId?: string | undefined; + scheduledRunId?: number | undefined; + actionKey: string; + displayName: string; + resourceClass: JobResourceClass; + status: JobExecutionStatus; + triggerType: "manual" | "schedule" | "startup" | "system"; + queuedAt: string; + availableAt: string; + startedAt?: string | undefined; + finishedAt?: string | undefined; + heartbeatAt?: string | undefined; + cancelRequestedAt?: string | undefined; + cancellable: boolean; + attempt: number; + message?: string | undefined; +} + +export interface JobExecutionSummary { + activeResourceClasses: JobResourceClass[]; + oldestQueuedAgeMs?: number | undefined; + oldestQueuedAt?: string | undefined; + queued: number; + running: number; + workerCapacity: number; + workerCount: number; + workerLastHeartbeatAt?: string | undefined; + workerOnline: boolean; +} + +interface JobExecutionsResponse { + executions: JobExecution[]; + summary: JobExecutionSummary; +} + +export const jobExecutionKeys = { + all: ["job-executions"] as const, + list: () => [...jobExecutionKeys.all, "list"] as const, +}; + +export function useJobExecutions() { + return useQuery({ + queryKey: jobExecutionKeys.list(), + queryFn: () => apiFetchRequired("/job-executions"), + refetchInterval: (query) => { + const summary = query.state.data?.summary; + return summary && (summary.queued > 0 || summary.running > 0) ? 2000 : 15_000; + }, + }); +} + +export function useCancelJobExecution() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + apiPostRequired<{ execution: JobExecution; isOk: boolean }>( + `/job-executions/${encodeURIComponent(id)}/cancel` + ), + onSuccess: (result) => { + void queryClient.invalidateQueries({ queryKey: jobExecutionKeys.all }); + void queryClient.invalidateQueries({ queryKey: scheduledJobKeys.list() }); + if (result.execution.scheduledJobId) { + void queryClient.invalidateQueries({ + queryKey: scheduledJobKeys.runs(result.execution.scheduledJobId), + }); + } + }, + }); +} diff --git a/src/hooks/useScheduledJobs.ts b/src/hooks/useScheduledJobs.ts index 5af657540..6c1ef2134 100644 --- a/src/hooks/useScheduledJobs.ts +++ b/src/hooks/useScheduledJobs.ts @@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import type { JobDisableIntent } from "../types/job"; import { apiFetchRequired, apiPatchRequired, apiPostRequired } from "./useApi"; +import type { JobResourceClass } from "./useJobExecutions"; /** Represents a backend-native scheduled job. */ export interface ScheduledJob { @@ -20,6 +21,9 @@ export interface ScheduledJob { createdAt: string; updatedAt: string; lastRun?: ScheduledJobRun | undefined; + resourceClass: JobResourceClass; + timeoutMs: number; + isQueued: boolean; isRunning: boolean; } @@ -27,12 +31,17 @@ export interface ScheduledJob { export interface ScheduledJobRun { id: number; jobId: string; - status: "running" | "success" | "failed"; - triggerType: "manual" | "schedule"; + status: "queued" | "running" | "success" | "failed" | "cancelled"; + triggerType: "manual" | "schedule" | "startup"; startedAt: string; + queuedAt: string; finishedAt?: string | undefined; message?: string | undefined; output: Record; + executionId?: string | undefined; + resourceClass: JobResourceClass; + cancelRequestedAt?: string | undefined; + cancellable: boolean; } export type ScheduledJobPatch = Partial< @@ -51,6 +60,36 @@ interface ScheduledJobsResponse { jobs: ScheduledJob[]; } +const legacyScheduledJobTimeoutMs = 5 * 60 * 1000; + +function normalizeScheduledJobRun( + run: ScheduledJobRun, + resourceClass: JobResourceClass +): ScheduledJobRun { + return { + ...run, + cancellable: run.cancellable ?? false, + queuedAt: run.queuedAt ?? run.startedAt, + resourceClass: run.resourceClass ?? resourceClass, + }; +} + +/** Keeps a new frontend usable during the brief old-backend deployment window. */ +function normalizeScheduledJob(job: ScheduledJob): ScheduledJob { + const resourceClass = job.resourceClass ?? "light"; + const lastRun = job.lastRun + ? normalizeScheduledJobRun(job.lastRun, resourceClass) + : undefined; + return { + ...job, + isQueued: job.isQueued ?? lastRun?.status === "queued", + isRunning: job.isRunning ?? lastRun?.status === "running", + lastRun, + resourceClass, + timeoutMs: job.timeoutMs ?? legacyScheduledJobTimeoutMs, + }; +} + /** Preserves a failed scheduled run so callers can surface its recorded output. */ export class ScheduledJobRunError extends Error { readonly run: ScheduledJobRun; @@ -74,8 +113,11 @@ export function useScheduledJobs() { return useQuery({ queryKey: scheduledJobKeys.list(), queryFn: () => apiFetchRequired("/jobs"), - select: (data) => data.jobs, - refetchInterval: 30_000, + select: (data) => data.jobs.map((job) => normalizeScheduledJob(job)), + refetchInterval: (query) => + query.state.data?.jobs.some((job) => job.isQueued || job.isRunning) + ? 2000 + : 30_000, }); } @@ -87,9 +129,14 @@ export function useScheduledJobRuns(id: string) { apiFetchRequired( `/jobs/${encodeURIComponent(id)}/runs` ), - select: (data) => data.runs, + select: (data) => data.runs.map((run) => normalizeScheduledJobRun(run, "light")), enabled: id.length > 0, - refetchInterval: 30_000, + refetchInterval: (query) => + query.state.data?.runs.some( + (run) => run.status === "queued" || run.status === "running" + ) + ? 2000 + : 30_000, }); } @@ -121,7 +168,11 @@ export function useRunScheduledJobNow() { const result = await apiPostRequired<{ isOk: boolean; run: ScheduledJobRun }>( `/jobs/${encodeURIComponent(id)}/run` ); - if (!result.isOk || result.run.status === "failed") { + if ( + !result.isOk || + result.run.status === "failed" || + result.run.status === "cancelled" + ) { throw new ScheduledJobRunError(result.run); } return result; diff --git a/src/pages/Jobs.tsx b/src/pages/Jobs.tsx index f1fff25d4..008e547dc 100644 --- a/src/pages/Jobs.tsx +++ b/src/pages/Jobs.tsx @@ -3,6 +3,7 @@ import { useEffect, useRef, useState } from "react"; import { CronJobDetails, CronJobList } from "../components/features/cron"; import { JobDisableIntentStatus } from "../components/features/jobs/JobDisableIntentStatus"; +import { JobExecutionQueueCard } from "../components/features/jobs/JobExecutionQueueCard"; import { Alert } from "../components/ui/Alert"; import { Badge } from "../components/ui/Badge"; import { Button } from "../components/ui/Button"; @@ -143,7 +144,9 @@ function getInitialCronJobId(): string { function scheduledJobStatusVariant(job: ScheduledJob) { if (!job.enabled) return "warning" as const; + if (job.isQueued || job.lastRun?.status === "queued") return "info" as const; if (job.isRunning || job.lastRun?.status === "running") return "info" as const; + if (job.lastRun?.status === "cancelled") return "warning" as const; if (job.lastRun?.status === "failed") return "error" as const; if (job.lastRun?.status === "success") return "success" as const; return "default" as const; @@ -151,6 +154,7 @@ function scheduledJobStatusVariant(job: ScheduledJob) { function scheduledJobStatusLabel(job: ScheduledJob): string { if (!job.enabled) return "Disabled"; + if (job.isQueued || job.lastRun?.status === "queued") return "Queued"; if (job.isRunning || job.lastRun?.status === "running") return "Running"; return job.lastRun?.status || "Never run"; } @@ -340,7 +344,7 @@ function ScheduledJobDetails({ @@ -450,7 +460,13 @@ function ScheduledJobDetails({ ) : undefined} -
+
+ +
Resource class
+
+ {(job.resourceClass ?? "light").replace("-", " ")} +
+
Action
@@ -496,7 +512,10 @@ function ScheduledJobDetails({ {run.triggerType} run #{run.id}
- Started {formatDate(run.startedAt)} + Queued {formatDate(run.queuedAt)} + {run.status === "queued" + ? "" + : ` · started ${formatDate(run.startedAt)}`} {run.finishedAt ? ` · finished ${formatDate(run.finishedAt)}` : ""} @@ -514,7 +533,9 @@ function ScheduledJobDetails({ ? "success" : run.status === "failed" ? "error" - : "info" + : run.status === "cancelled" + ? "warning" + : "info" } > {run.status} @@ -941,45 +962,52 @@ export function Jobs() {

) : view === "scheduled" ? ( -
- - { - handleScheduledToggle( - currentScheduledJob as ScheduledJob, - isEnabled - ); - }} - onConfigureDisable={() => - openDisableModal({ - kind: "scheduled", - job: currentScheduledJob as ScheduledJob, - }) - } - onRunNow={() => { - void handleScheduledRun(currentScheduledJob as ScheduledJob); - }} - onSave={() => { - void handleScheduledSave(currentScheduledJob as ScheduledJob); - }} - /> +
+ +
+ + { + handleScheduledToggle( + currentScheduledJob as ScheduledJob, + isEnabled + ); + }} + onConfigureDisable={() => + openDisableModal({ + kind: "scheduled", + job: currentScheduledJob as ScheduledJob, + }) + } + onRunNow={() => { + void handleScheduledRun( + currentScheduledJob as ScheduledJob + ); + }} + onSave={() => { + void handleScheduledSave( + currentScheduledJob as ScheduledJob + ); + }} + /> +
) : (
diff --git a/src/test/componentBehavior.test.tsx b/src/test/componentBehavior.test.tsx index 5b0fe0065..8f69f1de6 100644 --- a/src/test/componentBehavior.test.tsx +++ b/src/test/componentBehavior.test.tsx @@ -78,6 +78,7 @@ import { PreviewToggle } from "../components/features/files/PreviewToggle"; import { CodePreview } from "../components/features/files/viewers/CodePreview"; import { JsonPreview } from "../components/features/files/viewers/JsonPreview"; import { MarkdownPreview } from "../components/features/files/viewers/MarkdownPreview"; +import { JobExecutionQueueCard } from "../components/features/jobs/JobExecutionQueueCard"; import { LogLine } from "../components/features/logs/LogLine"; import { MyCommentCard } from "../components/features/moltbook/MyCommentCard"; import { MyPostCard } from "../components/features/moltbook/MyPostCard"; @@ -3177,6 +3178,94 @@ describe("shared component helpers", () => { queryClient.clear(); }); + it("shows queue pressure and cancels an active job execution", async () => { + let executionStatus: "cancelled" | "queued" = "queued"; + const fetchMock = jest.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + const execution = { + actionKey: "backup.run", + attempt: 0, + availableAt: "2026-07-22T10:00:00.000Z", + cancelRequestedAt: undefined, + cancellable: true, + displayName: "Host backup", + finishedAt: + executionStatus === "cancelled" + ? "2026-07-22T10:01:00.000Z" + : undefined, + id: "execution-1", + queuedAt: "2026-07-22T10:00:00.000Z", + resourceClass: "host-heavy", + scheduledJobId: "backup.kopia", + scheduledRunId: 42, + startedAt: undefined, + status: executionStatus, + triggerType: "manual", + }; + + if (url === "/api/job-executions" && method === "GET") { + return Response.json({ + executions: [execution], + summary: { + activeResourceClasses: [], + oldestQueuedAt: + executionStatus === "queued" + ? execution.queuedAt + : undefined, + queued: executionStatus === "queued" ? 1 : 0, + running: 0, + workerCapacity: 1, + workerCount: 1, + workerOnline: true, + }, + }); + } + + if ( + url === "/api/job-executions/execution-1/cancel" && + method === "POST" + ) { + executionStatus = "cancelled"; + return Response.json({ + execution: { ...execution, status: "cancelled" }, + isOk: true, + }); + } + + throw new Error(`Unexpected queue API call: ${method} ${url}`); + } + ); + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: fetchMock, + writable: true, + }); + + const queryClient = createQueryClient(); + render( + + + + ); + + expect(await screen.findByText("Host backup")).toBeInTheDocument(); + expect(screen.getByText("host heavy")).toBeInTheDocument(); + expect(screen.getByText("Worker idle")).toBeInTheDocument(); + + await userEvent.click(screen.getByRole("button", { name: "Cancel Host backup" })); + + await waitFor(() => { + expect(executionStatus).toBe("cancelled"); + expect( + screen.queryByRole("button", { name: "Cancel Host backup" }) + ).not.toBeInTheDocument(); + }); + + queryClient.clear(); + }); + it("marks unavailable Git repositories as missing instead of clean", async () => { Object.defineProperty(globalThis, "fetch", { configurable: true, diff --git a/src/test/frontendBehavior.test.tsx b/src/test/frontendBehavior.test.tsx index a652ce3e1..e51eb6f03 100644 --- a/src/test/frontendBehavior.test.tsx +++ b/src/test/frontendBehavior.test.tsx @@ -103,6 +103,7 @@ import { useWalgBackup, } from "../hooks/useBackups"; import { + cacheKeys, useCacheEntry, useCacheHeartbeat, useCacheStatus, @@ -2453,6 +2454,65 @@ describe("Mira Dashboard frontend behavior", () => { ).resolves.toMatchObject({ keys: ["weather.spydeberg"] }); }); + it("attempts every requested cache refresh and retains partial successes", async () => { + const originalFetch = fetch; + const fetchMock = jest.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (init?.method !== "POST") { + throw new Error(`Unexpected cache API call: ${init?.method} ${url}`); + } + if (url === "/api/cache/cache.fail/refresh") { + return Response.json({ error: "refresh failed" }, { status: 500 }); + } + const key = url.replace("/api/cache/", "").replace("/refresh", ""); + return Response.json({ + entry: { + consecutiveFailures: 0, + data: { key }, + key, + meta: {}, + source: "test", + status: "fresh", + }, + isOk: true, + }); + } + ); + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: fetchMock, + writable: true, + }); + + try { + const refreshCache = renderHookWithQueryClient(() => useRefreshCacheEntry()); + await expect( + refreshCache.result.current.mutateAsync( + "cache.first,cache.fail,cache.last" + ) + ).rejects.toThrow("refresh failed"); + + expect(fetchMock.mock.calls.map(([input]) => String(input))).toEqual([ + "/api/cache/cache.first/refresh", + "/api/cache/cache.fail/refresh", + "/api/cache/cache.last/refresh", + ]); + expect( + refreshCache.queryClient.getQueryData(cacheKeys.entry("cache.first")) + ).toMatchObject({ key: "cache.first" }); + expect( + refreshCache.queryClient.getQueryData(cacheKeys.entry("cache.last")) + ).toMatchObject({ key: "cache.last" }); + } finally { + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: originalFetch, + writable: true, + }); + } + }); + it("clears cached Docker stats once live containers report no stats", async () => { const fetchMock = jest.fn( async (input: RequestInfo | URL, init?: RequestInit) => { diff --git a/src/test/pageBehavior.test.tsx b/src/test/pageBehavior.test.tsx index 64bdc565e..7940bf127 100644 --- a/src/test/pageBehavior.test.tsx +++ b/src/test/pageBehavior.test.tsx @@ -1265,12 +1265,29 @@ function apiResponse(url: string, method: string, init?: RequestInit) { createdAt: "2026-06-24T08:00:00.000Z", updatedAt: "2026-06-24T08:00:00.000Z", lastRun: jobsApiState.heartbeatRuns[0], + resourceClass: "light", + timeoutMs: 300_000, + isQueued: false, isRunning: false, }, ], }); } + if (url === "/api/job-executions") { + return Response.json({ + executions: [], + summary: { + activeResourceClasses: [], + queued: 0, + running: 0, + workerCapacity: 1, + workerCount: 1, + workerOnline: true, + }, + }); + } + if (url === "/api/reports") { return Response.json({ items: [ @@ -1333,6 +1350,9 @@ function apiResponse(url: string, method: string, init?: RequestInit) { actionPayload: {}, createdAt: "2026-06-24T08:00:00.000Z", updatedAt: "2026-06-24T08:05:00.000Z", + resourceClass: "light", + timeoutMs: 300_000, + isQueued: false, isRunning: false, }, }); @@ -1360,6 +1380,9 @@ function apiResponse(url: string, method: string, init?: RequestInit) { actionPayload: {}, createdAt: "2026-06-24T08:00:00.000Z", updatedAt: "2026-06-24T08:05:00.000Z", + resourceClass: "light", + timeoutMs: 300_000, + isQueued: false, isRunning: false, }, }); diff --git a/src/test/setup.ts b/src/test/setup.ts index 9c1d4432d..4ab084a85 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -1,11 +1,13 @@ import { GlobalRegistrator } from "@happy-dom/global-registrator"; -import * as matchers from "@testing-library/jest-dom/matchers"; import { afterEach, expect, jest } from "bun:test"; -expect.extend(matchers); - GlobalRegistrator.register(); +// jest-dom v7 eagerly imports Testing Library's document-bound helpers. +// Register Happy DOM before loading the matchers so `screen` binds to this document. +const matcherModule = await import("@testing-library/jest-dom/matchers"); +expect.extend(matcherModule as Omit); + Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", { configurable: true, value: true, diff --git a/systemd/mira-dashboard-worker.service b/systemd/mira-dashboard-worker.service new file mode 100644 index 000000000..ad54e10dc --- /dev/null +++ b/systemd/mira-dashboard-worker.service @@ -0,0 +1,26 @@ +[Unit] +Description=Mira Dashboard job worker +After=network-online.target openclaw-gateway.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/ubuntu/projects/mira-dashboard/backend +Environment=NODE_ENV=production +Environment=MIRA_DASHBOARD_EXECUTION_ROLE=worker +Environment=MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 +Environment=MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard-worker.service +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/workerStart.js +Restart=on-failure +RestartSec=5 +KillMode=control-group +TimeoutStopSec=45 +Nice=10 +CPUWeight=20 +IOWeight=20 +MemoryHigh=4G +MemoryMax=6G +TasksMax=256 + +[Install] +WantedBy=default.target diff --git a/systemd/mira-dashboard.service b/systemd/mira-dashboard.service new file mode 100644 index 000000000..7c5b8a1b1 --- /dev/null +++ b/systemd/mira-dashboard.service @@ -0,0 +1,25 @@ +[Unit] +Description=Mira Dashboard web +After=network-online.target openclaw-gateway.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/ubuntu/projects/mira-dashboard/backend +Environment=NODE_ENV=production +Environment=MIRA_DASHBOARD_EXECUTION_ROLE=web +Environment=MIRA_DASHBOARD_ENABLE_JOB_SCOPES=1 +Environment=MIRA_DASHBOARD_JOB_SCOPE_OWNER=mira-dashboard.service +ExecStart=/usr/local/bin/doppler run --config prd --project rajohan -- /home/ubuntu/.bun/bin/bun dist/serverStart.js +Restart=on-failure +RestartSec=5 +KillMode=control-group +TimeoutStopSec=45 +CPUWeight=100 +IOWeight=100 +MemoryHigh=3G +MemoryMax=5G +TasksMax=256 + +[Install] +WantedBy=default.target