diff --git a/packages/1-prisma-cloud/0-lowering/dev-emulators/src/__tests__/postgres.test.ts b/packages/1-prisma-cloud/0-lowering/dev-emulators/src/__tests__/postgres.test.ts index 1c06c6cf3..237d9e063 100644 --- a/packages/1-prisma-cloud/0-lowering/dev-emulators/src/__tests__/postgres.test.ts +++ b/packages/1-prisma-cloud/0-lowering/dev-emulators/src/__tests__/postgres.test.ts @@ -70,7 +70,11 @@ describe('database lifecycle', () => { await client.deleteApp('pgtest-lifecycle'); expect(await client.listDatabases('pgtest-lifecycle')).toHaveLength(0); - }, 30_000); + // 60s, not 30s: this boots a real Postgres and every step is awaited, so + // there is no race to fix here — it is simply slow, and 30s left no room + // for a loaded CI runner (observed timing out at 30151ms against a ~8s + // normal run). The heavier tests below already sit at 45-120s. + }, 60_000); test('ensure is idempotent — a second call returns the same URL without restarting', async () => { await ensureFreshDaemon('postgres', registryRoot); @@ -81,7 +85,8 @@ describe('database lifecycle', () => { expect(second.url).toBe(first.url); await client.deleteApp('pgtest-idempotent'); - }, 30_000); + // Same daemon-boot cost as the test above, so the same budget. + }, 60_000); }); describe('port stability across a daemon restart', () => { diff --git a/packages/1-prisma-cloud/0-lowering/dev-emulators/src/daemon.ts b/packages/1-prisma-cloud/0-lowering/dev-emulators/src/daemon.ts index 7c8943382..4f4077ca6 100644 --- a/packages/1-prisma-cloud/0-lowering/dev-emulators/src/daemon.ts +++ b/packages/1-prisma-cloud/0-lowering/dev-emulators/src/daemon.ts @@ -40,8 +40,19 @@ const MAX_PORT = 65_535; const EXISTING_HEALTH_TIMEOUT_MS = 2000; const START_HEALTH_BUDGET_MS = 10_000; const HEALTH_POLL_INTERVAL_MS = 200; -const TERMINATE_GRACE_MS = 5000; +/** + * How long SIGTERM gets before SIGKILL. Generous on purpose: the daemon hosts + * its `@prisma/dev` servers in-process, and only a graceful exit closes them + * and releases each server's name lock. A SIGKILLed daemon leaves those locks + * behind, and the next daemon then has to wait out proper-lockfile's stale + * threshold before it can start the same database — which surfaces as + * "already running". The wait below ends as soon as the process is gone, so + * this budget is only ever spent on a daemon that is genuinely not exiting. + */ +const TERMINATE_GRACE_MS = 20_000; const TERMINATE_POLL_INTERVAL_MS = 150; +/** After SIGKILL, how long to wait for the OS to actually reap the process. */ +const TERMINATE_KILL_WAIT_MS = 5000; const LOCK_RETRY_INTERVAL_MS = 250; const LOCK_WAIT_BUDGET_MS = 10_000; const LOCK_STALE_MS = 10_000; @@ -234,7 +245,15 @@ async function terminate(pid: number, graceMs: number): Promise { try { process.kill(pid, 'SIGKILL'); } catch { - // already gone + return; // already gone + } + // SIGKILL is delivered asynchronously. Returning here would let the caller + // start a replacement daemon while this one still holds its ports and + // its servers' name locks. + const killDeadline = Date.now() + TERMINATE_KILL_WAIT_MS; + while (Date.now() < killDeadline) { + if (!isPidAlive(pid)) return; + await sleep(TERMINATE_POLL_INTERVAL_MS); } } } diff --git a/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/state-api.test.ts b/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/state-api.test.ts index 1f3a80e37..5235f63e7 100644 --- a/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/state-api.test.ts +++ b/packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/state-api.test.ts @@ -42,6 +42,15 @@ const scope: LeaseScope = { let fake: FakeStateApi; +/** Resolves once `condition` holds, so a test waits for work instead of racing a fixed sleep against it. */ +async function until(condition: () => boolean, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() >= deadline) throw new Error(`condition still false after ${timeoutMs}ms`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + beforeAll(async () => { fake = new FakeStateApi(); await fake.start(); @@ -246,7 +255,10 @@ describe('the deploy lease', () => { const fiber = yield* Effect.forkChild( heartbeatDeployLease(sdkClient(), scope, lease, '10 millis'), ); - yield* Effect.sleep('100 millis'); + // Wait for the second heartbeat rather than sleeping a fixed span and + // hoping it fits: a loaded machine can starve the interval for longer + // than any sleep worth writing here, which is what made this flake. + yield* Effect.promise(() => until(() => fake.countRequests(/PATCH .*\/lease/) >= 2)); yield* Fiber.interrupt(fiber); }), ); diff --git a/scripts/ci-cleanup-utils.test.ts b/scripts/ci-cleanup-utils.test.ts index f017a7151..2ed2587dc 100644 --- a/scripts/ci-cleanup-utils.test.ts +++ b/scripts/ci-cleanup-utils.test.ts @@ -131,6 +131,33 @@ describe('deleteProjectDeep', () => { } }); + it('reports a thrown transport error as "not gone" instead of propagating it', async () => { + // The sweep in ci-cleanup.ts deletes projects in a loop, so a thrown + // error here would abandon every project after this one. + const log: string[] = []; + const throwing: HttpCall = () => Promise.reject(new Error('socket hang up')); + assert.equal(await deleteProjectDeep(throwing, PROJECT, fastOpts(log)), false); + assert.ok(log.some((l) => l.includes('socket hang up'))); + }); + + it('retries past a transport blip in the post-teardown project delete', async () => { + const NO_SERVICES: HttpResponse = { status: 200, ok: true, body: '{"data":[]}' }; + const steps: (HttpResponse | Error)[] = [ + ACTIVE_DEPLOYMENT, // DELETE /projects — live compute blocks it + NO_SERVICES, // GET /apps + new Error('ECONNRESET'), // DELETE /projects — the blip + OK, // DELETE /projects — the retry + ]; + let i = 0; + const flaky: HttpCall = () => { + const next = steps[i++]; + if (next === undefined) throw new Error('unscripted call'); + return next instanceof Error ? Promise.reject(next) : Promise.resolve(next); + }; + assert.equal(await deleteProjectDeep(flaky, PROJECT, fastOpts()), true); + assert.equal(i, 4); + }); + it('on 409 active-deployment: lists services, deletes each, then retries the project delete', async () => { const log: string[] = []; const { http, calls } = scriptedHttp({ diff --git a/scripts/ci-cleanup-utils.ts b/scripts/ci-cleanup-utils.ts index cae20ad5c..577b6f436 100644 --- a/scripts/ci-cleanup-utils.ts +++ b/scripts/ci-cleanup-utils.ts @@ -120,7 +120,19 @@ export async function deleteProjectDeep( const projectAttempts = opts.projectDeleteAttempts ?? 6; const projectDelayMs = opts.projectDeleteDelayMs ?? 5_000; - const first = await http('DELETE', `/projects/${project.id}`); + // A sweep deletes many projects in a loop, so a thrown transport error + // would abandon every project after this one. Reported as an unusable + // response instead: "gone or not gone" is all a caller can act on, and the + // project-delete retry below then treats a blip as worth another attempt. + const call = async (method: 'GET' | 'DELETE', path: string): Promise => { + try { + return await http(method, path); + } catch (error) { + return { status: 0, ok: false, body: `transport error: ${String(error)}` }; + } + }; + + const first = await call('DELETE', `/projects/${project.id}`); if (first.ok || first.status === 404) return true; if (!isActiveDeployment409(first)) { opts.log( @@ -131,7 +143,7 @@ export async function deleteProjectDeep( // Live compute blocks the project delete — enumerate and tear down. opts.log(` "${project.name}" has an active deployment — tearing its apps down…`); - const listed = await http('GET', `/apps?projectId=${project.id}&limit=100`); + const listed = await call('GET', `/apps?projectId=${project.id}&limit=100`); if (!listed.ok) { opts.log(` could not list apps for "${project.name}": ${listed.status} ${listed.body}`); return false; @@ -139,7 +151,7 @@ export async function deleteProjectDeep( for (const service of parseServiceRows(listed.body)) { opts.log(` deleting app "${service.name}" (${service.id})…`); for (let attempt = 1; attempt <= serviceAttempts; attempt++) { - const res = await http('DELETE', `/apps/${service.id}`); + const res = await call('DELETE', `/apps/${service.id}`); if (res.ok || res.status === 404) break; if (isDeleteNotSafeYet409(res) && attempt < serviceAttempts) { // The deployment is still winding down — the one retryable state. @@ -153,7 +165,7 @@ export async function deleteProjectDeep( // Services gone (or as gone as they get) — re-try the project delete. for (let attempt = 1; attempt <= projectAttempts; attempt++) { - const res = await http('DELETE', `/projects/${project.id}`); + const res = await call('DELETE', `/projects/${project.id}`); if (res.ok || res.status === 404) return true; if (attempt < projectAttempts) { await sleep(projectDelayMs);