fix(ci): stop three timing races failing unrelated PRs

packages/1-prisma-cloud/0-lowering/dev-emulators/src/__tests__/postgres.test.ts

@@ -70,7 +70,11 @@ describe('database lifecycle', () => {
 
     await client.deleteApp('pgtest-lifecycle');
     expect(await client.listDatabases('pgtest-lifecycle')).toHaveLength(0);
-  }, 30_000);
+    // 60s, not 30s: this boots a real Postgres and every step is awaited, so
+    // there is no race to fix here — it is simply slow, and 30s left no room
+    // for a loaded CI runner (observed timing out at 30151ms against a ~8s
+    // normal run). The heavier tests below already sit at 45-120s.
+  }, 60_000);
 
   test('ensure is idempotent — a second call returns the same URL without restarting', async () => {
     await ensureFreshDaemon('postgres', registryRoot);
@@ -81,7 +85,8 @@ describe('database lifecycle', () => {
     expect(second.url).toBe(first.url);
 
     await client.deleteApp('pgtest-idempotent');
-  }, 30_000);
+    // Same daemon-boot cost as the test above, so the same budget.
+  }, 60_000);
 });
 
 describe('port stability across a daemon restart', () => {

packages/1-prisma-cloud/0-lowering/dev-emulators/src/daemon.ts

@@ -40,8 +40,19 @@ const MAX_PORT = 65_535;
 const EXISTING_HEALTH_TIMEOUT_MS = 2000;
 const START_HEALTH_BUDGET_MS = 10_000;
 const HEALTH_POLL_INTERVAL_MS = 200;
-const TERMINATE_GRACE_MS = 5000;
+/**
+ * How long SIGTERM gets before SIGKILL. Generous on purpose: the daemon hosts
+ * its `@prisma/dev` servers in-process, and only a graceful exit closes them
+ * and releases each server's name lock. A SIGKILLed daemon leaves those locks
+ * behind, and the next daemon then has to wait out proper-lockfile's stale
+ * threshold before it can start the same database — which surfaces as
+ * "already running". The wait below ends as soon as the process is gone, so
+ * this budget is only ever spent on a daemon that is genuinely not exiting.
+ */
+const TERMINATE_GRACE_MS = 20_000;
 const TERMINATE_POLL_INTERVAL_MS = 150;
+/** After SIGKILL, how long to wait for the OS to actually reap the process. */
+const TERMINATE_KILL_WAIT_MS = 5000;
 const LOCK_RETRY_INTERVAL_MS = 250;
 const LOCK_WAIT_BUDGET_MS = 10_000;
 const LOCK_STALE_MS = 10_000;
@@ -234,7 +245,15 @@ async function terminate(pid: number, graceMs: number): Promise<void> {
     try {
       process.kill(pid, 'SIGKILL');
     } catch {
-      // already gone
+      return; // already gone
+    }
+    // SIGKILL is delivered asynchronously. Returning here would let the caller
+    // start a replacement daemon while this one still holds its ports and
+    // its servers' name locks.
+    const killDeadline = Date.now() + TERMINATE_KILL_WAIT_MS;
+    while (Date.now() < killDeadline) {
+      if (!isPidAlive(pid)) return;
+      await sleep(TERMINATE_POLL_INTERVAL_MS);
     }
   }
 }

packages/1-prisma-cloud/0-lowering/lowering/src/state/__tests__/state-api.test.ts

@@ -42,6 +42,15 @@ const scope: LeaseScope = {
 
 let fake: FakeStateApi;
 
+/** Resolves once `condition` holds, so a test waits for work instead of racing a fixed sleep against it. */
+async function until(condition: () => boolean, timeoutMs = 10_000): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (!condition()) {
+    if (Date.now() >= deadline) throw new Error(`condition still false after ${timeoutMs}ms`);
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+}
+
 beforeAll(async () => {
   fake = new FakeStateApi();
   await fake.start();
@@ -246,7 +255,10 @@ describe('the deploy lease', () => {
         const fiber = yield* Effect.forkChild(
           heartbeatDeployLease(sdkClient(), scope, lease, '10 millis'),
         );
-        yield* Effect.sleep('100 millis');
+        // Wait for the second heartbeat rather than sleeping a fixed span and
+        // hoping it fits: a loaded machine can starve the interval for longer
+        // than any sleep worth writing here, which is what made this flake.
+        yield* Effect.promise(() => until(() => fake.countRequests(/PATCH .*\/lease/) >= 2));
         yield* Fiber.interrupt(fiber);
       }),
     );

scripts/ci-cleanup-utils.test.ts

@@ -131,6 +131,33 @@ describe('deleteProjectDeep', () => {
     }
   });
 
+  it('reports a thrown transport error as "not gone" instead of propagating it', async () => {
+    // The sweep in ci-cleanup.ts deletes projects in a loop, so a thrown
+    // error here would abandon every project after this one.
+    const log: string[] = [];
+    const throwing: HttpCall = () => Promise.reject(new Error('socket hang up'));
+    assert.equal(await deleteProjectDeep(throwing, PROJECT, fastOpts(log)), false);
+    assert.ok(log.some((l) => l.includes('socket hang up')));
+  });
+
+  it('retries past a transport blip in the post-teardown project delete', async () => {
+    const NO_SERVICES: HttpResponse = { status: 200, ok: true, body: '{"data":[]}' };
+    const steps: (HttpResponse | Error)[] = [
+      ACTIVE_DEPLOYMENT, // DELETE /projects — live compute blocks it
+      NO_SERVICES, // GET /apps
+      new Error('ECONNRESET'), // DELETE /projects — the blip
+      OK, // DELETE /projects — the retry
+    ];
+    let i = 0;
+    const flaky: HttpCall = () => {
+      const next = steps[i++];
+      if (next === undefined) throw new Error('unscripted call');
+      return next instanceof Error ? Promise.reject(next) : Promise.resolve(next);
+    };
+    assert.equal(await deleteProjectDeep(flaky, PROJECT, fastOpts()), true);
+    assert.equal(i, 4);
+  });
+
   it('on 409 active-deployment: lists services, deletes each, then retries the project delete', async () => {
     const log: string[] = [];
     const { http, calls } = scriptedHttp({

scripts/ci-cleanup-utils.ts

@@ -120,7 +120,19 @@ export async function deleteProjectDeep(
   const projectAttempts = opts.projectDeleteAttempts ?? 6;
   const projectDelayMs = opts.projectDeleteDelayMs ?? 5_000;
 
-  const first = await http('DELETE', `/projects/${project.id}`);
+  // A sweep deletes many projects in a loop, so a thrown transport error
+  // would abandon every project after this one. Reported as an unusable
+  // response instead: "gone or not gone" is all a caller can act on, and the
+  // project-delete retry below then treats a blip as worth another attempt.
+  const call = async (method: 'GET' | 'DELETE', path: string): Promise<HttpResponse> => {
+    try {
+      return await http(method, path);
+    } catch (error) {
+      return { status: 0, ok: false, body: `transport error: ${String(error)}` };
+    }
+  };
+
+  const first = await call('DELETE', `/projects/${project.id}`);
   if (first.ok || first.status === 404) return true;
   if (!isActiveDeployment409(first)) {
     opts.log(
@@ -131,15 +143,15 @@ export async function deleteProjectDeep(
 
   // Live compute blocks the project delete — enumerate and tear down.
   opts.log(`  "${project.name}" has an active deployment — tearing its apps down…`);
-  const listed = await http('GET', `/apps?projectId=${project.id}&limit=100`);
+  const listed = await call('GET', `/apps?projectId=${project.id}&limit=100`);
   if (!listed.ok) {
     opts.log(`  could not list apps for "${project.name}": ${listed.status} ${listed.body}`);
     return false;
   }
   for (const service of parseServiceRows(listed.body)) {
     opts.log(`    deleting app "${service.name}" (${service.id})…`);
     for (let attempt = 1; attempt <= serviceAttempts; attempt++) {
-      const res = await http('DELETE', `/apps/${service.id}`);
+      const res = await call('DELETE', `/apps/${service.id}`);
       if (res.ok || res.status === 404) break;
       if (isDeleteNotSafeYet409(res) && attempt < serviceAttempts) {
         // The deployment is still winding down — the one retryable state.
@@ -153,7 +165,7 @@ export async function deleteProjectDeep(
 
   // Services gone (or as gone as they get) — re-try the project delete.
   for (let attempt = 1; attempt <= projectAttempts; attempt++) {
-    const res = await http('DELETE', `/projects/${project.id}`);
+    const res = await call('DELETE', `/projects/${project.id}`);
     if (res.ok || res.status === 404) return true;
     if (attempt < projectAttempts) {
       await sleep(projectDelayMs);