feat: isolate E2E test execution in sandboxes

shared/ports.ts

@@ -53,6 +53,41 @@ export const PROXY_FALLBACK_PORT_START = PROXY_PORT_BASE + PROXY_PORT_RANGE;
 /** How many consecutive fallback ports to scan before giving up. */
 export const PROXY_FALLBACK_MAX_ATTEMPTS = 50;
 
+/**
+ * Start of the band reserved for run-scoped E2E test servers. It sits directly
+ * above the proxy fallback band so a sandbox server never squats on the
+ * deterministic app or proxy port of some *other* app that merely happens to be
+ * stopped right now — which would make that app fail to start later, with
+ * nothing to point at as the cause. An OS-assigned ephemeral port cannot give
+ * that guarantee: on Linux the ephemeral range (32768–60999) covers almost all
+ * of the reserved bands below.
+ */
+export const E2E_TEST_SERVER_PORT_START =
+  PROXY_FALLBACK_PORT_START + PROXY_FALLBACK_MAX_ATTEMPTS;
+/** Width of the E2E test-server band, so it spans 52150..52349. */
+export const E2E_TEST_SERVER_PORT_RANGE = 200;
+
+/**
+ * Whether a port falls in a band Dyad hands out deterministically, and so must
+ * not be taken by anything that only needs *some* free port.
+ */
+export function isReservedDyadPort(port: number): boolean {
+  const e2ePortBlockBase = getE2ePortBlockBase();
+  if (
+    e2ePortBlockBase != null &&
+    port >= e2ePortBlockBase &&
+    port < e2ePortBlockBase + E2E_PORT_BLOCK_SIZE
+  ) {
+    return true;
+  }
+  return (
+    (port >= APP_PORT_BASE && port < APP_PORT_BASE + APP_PORT_RANGE) ||
+    (port >= PROXY_PORT_BASE && port < PROXY_PORT_BASE + PROXY_PORT_RANGE) ||
+    (port >= PROXY_FALLBACK_PORT_START &&
+      port < PROXY_FALLBACK_PORT_START + PROXY_FALLBACK_MAX_ATTEMPTS)
+  );
+}
+
 export function getProxyFallbackPortStart(): number {
   const e2ePortBlockBase = getE2ePortBlockBase();
   if (e2ePortBlockBase != null) {

src/atoms/testRuntimeAtoms.ts

@@ -44,7 +44,7 @@ export type TestRunPhase =
   | "setup" // first-run Playwright bootstrap streaming
   | "running" // playwright test executing
   | "stopping" // Stop pressed; killing the Playwright process tree
-  | "cleaning-up"; // tests gone; isolation teardown still restoring the app
+  | "cleaning-up"; // tests gone; isolated provider data is still being removed
 
 export interface TestRunState {
   phase: TestRunPhase;
@@ -72,6 +72,12 @@ export interface TestRunState {
    * a run completes.
    */
   isolation?: TestIsolation;
+  /**
+   * Whether the run executed in an isolated sandbox. Drives the cleanup copy:
+   * the fallback path (Docker/cloud runtime, or the user's opt-out) creates no
+   * workspace, so there is no sandbox to claim Dyad is deleting.
+   */
+  sandboxed?: boolean;
   startedAt?: number;
 }
 
@@ -275,6 +281,8 @@ export const applyTestRunStartedAtom = atom(
             ),
         runError: undefined,
         isolation: undefined,
+        // Reported by the main process once the run has picked its path.
+        sandboxed: undefined,
         startedAt: startedAt ?? Date.now(),
       }),
     });

src/components/SandboxedE2eTestsSwitch.tsx

@@ -0,0 +1,29 @@
+import { useSettings } from "@/hooks/useSettings";
+import { Label } from "@/components/ui/label";
+import { Switch } from "@/components/ui/switch";
+
+/**
+ * Escape hatch for the sandboxed E2E runtime. Stored inverted
+ * (`disableSandboxedE2eTests`) so the sandbox stays the default for everyone
+ * who never opens this, and only an explicit opt-out falls back to running
+ * against the normal preview.
+ */
+export function SandboxedE2eTestsSwitch() {
+  const { settings, updateSettings } = useSettings();
+  const enabled = !settings?.disableSandboxedE2eTests;
+  return (
+    <div className="flex items-center space-x-2">
+      <Switch
+        id="enable-sandboxed-e2e-tests"
+        aria-label="Run E2E Tests in an Isolated Sandbox"
+        checked={enabled}
+        onCheckedChange={(checked) => {
+          updateSettings({ disableSandboxedE2eTests: !checked });
+        }}
+      />
+      <Label htmlFor="enable-sandboxed-e2e-tests">
+        Run E2E Tests in an Isolated Sandbox
+      </Label>
+    </div>
+  );
+}

src/components/chat/CancellationBanner.test.tsx

@@ -18,10 +18,12 @@ vi.mock("react-i18next", () => ({
       ({
         stoppingGeneration: "Stopping…",
         cancellationEndingTestRun: "Ending the test run.",
-        cancellationRestoringTestApp:
-          "Restoring your app's database and preview. This can take a while.",
+        cancellationRemovingTestDatabase:
+          "Removing the temporary test database. This can take a while.",
         cancellationCleaningTestData:
           "Cleaning up the test data from this run.",
+        cancellationCleaningTestSandbox:
+          "Cleaning up this run's test sandbox and data.",
       })[key] ?? key,
   }),
 }));
@@ -30,6 +32,7 @@ function renderBanner(runState?: {
   phase: TestRunPhase;
   isolationMode?: TestIsolation["mode"];
   source?: "panel" | "agent";
+  sandboxed?: boolean;
 }) {
   const store = createStore();
   if (runState) {
@@ -45,6 +48,7 @@ function renderBanner(runState?: {
             isolation: runState.isolationMode
               ? { mode: runState.isolationMode }
               : undefined,
+            sandboxed: runState.sandboxed,
           },
         ],
       ]),
@@ -87,24 +91,46 @@ describe("CancellationBanner", () => {
     });
 
     expect(
-      screen.getByText(/Restoring your app's database and preview/),
+      screen.getByText(/Removing the temporary test database/),
     ).toBeTruthy();
     expect(screen.getByText(/can take a while/)).toBeTruthy();
+    // The run had its own sandbox and its own server; the user's preview and
+    // `.env.local` were never touched, so nothing is being restored.
+    expect(screen.queryByText(/Restoring/i)).toBeNull();
   });
 
-  it("does not claim a restore on the Supabase path", () => {
-    // That teardown only deletes the temporary test user — no env swap, no
-    // dev-server restart, nothing the user sees.
+  it("names the sandbox on the Supabase path when the run took one", () => {
+    // That teardown only deletes the temporary test user and the run's sandbox
+    // copy — no env swap, no dev-server restart, nothing the user sees.
     renderBanner({
       phase: "cleaning-up",
       isolationMode: "supabase-test-user",
       source: "agent",
+      sandboxed: true,
     });
 
-    expect(screen.getByText(/Cleaning up the test data/)).toBeTruthy();
+    expect(
+      screen.getByText(/Cleaning up this run's test sandbox/),
+    ).toBeTruthy();
     expect(screen.queryByText(/Restoring/)).toBeNull();
   });
 
+  it("claims no sandbox for a run that never took one", () => {
+    // The fallback path (Docker/cloud runtime, or the opt-out) creates no
+    // workspace, and neither does a run whose setup failed before the copy.
+    renderBanner({
+      phase: "cleaning-up",
+      isolationMode: "supabase-test-user",
+      source: "agent",
+      sandboxed: false,
+    });
+
+    expect(
+      screen.getByText("Cleaning up the test data from this run."),
+    ).toBeTruthy();
+    expect(screen.queryByText(/sandbox/i)).toBeNull();
+  });
+
   it("reports the kill before the teardown starts", () => {
     renderBanner({ phase: "stopping", source: "agent" });
 

src/components/chat/CancellationBanner.tsx

@@ -11,11 +11,11 @@ import {
  * Pinned above the composer while a stopped turn settles.
  *
  * Stopping is not instant. The agent awaits its in-flight tool, and a
- * `run_tests` call first kills the Playwright process tree and then runs an
- * isolation teardown that accepts no AbortSignal — restoring `.env.local`,
- * restarting the dev server and deleting the temporary Neon branch, whose
- * delete retries with backoff. That wait can pass a minute, and the composer
- * stays locked for all of it.
+ * `run_tests` call first kills the Playwright process tree and then runs a
+ * teardown that accepts no AbortSignal — deleting the temporary Neon branch,
+ * whose delete retries with backoff, and removing the run's sandbox copy of the
+ * app. That wait can pass a minute, and the composer stays locked for all of
+ * it. The user's own `.env.local` and preview are never touched.
  *
  * The transcript's inline status card scrolls out of view; this stays fused to
  * the composer the user just clicked Stop in, so the wait is never unexplained.
@@ -38,8 +38,10 @@ export function CancellationBanner({ appId }: { appId?: number | null }) {
       ? null
       : runState.phase === "cleaning-up"
         ? runState.isolation?.mode === "neon-branch"
-          ? t("cancellationRestoringTestApp")
-          : t("cancellationCleaningTestData")
+          ? t("cancellationRemovingTestDatabase")
+          : runState.sandboxed
+            ? t("cancellationCleaningTestSandbox")
+            : t("cancellationCleaningTestData")
         : runState.phase === "stopping"
           ? t("cancellationEndingTestRun")
           : null;