feat(flows): fix large asset downloads and add per-file progress

.changeset/asset-download-progress-counter.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": patch
+---
+
+Show a per-file counter while `flows pull` downloads team-storage assets. The progress line now reads "Downloading team-storage assets (2/12)" and advances as each file starts. The total counts only files that download; reused and skipped files are not included. In human mode the spinner label updates in place, and when a download fails the error line keeps the last counter so you can see where it stopped. Agent mode writes one progress line per file to stderr. The json output does not change.

.changeset/stall-timeout-signed-url-downloads.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": patch
+---
+
+Let large signed-URL downloads finish on slow links. Before this change, flow-bundle and team-storage asset downloads had a fixed 30-second deadline for the full download. A large asset, for example a video file, could not finish in time and `flows pull` failed with a timeout. The 30-second window is now a stall timeout. The timer resets each time data arrives, so a slow download that makes progress can run to completion. A download that receives no data for 30 seconds still fails, and the error message now says the download stalled.

.changeset/stream-signed-url-downloads.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": patch
+---
+
+Stream signed-URL downloads to disk instead of buffering them in memory. Before this change, the CLI held the whole file in memory and briefly needed about twice the file size, so a multi-gigabyte asset could exceed the memory limit of a small CI container. The download now writes each chunk to a `.part` file and renames it into place when the download completes, so peak memory stays near one chunk for any file size. A pull of a 1.4 GB asset now peaks at about 290 MB of memory instead of 3.3 GB. A failed download removes the partial file, and slow disk writes do not count toward the 30-second stall timeout.

src/core/messages/auth.ts

@@ -65,7 +65,7 @@ export const authMessages = {
       networkUnreachable:
         "Could not reach the flow bundle storage. Check your network connection and try again.",
       timedOut: (timeoutMs: number) =>
-        `Downloading the flow bundle timed out after ${formatSeconds(timeoutMs)}. Please try again.`,
+        `Downloading the flow bundle stalled — no data arrived for ${formatSeconds(timeoutMs)}. Please try again.`,
       malformed:
         "The flow bundle download was malformed. Please run `qawolf flows pull` again.",
     },

src/core/messages/flows.ts

@@ -33,6 +33,8 @@ export const flowsMessages = {
     aborted: "Aborted; no changes.",
     extractingBundle: "Extracting bundle",
     downloadingTeamStorageAssets: "Downloading team-storage assets",
+    downloadingTeamStorageAssetsProgress: (current: number, total: number) =>
+      `Downloading team-storage assets (${String(current)}/${String(total)})`,
     teamStorageRequiresTeamKey:
       "Team storage requires a team API key; organization keys are not supported here.",
     summary: (result: PullSummaryInput, assetsAbs: string) => {

src/domains/flows/pull/handler.test.ts

@@ -33,14 +33,16 @@ afterEach(async () => {
 // Real withProgress: runs every task in order, returns their results.
 // makeFakeUI's stub resolves to []; we need the inner tasks to execute so
 // the handler reaches its JSON-output branch with real result data.
-function makeJsonUi(): UI {
+function makeJsonUi(onUpdate?: (message: string) => void): UI {
   const ui = makeFakeUI();
   const withProgress = async (
-    steps: readonly { task: () => Promise<unknown> }[],
+    steps: readonly {
+      task: (update: (message: string) => void) => Promise<unknown>;
+    }[],
   ): Promise<unknown[]> => {
     const results: unknown[] = [];
     for (const step of steps) {
-      results.push(await step.task());
+      results.push(await step.task((message) => onUpdate?.(message)));
     }
     return results;
   };
@@ -57,6 +59,7 @@ function makeCtx(
   ui: UI,
   bundlePath: string,
   envVars: Record<string, string> = {},
+  syncTeamStorageAssets?: ReturnType<typeof mock>,
 ): AuthCommandContext {
   return {
     ui,
@@ -77,10 +80,12 @@ function makeCtx(
         ok: true,
         value: envVars,
       }),
-      syncTeamStorageAssets: mock().mockResolvedValue({
-        ok: true,
-        value: { downloadedCount: 0, reusedCount: 0, skippedCount: 0 },
-      }),
+      syncTeamStorageAssets:
+        syncTeamStorageAssets ??
+        mock().mockResolvedValue({
+          ok: true,
+          value: { downloadedCount: 0, reusedCount: 0, skippedCount: 0 },
+        }),
     }),
   };
 }
@@ -133,6 +138,37 @@ describe("handleFlowsPull json mode output", () => {
     expect(JSON.parse(JSON.stringify(payload))).toEqual(payload);
   });
 
+  it("relays per-file asset download progress to the progress step", async () => {
+    await buildBundle(bundleArchive, {
+      flows: [{ name: "a.flow.ts", data: "// a\n" }],
+    });
+    const updates: string[] = [];
+    const ui = makeJsonUi((message) => updates.push(message));
+    const syncMock = mock(
+      async (
+        _assetsAbs: string,
+        opts?: {
+          onProgress?: (p: { current: number; total: number }) => void;
+        },
+      ) => {
+        opts?.onProgress?.({ current: 1, total: 3 });
+        opts?.onProgress?.({ current: 2, total: 3 });
+        return {
+          ok: true,
+          value: { downloadedCount: 3, reusedCount: 0, skippedCount: 0 },
+        };
+      },
+    );
+    const ctx = makeCtx(ui, bundleArchive, {}, syncMock);
+
+    await handleFlowsPull(ctx, { env: "env-abc", out: destDir });
+
+    expect(updates).toEqual([
+      "Downloading team-storage assets (1/3)",
+      "Downloading team-storage assets (2/3)",
+    ]);
+  });
+
   it("does not call ui.output when mode is not json", async () => {
     await buildBundle(bundleArchive, {
       flows: [{ name: "a.flow.ts", data: "// a\n" }],

src/domains/flows/pull/handler.ts

@@ -92,9 +92,19 @@ export async function handleFlowsPull(
         },
         {
           message: flowsMessages.pull.downloadingTeamStorageAssets,
-          task: async () => {
-            const result =
-              await ctx.platformClient.syncTeamStorageAssets(assetsAbs);
+          task: async (update) => {
+            const result = await ctx.platformClient.syncTeamStorageAssets(
+              assetsAbs,
+              {
+                onProgress: ({ current, total }) =>
+                  update(
+                    flowsMessages.pull.downloadingTeamStorageAssetsProgress(
+                      current,
+                      total,
+                    ),
+                  ),
+              },
+            );
             if (!result.ok) throw new Error(result.error);
             return result.value;
           },

src/shell/fs.testUtils.dirTree.test.ts

@@ -0,0 +1,149 @@
+import { describe, expect, it } from "bun:test";
+import { isNoEntError } from "~/core/errors.js";
+import { makeMemoryFs } from "./fs.testUtils.js";
+
+describe("makeMemoryFs directory tree", () => {
+  // readdir
+  it("should return top-level entries when listing root", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/a");
+    await fs.writeFile("/b.txt", "");
+    const entries = await fs.readdir("/");
+    expect(entries.sort()).toEqual(["a", "b.txt"]);
+  });
+
+  it("should return direct child names when directory exists", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/a");
+    await fs.writeFile("/a/f.txt", "");
+    await fs.mkdir("/a/sub");
+    const entries = await fs.readdir("/a");
+    expect(entries.sort()).toEqual(["f.txt", "sub"]);
+  });
+
+  it("should throw ENOENT when directory does not exist", async () => {
+    const fs = makeMemoryFs();
+    let caughtError: unknown;
+    try {
+      await fs.readdir("/missing");
+    } catch (e) {
+      caughtError = e;
+    }
+    expect(isNoEntError(caughtError)).toBe(true);
+  });
+
+  it("should throw ENOTDIR when path is a file", async () => {
+    const fs = makeMemoryFs();
+    await fs.writeFile("/f.txt", "data");
+    let caughtError: unknown;
+    try {
+      await fs.readdir("/f.txt");
+    } catch (e) {
+      caughtError = e;
+    }
+    expect((caughtError as NodeJS.ErrnoException).code).toBe("ENOTDIR");
+  });
+
+  // readdirWithTypes
+  it("should return correct types when listing root", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/dir");
+    await fs.writeFile("/f.txt", "");
+    const entries = await fs.readdirWithTypes("/");
+    const d = entries.find((e) => e.name === "dir");
+    const f = entries.find((e) => e.name === "f.txt");
+    expect(d?.isDirectory()).toBe(true);
+    expect(d?.isFile()).toBe(false);
+    expect(f?.isFile()).toBe(true);
+    expect(f?.isDirectory()).toBe(false);
+  });
+
+  it("should return FsDirent with isFile true for files", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/d");
+    await fs.writeFile("/d/a.txt", "");
+    const entries = await fs.readdirWithTypes("/d");
+    const f = entries.find((e) => e.name === "a.txt");
+    expect(f?.isFile()).toBe(true);
+    expect(f?.isDirectory()).toBe(false);
+  });
+
+  it("should return FsDirent with isDirectory true for subdirs", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/p");
+    await fs.mkdir("/p/sub");
+    const entries = await fs.readdirWithTypes("/p");
+    const d = entries.find((e) => e.name === "sub");
+    expect(d?.isDirectory()).toBe(true);
+    expect(d?.isFile()).toBe(false);
+  });
+
+  it("should throw ENOTDIR from readdirWithTypes when path is a file", async () => {
+    const fs = makeMemoryFs();
+    await fs.writeFile("/f.txt", "data");
+    let caughtError: unknown;
+    try {
+      await fs.readdirWithTypes("/f.txt");
+    } catch (e) {
+      caughtError = e;
+    }
+    expect((caughtError as NodeJS.ErrnoException).code).toBe("ENOTDIR");
+  });
+
+  it("should snapshot isFile/isDirectory at readdir time, not reflect later mutations", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/d");
+    await fs.writeFile("/d/f.txt", "data");
+    const entries = await fs.readdirWithTypes("/d");
+    const f = entries.find((e) => e.name === "f.txt")!;
+    expect(f.isFile()).toBe(true);
+    await fs.unlink("/d/f.txt");
+    expect(f.isFile()).toBe(true);
+  });
+
+  // rename
+  it("should move a file to the new path", async () => {
+    const fs = makeMemoryFs();
+    await fs.writeFile("/src.txt", "hello");
+    await fs.rename("/src.txt", "/dst.txt");
+    expect(await fs.readFile("/dst.txt")).toBe("hello");
+    expect(await fs.pathExists("/src.txt")).toBe(false);
+  });
+
+  it("should move a directory and its children to the new path", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/src", { recursive: true });
+    await fs.writeFile("/src/a.txt", "hello");
+    await fs.mkdir("/src/sub");
+    await fs.writeFile("/src/sub/b.txt", "world");
+    await fs.mkdir("/dst-parent");
+    await fs.rename("/src", "/dst-parent/dst");
+    expect(await fs.readFile("/dst-parent/dst/a.txt")).toBe("hello");
+    expect(await fs.readFile("/dst-parent/dst/sub/b.txt")).toBe("world");
+    expect(await fs.pathExists("/src")).toBe(false);
+    expect(await fs.pathExists("/src/a.txt")).toBe(false);
+  });
+
+  it("should throw ENOENT when renaming a missing path", async () => {
+    const fs = makeMemoryFs();
+    let caughtError: unknown;
+    try {
+      await fs.rename("/nope.txt", "/dst.txt");
+    } catch (e) {
+      caughtError = e;
+    }
+    expect(isNoEntError(caughtError)).toBe(true);
+  });
+
+  it("should throw ENOENT when renaming a directory to a non-existent parent", async () => {
+    const fs = makeMemoryFs();
+    await fs.mkdir("/src");
+    let caughtError: unknown;
+    try {
+      await fs.rename("/src", "/missing/dst");
+    } catch (e) {
+      caughtError = e;
+    }
+    expect(isNoEntError(caughtError)).toBe(true);
+  });
+});