From 7478d21880bb1a687f3c4a19ea00a1d3f57a6b09 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 18 Sep 2026 07:55:34 +0000 Subject: [PATCH 1/2] fix(tui): stream large Git index and status output for project tree Project-tree Git reads used execFile without maxBuffer, so indexes over 1 MiB failed and the tree silently fell back to a 10k/depth-8 scan. Parse ls-files and status incrementally from a spawned process with explicit byte and entry bounds, and keep timeout or output-limit failures distinct from a non-Git workspace. Closes #2120 Co-authored-by: turk --- .../project-tree/ProjectTreeStore.ts | 96 ++- .../tui/workbench/project-tree/gitStatus.ts | 580 +++++++++++++++--- .../tui/workbench/surfaces/PreviewSurface.tsx | 9 +- .../project-tree-git-listing.test.ts | 373 +++++++++++ .../tests/tui/workbench/project-tree.test.ts | 15 +- 5 files changed, 959 insertions(+), 114 deletions(-) create mode 100644 runtime/tests/tui/workbench/project-tree-git-listing.test.ts diff --git a/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts b/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts index 21f8bad4f9..4db46506fc 100644 --- a/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts +++ b/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts @@ -16,7 +16,11 @@ import { collectGitBranch, collectGitStatus, listGitFiles, + listingWarning, + shouldScanWorkspaceFallback, type GitStatusByPath, + type GitFilesListing, + type ProjectTreeGitCommandOptions, } from "./gitStatus.js"; import { normalizeWorkspacePathForReferences } from "../pathReferences.js"; import type { @@ -141,10 +145,16 @@ export class ProjectTreeStore { // auto-revealed. Stays null until the first scan establishes the baseline — the // initial repo tree must NOT auto-expand (that would explode a large repo). #knownDirectories: ReadonlySet | null = null; + #gitOptions: ProjectTreeGitCommandOptions; - constructor(cwd = process.cwd(), refreshIntervalMs = 5_000) { + constructor( + cwd = process.cwd(), + refreshIntervalMs = 5_000, + gitOptions: ProjectTreeGitCommandOptions = {}, + ) { this.#cwd = cwd; this.#refreshIntervalMs = refreshIntervalMs; + this.#gitOptions = gitOptions; this.#mutationWorkspaceRootIdentity = captureInitialWorkspaceRootIdentity(cwd); } @@ -209,20 +219,33 @@ export class ProjectTreeStore { this.#loading = true; this.#emit(); try { - const [paths, gitStatus, gitBranch] = await Promise.all([ - listWorkspacePaths(this.#cwd), - collectGitStatus(this.#cwd), - collectGitBranch(this.#cwd), + const [pathListing, gitStatus, gitBranch] = await Promise.all([ + listWorkspacePaths(this.#cwd, this.#gitOptions), + collectGitStatus(this.#cwd, this.#gitOptions), + collectGitBranch(this.#cwd, this.#gitOptions), ]); if (version !== this.#refreshVersion) return; - this.#autoExpandNewDirectories(paths); - this.#paths = paths; - this.#gitStatus = gitStatus; - this.#gitBranch = gitBranch; + const warning = + listingWarning(pathListing) ?? + listingWarning(gitStatus) ?? + listingWarning(gitBranch); + if (pathListing.paths.length === 0 && warning) { + this.#loading = false; + this.#error = warning; + this.#emit(); + return; + } + this.#autoExpandNewDirectories(pathListing.paths); + this.#paths = pathListing.paths; + this.#gitStatus = gitStatus.status; + this.#gitBranch = gitBranch.branch; this.#cursorPath = - this.#cursorPath ?? firstFilePath(paths) ?? paths[0] ?? null; + this.#cursorPath ?? + firstFilePath(pathListing.paths) ?? + pathListing.paths[0] ?? + null; this.#loading = false; - this.#error = null; + this.#error = warning; this.#emit(); } catch (error) { if (version !== this.#refreshVersion) return; @@ -986,20 +1009,55 @@ async function readTopLevelPaths(cwd: string): Promise { .sort((a, b) => a.localeCompare(b)); } -async function listWorkspacePaths(cwd: string): Promise { - const gitPaths = await listGitFiles(cwd); - if (gitPaths && gitPaths.length > 0) return gitPaths; +export type WorkspacePathListing = { + readonly paths: readonly string[]; + readonly source: "git" | "scan" | "top-level"; + readonly kind: GitFilesListing["kind"]; + readonly warning: string | null; +}; + +/** + * Git listings that fail or hit an explicit bound keep their Git result. + * Only a non-Git workspace or an empty successful index may fall back to + * the bounded filesystem scan. + */ +export async function listWorkspacePaths( + cwd: string, + gitOptions: ProjectTreeGitCommandOptions = {}, +): Promise { + const gitListing = await listGitFiles(cwd, gitOptions); + if (!shouldScanWorkspaceFallback(gitListing)) { + return { + paths: gitListing.paths, + source: "git", + kind: gitListing.kind, + warning: listingWarning(gitListing), + }; + } const scannedPaths = await scanWorkspacePaths(cwd); - if (scannedPaths.length > 0) return scannedPaths; + if (scannedPaths.length > 0) { + return { + paths: scannedPaths, + source: "scan", + kind: gitListing.kind, + warning: null, + }; + } - return readTopLevelPaths(cwd); + return { + paths: await readTopLevelPaths(cwd), + source: "top-level", + kind: gitListing.kind, + warning: null, + }; } /** - * Hard bounds for the fallback workspace scan (non-git workspaces only — git - * repos list files via `git ls-files`, which is fast and already bounded by - * the repo). + * Hard bounds for the fallback workspace scan (non-git workspaces and empty + * Git indexes only). A failed or truncated Git listing must not fall through + * here: that would silently replace a multi-megabyte index with a 10,000-entry + * depth-8 walk. * * The scan MUST be bounded: a user launching agenc from a huge cwd (e.g. * `$HOME`, millions of entries under ~/Library and project node_modules) diff --git a/runtime/src/tui/workbench/project-tree/gitStatus.ts b/runtime/src/tui/workbench/project-tree/gitStatus.ts index 37d0ca5236..2e869f7999 100644 --- a/runtime/src/tui/workbench/project-tree/gitStatus.ts +++ b/runtime/src/tui/workbench/project-tree/gitStatus.ts @@ -1,9 +1,84 @@ -import { execFile } from "node:child_process"; +import { spawn } from "node:child_process"; +import { StringDecoder } from "node:string_decoder"; import type { ProjectTreeGitBranch, ProjectTreeGitState } from "../types.js"; export type GitStatusByPath = ReadonlyMap; +/** Timeout for each project-tree Git read. */ +export const GIT_LISTING_TIMEOUT_MS = 5_000; +/** + * Hard stdout ceiling for streamed Git listings. Larger than Node's 1 MiB + * `execFile` default so a typical large index is listed in full, but still + * bounds memory if a repository keeps growing. + */ +export const GIT_LISTING_MAX_BYTES = 16 * 1024 * 1024; +/** Entry ceiling applied while parsing NUL-delimited Git paths. */ +export const GIT_LISTING_MAX_ENTRIES = 250_000; +const GIT_LISTING_MAX_STDERR_BYTES = 64 * 1024; + +export type GitListingKind = + | "ok" + | "truncated" + | "not-git" + | "timeout" + | "output-limit" + | "error"; + +export type ProjectTreeGitCommandOptions = { + readonly git?: string; + readonly timeoutMs?: number; + readonly maxBytes?: number; + readonly maxEntries?: number; +}; + +export type GitFilesListing = { + readonly kind: GitListingKind; + readonly paths: readonly string[]; + readonly truncated: boolean; + readonly message: string | null; +}; + +export type GitStatusListing = { + readonly kind: GitListingKind; + readonly status: Map; + readonly truncated: boolean; + readonly message: string | null; +}; + +export type GitBranchListing = { + readonly kind: GitListingKind; + readonly branch: ProjectTreeGitBranch | null; + readonly truncated: boolean; + readonly message: string | null; +}; + +type ResolvedGitCommandOptions = { + readonly git: string; + readonly timeoutMs: number; + readonly maxBytes: number; + readonly maxEntries: number; +}; + +type StreamGitResult = { + readonly code: number | null; + readonly stderr: string; + readonly timedOut: boolean; + readonly byteLimitReached: boolean; + readonly entryLimitReached: boolean; + readonly spawnError: NodeJS.ErrnoException | null; +}; + +type BranchParseState = { + branch: string | null; + head: string | null; + upstream?: string; + ahead?: number; + behind?: number; + dirtyCount: number; + sawHeader: boolean; +}; + export function parseGitStatusPorcelain(raw: string): Map { const out = new Map(); for (const line of raw.split("\n")) { @@ -17,34 +92,84 @@ export function parseGitStatusPorcelain(raw: string): Map { +export function parseGitStatusPorcelainZ( + raw: string, +): Map { const out = new Map(); - const fields = raw.split("\0"); - for (let index = 0; index < fields.length;) { - const entry = fields[index++]!; - if (entry.length < 4) continue; - const code = entry.slice(0, 2); - const path = entry.slice(3); - if (path) out.set(path, statusForCode(code)); - if (isRenameOrCopyCode(code)) index += 1; - } + const parsed = consumeNulFields("", raw); + applyGitStatusFields(out, [...parsed.fields, parsed.pending]); return out; } -export function collectGitStatus(cwd: string): Promise> { - return new Promise((resolve) => { - execFile( - "git", - ["-c", "core.quotepath=false", "status", "--porcelain=v1", "-z", "--untracked-files=all"], - { cwd, encoding: "utf8", timeout: 5_000 }, - (error, stdout) => { - if (error) { - resolve(new Map()); - return; +/** + * Split NUL-delimited Git output across chunk boundaries. The trailing + * incomplete field stays in `pending` so a filename that contains newlines + * (legal in `ls-files -z` / `status -z`) is never split on `\n`. + */ +export function consumeNulFields( + pending: string, + chunk: string, +): { readonly fields: readonly string[]; readonly pending: string } { + const parts = (pending + chunk).split("\0"); + const nextPending = parts.pop() ?? ""; + return { fields: parts, pending: nextPending }; +} + +export function collectGitStatus( + cwd: string, + options: ProjectTreeGitCommandOptions = {}, +): Promise { + const resolved = resolveGitCommandOptions(options); + const status = new Map(); + let pending = ""; + let expectingRenameSource = false; + + return streamGit( + [ + "-c", + "core.quotepath=false", + "status", + "--porcelain=v1", + "-z", + "--untracked-files=all", + ], + cwd, + resolved, + (chunk) => { + const parsed = consumeNulFields(pending, chunk); + pending = parsed.pending; + for (const field of parsed.fields) { + if (expectingRenameSource) { + expectingRenameSource = false; + continue; } - resolve(parseGitStatusPorcelainZ(stdout)); - }, - ); + if (field.length < 4) continue; + const code = field.slice(0, 2); + const path = field.slice(3); + if (path) status.set(path, statusForCode(code)); + if (isRenameOrCopyCode(code)) expectingRenameSource = true; + if (status.size >= resolved.maxEntries) return "stop"; + } + return "continue"; + }, + ).then((result) => { + if ( + !result.byteLimitReached && + !result.entryLimitReached && + !expectingRenameSource && + pending.length >= 4 + ) { + const code = pending.slice(0, 2); + const path = pending.slice(3); + if (path) status.set(path, statusForCode(code)); + } + const kind = classifyGitResult(result); + return { + kind, + status, + truncated: isTruncatedKind(kind), + message: listingMessage(kind, result.stderr, resolved), + }; }); } @@ -56,92 +181,371 @@ export function collectGitStatus(cwd: string): Promise { - return new Promise((resolve) => { - execFile( - "git", - ["status", "--porcelain=v2", "--branch", "--untracked-files=all"], - { cwd, encoding: "utf8", timeout: 5_000 }, - (error, stdout) => { - resolve(error ? null : parseGitBranchPorcelainV2(stdout)); - }, - ); + options: ProjectTreeGitCommandOptions = {}, +): Promise { + const resolved = resolveGitCommandOptions(options); + const state = createBranchParseState(); + let pending = ""; + + return streamGit( + ["status", "--porcelain=v2", "--branch", "--untracked-files=all"], + cwd, + resolved, + (chunk) => { + const combined = pending + chunk; + const lines = combined.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) consumeGitBranchLine(state, line); + return "continue"; + }, + ).then((result) => { + if (pending.length > 0) consumeGitBranchLine(state, pending); + const kind = classifyGitResult(result); + return { + kind, + branch: kind === "not-git" ? null : finishBranchParse(state), + truncated: isTruncatedKind(kind), + message: listingMessage(kind, result.stderr, resolved), + }; }); } export function parseGitBranchPorcelainV2( raw: string, ): ProjectTreeGitBranch | null { - let branch: string | null = null; - let head: string | null = null; - let upstream: string | undefined; - let ahead: number | undefined; - let behind: number | undefined; - let dirtyCount = 0; - let sawHeader = false; + const state = createBranchParseState(); + for (const line of raw.split("\n")) consumeGitBranchLine(state, line); + return finishBranchParse(state); +} - for (const line of raw.split("\n")) { - if (line.length === 0) continue; - if (line.startsWith("# branch.")) { - sawHeader = true; - const [key, ...rest] = line.slice(2).split(" "); - const value = rest.join(" "); - // git spells a detached HEAD "(detached)" and an unborn branch - // "(initial)"; neither is a branch name a user could check out. - if (key === "branch.head") { - branch = value.startsWith("(") ? null : value; - } else if (key === "branch.oid") { - head = value.startsWith("(") ? null : value.slice(0, 7); - } else if (key === "branch.upstream") { - upstream = value; - } else if (key === "branch.ab") { - const match = /^\+(\d+) -(\d+)$/u.exec(value); - if (match !== null) { - ahead = Number(match[1]); - behind = Number(match[2]); - } +export function listGitFiles( + cwd: string, + options: ProjectTreeGitCommandOptions = {}, +): Promise { + const resolved = resolveGitCommandOptions(options); + const paths: string[] = []; + let pending = ""; + + return streamGit( + [ + "-c", + "core.quotepath=false", + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ], + cwd, + resolved, + (chunk) => { + const parsed = consumeNulFields(pending, chunk); + pending = parsed.pending; + for (const field of parsed.fields) { + if (!field) continue; + paths.push(field); + if (paths.length >= resolved.maxEntries) return "stop"; } - continue; + return "continue"; + }, + ).then((result) => { + if ( + pending && + !result.byteLimitReached && + !result.entryLimitReached + ) { + paths.push(pending); + } + const kind = classifyGitResult(result); + return { + kind, + paths: paths.sort((a, b) => a.localeCompare(b)), + truncated: isTruncatedKind(kind), + message: listingMessage(kind, result.stderr, resolved), + }; + }); +} + +/** + * Scan fallback is only for a non-Git cwd or an empty successful Git index. + * Timeout, output-limit, truncated, and other Git failures must stay visible + * instead of being replaced by the 10,000-entry filesystem walk. + */ +export function shouldScanWorkspaceFallback(listing: { + readonly kind: GitListingKind; + readonly paths: readonly string[]; +}): boolean { + switch (listing.kind) { + case "ok": + return listing.paths.length === 0; + case "not-git": + return true; + case "truncated": + case "timeout": + case "output-limit": + case "error": + return false; + default: { + const exhaustive: never = listing.kind; + return exhaustive; } - if (line.startsWith("#")) continue; - // Every remaining record is one changed path: 1/2 tracked, u unmerged, - // ? untracked, ! ignored (never emitted without --ignored). - if (/^[12u?]\s/u.test(line)) dirtyCount += 1; } +} + +export function listingWarning(listing: { + readonly kind: GitListingKind; + readonly message: string | null; +}): string | null { + switch (listing.kind) { + case "ok": + case "not-git": + return null; + case "truncated": + case "timeout": + case "output-limit": + case "error": + return listing.message; + default: { + const exhaustive: never = listing.kind; + return exhaustive; + } + } +} - if (!sawHeader) return null; +function resolveGitCommandOptions( + options: ProjectTreeGitCommandOptions, +): ResolvedGitCommandOptions { return { - branch, - head, - ...(upstream !== undefined ? { upstream } : {}), - ...(ahead !== undefined ? { ahead } : {}), - ...(behind !== undefined ? { behind } : {}), - dirtyCount, + git: options.git ?? "git", + timeoutMs: options.timeoutMs ?? GIT_LISTING_TIMEOUT_MS, + maxBytes: options.maxBytes ?? GIT_LISTING_MAX_BYTES, + maxEntries: options.maxEntries ?? GIT_LISTING_MAX_ENTRIES, }; } -export function listGitFiles(cwd: string): Promise { +function streamGit( + args: readonly string[], + cwd: string, + options: ResolvedGitCommandOptions, + onChunk: (text: string) => "continue" | "stop", +): Promise { return new Promise((resolve) => { - execFile( - "git", - ["-c", "core.quotepath=false", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], - { cwd, encoding: "utf8", timeout: 5_000 }, - (error, stdout) => { - if (error) { - resolve(null); - return; - } - resolve( - stdout.split("\0").filter(Boolean).sort((a, b) => a.localeCompare(b)), - ); - }, - ); + const child = spawn(options.git, [...args], { + cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const decoder = new StringDecoder("utf8"); + let stderr = ""; + let stderrBytes = 0; + let stdoutBytes = 0; + let timedOut = false; + let byteLimitReached = false; + let entryLimitReached = false; + let spawnError: NodeJS.ErrnoException | null = null; + let settled = false; + let stopping = false; + + const finish = (code: number | null): void => { + if (settled) return; + settled = true; + if (timeout) clearTimeout(timeout); + const tail = decoder.end(); + if (tail && !stopping) { + if (onChunk(tail) === "stop") entryLimitReached = true; + } + resolve({ + code, + stderr, + timedOut, + byteLimitReached, + entryLimitReached, + spawnError, + }); + }; + + const stopChild = (): void => { + stopping = true; + try { + child.kill("SIGTERM"); + } catch { + // Process may already have exited. + } + }; + + const timeout = + options.timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + stopChild(); + }, options.timeoutMs) + : undefined; + timeout?.unref?.(); + + child.stdout?.on("data", (chunk: Buffer) => { + if (stopping || settled) return; + const remaining = options.maxBytes - stdoutBytes; + if (remaining <= 0) { + byteLimitReached = true; + stopChild(); + return; + } + const slice = chunk.length > remaining ? chunk.subarray(0, remaining) : chunk; + stdoutBytes += slice.length; + if (slice.length < chunk.length) byteLimitReached = true; + const decision = onChunk(decoder.write(slice)); + if (decision === "stop") entryLimitReached = true; + if (byteLimitReached || entryLimitReached) stopChild(); + }); + + child.stderr?.on("data", (chunk: Buffer) => { + if (stderrBytes >= GIT_LISTING_MAX_STDERR_BYTES) return; + const remaining = GIT_LISTING_MAX_STDERR_BYTES - stderrBytes; + const slice = chunk.length > remaining ? chunk.subarray(0, remaining) : chunk; + stderrBytes += slice.length; + stderr += slice.toString("utf8"); + }); + + child.once("error", (error: NodeJS.ErrnoException) => { + spawnError = error; + if (timeout) clearTimeout(timeout); + finish(127); + }); + + child.once("close", (code) => { + finish(code); + }); }); } +function classifyGitResult(result: StreamGitResult): GitListingKind { + if (result.timedOut) return "timeout"; + if (result.byteLimitReached) return "output-limit"; + if (result.entryLimitReached) return "truncated"; + if (result.spawnError) { + return result.spawnError.code === "ENOENT" ? "not-git" : "error"; + } + if (result.code === 0) return "ok"; + if (isNotGitRepository(result.stderr)) return "not-git"; + return "error"; +} + +function isNotGitRepository(stderr: string): boolean { + return /not a git repository/i.test(stderr); +} + +function isTruncatedKind(kind: GitListingKind): boolean { + switch (kind) { + case "truncated": + case "output-limit": + return true; + case "ok": + case "not-git": + case "timeout": + case "error": + return false; + default: { + const exhaustive: never = kind; + return exhaustive; + } + } +} + +function listingMessage( + kind: GitListingKind, + stderr: string, + options: ResolvedGitCommandOptions, +): string | null { + switch (kind) { + case "ok": + return null; + case "not-git": + return "not a git repository"; + case "timeout": + return `Git listing timed out after ${options.timeoutMs}ms`; + case "output-limit": + return `Git listing exceeded the ${options.maxBytes} byte output bound`; + case "truncated": + return `Git listing truncated at ${options.maxEntries} files`; + case "error": + return stderr.trim() || "Git listing failed"; + default: { + const exhaustive: never = kind; + return exhaustive; + } + } +} + +function applyGitStatusFields( + out: Map, + fields: readonly string[], +): void { + for (let index = 0; index < fields.length; ) { + const entry = fields[index++]!; + if (entry.length < 4) continue; + const code = entry.slice(0, 2); + const path = entry.slice(3); + if (path) out.set(path, statusForCode(code)); + if (isRenameOrCopyCode(code)) index += 1; + } +} + +function createBranchParseState(): BranchParseState { + return { + branch: null, + head: null, + dirtyCount: 0, + sawHeader: false, + }; +} + +function consumeGitBranchLine(state: BranchParseState, line: string): void { + if (line.length === 0) return; + if (line.startsWith("# branch.")) { + state.sawHeader = true; + const [key, ...rest] = line.slice(2).split(" "); + const value = rest.join(" "); + // git spells a detached HEAD "(detached)" and an unborn branch + // "(initial)"; neither is a branch name a user could check out. + if (key === "branch.head") { + state.branch = value.startsWith("(") ? null : value; + } else if (key === "branch.oid") { + state.head = value.startsWith("(") ? null : value.slice(0, 7); + } else if (key === "branch.upstream") { + state.upstream = value; + } else if (key === "branch.ab") { + const match = /^\+(\d+) -(\d+)$/u.exec(value); + if (match !== null) { + state.ahead = Number(match[1]); + state.behind = Number(match[2]); + } + } + return; + } + if (line.startsWith("#")) return; + // Every remaining record is one changed path: 1/2 tracked, u unmerged, + // ? untracked, ! ignored (never emitted without --ignored). + if (/^[12u?]\s/u.test(line)) state.dirtyCount += 1; +} + +function finishBranchParse(state: BranchParseState): ProjectTreeGitBranch | null { + if (!state.sawHeader) return null; + return { + branch: state.branch, + head: state.head, + ...(state.upstream !== undefined ? { upstream: state.upstream } : {}), + ...(state.ahead !== undefined ? { ahead: state.ahead } : {}), + ...(state.behind !== undefined ? { behind: state.behind } : {}), + dirtyCount: state.dirtyCount, + }; +} + function normalizePorcelainPath(pathPart: string): string { const rename = pathPart.match(/^(.+)\s+->\s+(.+)$/u); const value = rename?.[2] ?? pathPart; diff --git a/runtime/src/tui/workbench/surfaces/PreviewSurface.tsx b/runtime/src/tui/workbench/surfaces/PreviewSurface.tsx index fbb62690c3..f04bc3e629 100644 --- a/runtime/src/tui/workbench/surfaces/PreviewSurface.tsx +++ b/runtime/src/tui/workbench/surfaces/PreviewSurface.tsx @@ -195,8 +195,13 @@ export function PreviewSurface({ setGitStateState({ path: statusPath, status: null }); let mounted = true; collectGitStatus(getCwd()) - .then((status) => { - if (mounted) setGitStateState({ path: statusPath, status: status.get(statusPath) ?? "clean" }); + .then((listing) => { + if (mounted) { + setGitStateState({ + path: statusPath, + status: listing.status.get(statusPath) ?? "clean", + }); + } }) .catch((error) => { if (!mounted) return; diff --git a/runtime/tests/tui/workbench/project-tree-git-listing.test.ts b/runtime/tests/tui/workbench/project-tree-git-listing.test.ts new file mode 100644 index 0000000000..dc534f1ab8 --- /dev/null +++ b/runtime/tests/tui/workbench/project-tree-git-listing.test.ts @@ -0,0 +1,373 @@ +import { describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + collectGitBranch, + collectGitStatus, + consumeNulFields, + GIT_LISTING_MAX_BYTES, + listGitFiles, + listingWarning, + parseGitStatusPorcelainZ, + shouldScanWorkspaceFallback, +} from "../../../src/tui/workbench/project-tree/gitStatus.js"; +import { + listWorkspacePaths, + ProjectTreeStore, +} from "../../../src/tui/workbench/project-tree/ProjectTreeStore.js"; + +const LARGE_PATH_COUNT = 12_000; +const LARGE_PATH_PREFIX = "deep/nested/directory/file-"; + +describe("project-tree Git listing bounds (#2120)", () => { + it("reassembles NUL-delimited fields across chunks, including newline names", () => { + const first = consumeNulFields("", "src/has\nnew"); + expect(first.fields).toEqual([]); + expect(first.pending).toBe("src/has\nnew"); + + const second = consumeNulFields(first.pending, "line.ts\0README.md\0"); + expect(second.fields).toEqual(["src/has\nnewline.ts", "README.md"]); + expect(second.pending).toBe(""); + }); + + it("parses porcelain -z status for newline names and renames", () => { + const parsed = parseGitStatusPorcelainZ( + [ + "?? src/has\nnewline.ts", + "R src/new name.ts", + "src/old name.ts", + " M src/changed.ts", + ].join("\0") + "\0", + ); + + expect(parsed.get("src/has\nnewline.ts")).toBe("untracked"); + expect(parsed.get("src/new name.ts")).toBe("renamed"); + expect(parsed.has("src/old name.ts")).toBe(false); + expect(parsed.get("src/changed.ts")).toBe("modified"); + }); + + it("lists more than 1 MiB of NUL-delimited ls-files and status output", async () => { + const dir = await mkdtemp(join(tmpdir(), "agenc-tree-git-large-")); + try { + const git = await writeFakeGit( + dir, + ` +const args = process.argv.slice(2); +const count = ${LARGE_PATH_COUNT}; +const paths = []; +for (let i = 0; i < count; i++) { + paths.push("${LARGE_PATH_PREFIX}" + String(i).padStart(5, "0") + ".txt"); +} +const payload = paths.join("\\0") + "\\0"; +if (args.includes("ls-files")) { + process.stdout.write(payload); + process.exit(0); +} +if (args.includes("--porcelain=v1")) { + process.stdout.write(paths.map((path) => "?? " + path).join("\\0") + "\\0"); + process.exit(0); +} +if (args.includes("--porcelain=v2")) { + process.stdout.write( + [ + "# branch.oid abcdef0123456789", + "# branch.head main", + ...paths.map((path) => "? " + path), + ].join("\\n") + "\\n", + ); + process.exit(0); +} +process.exit(1); +`, + ); + + const files = await listGitFiles(dir, { git }); + const status = await collectGitStatus(dir, { git }); + const branch = await collectGitBranch(dir, { git }); + const listedBytes = Buffer.byteLength(`${files.paths.join("\0")}\0`); + + expect(listedBytes).toBeGreaterThan(1024 * 1024); + expect(listedBytes).toBeLessThan(GIT_LISTING_MAX_BYTES); + expect(files).toMatchObject({ + kind: "ok", + truncated: false, + message: null, + }); + expect(files.paths).toHaveLength(LARGE_PATH_COUNT); + expect(files.paths[0]).toBe(`${LARGE_PATH_PREFIX}00000.txt`); + expect(files.paths.at(-1)).toBe(`${LARGE_PATH_PREFIX}11999.txt`); + + expect(status.kind).toBe("ok"); + expect(status.status.size).toBe(LARGE_PATH_COUNT); + expect(status.status.get(`${LARGE_PATH_PREFIX}00000.txt`)).toBe( + "untracked", + ); + + expect(branch.kind).toBe("ok"); + expect(branch.branch).toMatchObject({ + branch: "main", + head: "abcdef0", + dirtyCount: LARGE_PATH_COUNT, + }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("distinguishes output-limit and timeout from a non-Git workspace", async () => { + const dir = await mkdtemp(join(tmpdir(), "agenc-tree-git-kinds-")); + try { + const limitedGit = await writeFakeGit( + dir, + ` +process.stdout.write("aaaaaaaa.tsx\\0bbbbbbbb.tsx\\0cccccccc.tsx\\0"); +process.exit(0); +`, + ); + const timeoutGit = await writeFakeGit( + dir, + ` +setTimeout(() => {}, 10_000); +`, + ); + const notGit = await listGitFiles(dir); + const limited = await listGitFiles(dir, { + git: limitedGit, + maxBytes: 16, + }); + const timedOut = await listGitFiles(dir, { + git: timeoutGit, + timeoutMs: 40, + }); + + expect(notGit.kind).toBe("not-git"); + expect(notGit.message).toBe("not a git repository"); + expect(limited.kind).toBe("output-limit"); + expect(limited.message).toMatch(/16 byte output bound/u); + expect(limited.kind).not.toBe(notGit.kind); + expect(timedOut.kind).toBe("timeout"); + expect(timedOut.message).toMatch(/timed out after 40ms/u); + expect(timedOut.kind).not.toBe(notGit.kind); + expect(listingWarning(notGit)).toBeNull(); + expect(listingWarning(limited)).toBe(limited.message); + expect(listingWarning(timedOut)).toBe(timedOut.message); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("lists a real Git filename that contains a newline", async () => { + const repo = await mkdtemp(join(tmpdir(), "agenc-tree-git-newline-")); + const fileName = "has\nnewline.ts"; + try { + execFileSync("git", ["init"], { cwd: repo, stdio: "ignore" }); + await writeFile(join(repo, fileName), "newline\n", "utf8"); + + const files = await listGitFiles(repo); + const status = await collectGitStatus(repo); + + expect(files.kind).toBe("ok"); + expect(files.paths).toContain(fileName); + expect(status.status.get(fileName)).toBe("untracked"); + } finally { + await rm(repo, { recursive: true, force: true }); + } + }); + + it("lists a real Git index larger than 1 MiB", async () => { + const repo = await mkdtemp(join(tmpdir(), "agenc-tree-git-index-")); + try { + execFileSync("git", ["init"], { cwd: repo, stdio: "ignore" }); + const blob = execFileSync("git", ["hash-object", "-w", "--stdin"], { + cwd: repo, + input: "", + }) + .toString("utf8") + .trim(); + const pathCount = 16_000; + const lines: string[] = []; + for (let index = 0; index < pathCount; index += 1) { + lines.push( + `100644 ${blob} 0\t${paddedIndexPath(index)}`, + ); + } + execFileSync("git", ["update-index", "--index-info"], { + cwd: repo, + input: `${lines.join("\n")}\n`, + }); + + const files = await listGitFiles(repo); + const listedBytes = Buffer.byteLength(`${files.paths.join("\0")}\0`); + + expect(listedBytes).toBeGreaterThan(1024 * 1024); + expect(files.kind).toBe("ok"); + expect(files.paths).toHaveLength(pathCount); + expect(files.paths[0]).toBe(paddedIndexPath(0)); + expect(files.paths.at(-1)).toBe(paddedIndexPath(pathCount - 1)); + } finally { + await rm(repo, { recursive: true, force: true }); + } + }); + + it("does not replace a failed Git listing with the truncated filesystem scan", async () => { + const repo = await mkdtemp(join(tmpdir(), "agenc-tree-git-noscan-")); + try { + execFileSync("git", ["init"], { cwd: repo, stdio: "ignore" }); + await mkdir(join(repo, "visible"), { recursive: true }); + await writeFile(join(repo, "visible", "keep.ts"), "keep\n", "utf8"); + await writeFile(join(repo, "scan-only.ts"), "scan\n", "utf8"); + + const timeoutGit = await writeFakeGit( + repo, + ` +setTimeout(() => {}, 10_000); +`, + ); + const limitedGit = await writeFakeGit( + repo, + ` +process.stdout.write("visible/keep.ts\\0extra-from-git.ts\\0"); +process.exit(0); +`, + ); + + const timedOut = await listWorkspacePaths(repo, { + git: timeoutGit, + timeoutMs: 40, + }); + const limited = await listWorkspacePaths(repo, { + git: limitedGit, + maxBytes: Buffer.byteLength("visible/keep.ts\0"), + }); + const healthy = await listWorkspacePaths(repo); + + expect(shouldScanWorkspaceFallback(timedOut)).toBe(false); + expect(timedOut.source).toBe("git"); + expect(timedOut.kind).toBe("timeout"); + expect(timedOut.paths).not.toContain("scan-only.ts"); + expect(timedOut.warning).toMatch(/timed out/u); + + expect(limited.source).toBe("git"); + expect(limited.kind).toBe("output-limit"); + expect(limited.paths).not.toContain("scan-only.ts"); + expect(limited.warning).toMatch(/byte output bound/u); + + expect(healthy.source).toBe("git"); + expect(healthy.kind).toBe("ok"); + expect(healthy.paths).toEqual( + expect.arrayContaining(["visible/keep.ts", "scan-only.ts"]), + ); + expect(healthy.warning).toBeNull(); + } finally { + await rm(repo, { recursive: true, force: true }); + } + }); + + it("still scans a non-Git workspace and an empty Git index", async () => { + const workspace = await mkdtemp(join(tmpdir(), "agenc-tree-git-scan-")); + const emptyGit = await mkdtemp(join(tmpdir(), "agenc-tree-empty-git-")); + try { + await mkdir(join(workspace, "src"), { recursive: true }); + await writeFile(join(workspace, "src", "app.ts"), "app\n", "utf8"); + execFileSync("git", ["init"], { cwd: emptyGit, stdio: "ignore" }); + await mkdir(join(emptyGit, "src", "empty"), { recursive: true }); + + const scanned = await listWorkspacePaths(workspace); + const emptyIndex = await listWorkspacePaths(emptyGit); + + expect(scanned.source).toBe("scan"); + expect(scanned.kind).toBe("not-git"); + expect(scanned.paths).toContain("src/app.ts"); + expect(emptyIndex.source).toBe("scan"); + expect(emptyIndex.kind).toBe("ok"); + expect(emptyIndex.paths).toContain("src/empty"); + expect(emptyIndex.warning).toBeNull(); + } finally { + await rm(workspace, { recursive: true, force: true }); + await rm(emptyGit, { recursive: true, force: true }); + } + }); + + it("surfaces a Git timeout on the project tree instead of a silent scan", async () => { + const repo = await mkdtemp(join(tmpdir(), "agenc-tree-store-timeout-")); + const timeoutGit = await writeFakeGit( + repo, + ` +setTimeout(() => {}, 10_000); +`, + ); + const store = new ProjectTreeStore(repo, 0, { + git: timeoutGit, + timeoutMs: 40, + }); + try { + execFileSync("git", ["init"], { cwd: repo, stdio: "ignore" }); + await writeFile(join(repo, "hidden-by-scan.ts"), "hidden\n", "utf8"); + + await store.refresh(); + const snapshot = store.getSnapshot(); + const paths = snapshot.rows.map((row) => row.path); + + expect(snapshot.error).toMatch(/timed out after 40ms/u); + expect(paths).not.toContain("hidden-by-scan.ts"); + } finally { + store.dispose(); + await rm(repo, { recursive: true, force: true }); + } + }); + + it("keeps a bounded Git listing visible on the project tree", async () => { + const repo = await mkdtemp(join(tmpdir(), "agenc-tree-store-bound-")); + const git = await writeFakeGit( + repo, + ` +if (process.argv.includes("ls-files")) { + process.stdout.write("kept.ts\\0dropped.ts\\0"); + process.exit(0); +} +if (process.argv.includes("--porcelain=v2")) { + process.stdout.write("# branch.oid abcdef0\\n# branch.head main\\n"); + process.exit(0); +} +process.exit(0); +`, + ); + const store = new ProjectTreeStore(repo, 0, { + git, + maxEntries: 1, + }); + try { + await writeFile(join(repo, "scan-only.ts"), "scan\n", "utf8"); + + await store.refresh(); + const snapshot = store.getSnapshot(); + const paths = snapshot.rows.map((row) => row.path); + + expect(snapshot.error).toMatch(/truncated at 1 files/u); + expect(paths).toContain("kept.ts"); + expect(paths).not.toContain("dropped.ts"); + expect(paths).not.toContain("scan-only.ts"); + } finally { + store.dispose(); + await rm(repo, { recursive: true, force: true }); + } + }); +}); + +function paddedIndexPath(index: number): string { + const stem = `tracked/dir/file-${String(index).padStart(5, "0")}-`; + return `${stem}${"x".repeat(64)}.txt`; +} + +async function writeFakeGit(dir: string, source: string): Promise { + const git = join(dir, `fake-git-${Math.random().toString(16).slice(2)}`); + await writeFile(git, `#!/usr/bin/env node\n${source}\n`, { + encoding: "utf8", + mode: 0o755, + }); + await chmod(git, 0o755); + return git; +} diff --git a/runtime/tests/tui/workbench/project-tree.test.ts b/runtime/tests/tui/workbench/project-tree.test.ts index 4fd9551305..e961fcf34e 100644 --- a/runtime/tests/tui/workbench/project-tree.test.ts +++ b/runtime/tests/tui/workbench/project-tree.test.ts @@ -228,8 +228,9 @@ describe("project tree helpers", () => { const status = await collectGitStatus(repo); - expect(status.get(fileName)).toBe("untracked"); - expect([...status.keys()]).not.toContain('"\\303\\251.ts"'); + expect(status.kind).toBe("ok"); + expect(status.status.get(fileName)).toBe("untracked"); + expect([...status.status.keys()]).not.toContain('"\\303\\251.ts"'); } finally { await rm(repo, { recursive: true, force: true }); } @@ -245,7 +246,10 @@ describe("project tree helpers", () => { await writeFile(join(repo, leading), "leading\n", "utf8"); await writeFile(join(repo, trailing), "trailing\n", "utf8"); - await expect(listGitFiles(repo)).resolves.toEqual([leading, trailing]); + await expect(listGitFiles(repo)).resolves.toMatchObject({ + kind: "ok", + paths: [leading, trailing], + }); } finally { await rm(repo, { recursive: true, force: true }); } @@ -279,8 +283,9 @@ describe("project tree helpers", () => { const status = await collectGitStatus(repo); - expect(status.get(newPath)).toBe("renamed"); - expect(status.has("d.ts")).toBe(false); + expect(status.kind).toBe("ok"); + expect(status.status.get(newPath)).toBe("renamed"); + expect(status.status.has("d.ts")).toBe(false); } finally { await rm(repo, { recursive: true, force: true }); } From f626adac32569024ce96b6dc0652de2893b112ac Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 18 Sep 2026 07:58:05 +0000 Subject: [PATCH 2/2] test(tui): drain large Git fixtures and keep bounded-tree warnings Use a flushed payload for >1 MiB fake Git output, and read the workspace listing warning field so truncated listings stay visible on the tree. Co-authored-by: turk --- .../project-tree/ProjectTreeStore.ts | 2 +- .../project-tree-git-listing.test.ts | 106 +++++++++++++----- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts b/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts index 4db46506fc..7d729c9a7c 100644 --- a/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts +++ b/runtime/src/tui/workbench/project-tree/ProjectTreeStore.ts @@ -226,7 +226,7 @@ export class ProjectTreeStore { ]); if (version !== this.#refreshVersion) return; const warning = - listingWarning(pathListing) ?? + pathListing.warning ?? listingWarning(gitStatus) ?? listingWarning(gitBranch); if (pathListing.paths.length === 0 && warning) { diff --git a/runtime/tests/tui/workbench/project-tree-git-listing.test.ts b/runtime/tests/tui/workbench/project-tree-git-listing.test.ts index dc534f1ab8..c602604623 100644 --- a/runtime/tests/tui/workbench/project-tree-git-listing.test.ts +++ b/runtime/tests/tui/workbench/project-tree-git-listing.test.ts @@ -49,38 +49,66 @@ describe("project-tree Git listing bounds (#2120)", () => { expect(parsed.get("src/changed.ts")).toBe("modified"); }); + it("reassembles more than 1 MiB of chunked NUL-delimited output", () => { + const paths = largeListingPaths(); + const newlinePath = "src/has\nnewline.ts"; + const raw = `${[newlinePath, ...paths].join("\0")}\0`; + expect(Buffer.byteLength(raw)).toBeGreaterThan(1024 * 1024); + + const assembled: string[] = []; + let pending = ""; + for (const chunk of splitString(raw, 65_536)) { + const parsed = consumeNulFields(pending, chunk); + pending = parsed.pending; + assembled.push(...parsed.fields); + } + if (pending) assembled.push(pending); + + expect(assembled).toHaveLength(paths.length + 1); + expect(assembled[0]).toBe(newlinePath); + expect(assembled.at(-1)).toBe(paths.at(-1)); + }); + it("lists more than 1 MiB of NUL-delimited ls-files and status output", async () => { const dir = await mkdtemp(join(tmpdir(), "agenc-tree-git-large-")); try { + const paths = largeListingPaths(); + const lsFilesPayload = Buffer.from(`${paths.join("\0")}\0`); + const statusPayload = Buffer.from( + `${paths.map((path) => `?? ${path}`).join("\0")}\0`, + ); + const branchPayload = Buffer.from( + [ + "# branch.oid abcdef0123456789", + "# branch.head main", + ...paths.map((path) => `? ${path}`), + "", + ].join("\n"), + ); + expect(lsFilesPayload.length).toBeGreaterThan(1024 * 1024); + expect(statusPayload.length).toBeGreaterThan(1024 * 1024); + + await writeFile(join(dir, "ls-files.bin"), lsFilesPayload); + await writeFile(join(dir, "status.bin"), statusPayload); + await writeFile(join(dir, "branch.bin"), branchPayload); const git = await writeFakeGit( dir, ` +const fs = require("fs"); +const path = require("path"); +const root = ${JSON.stringify(dir)}; const args = process.argv.slice(2); -const count = ${LARGE_PATH_COUNT}; -const paths = []; -for (let i = 0; i < count; i++) { - paths.push("${LARGE_PATH_PREFIX}" + String(i).padStart(5, "0") + ".txt"); -} -const payload = paths.join("\\0") + "\\0"; -if (args.includes("ls-files")) { - process.stdout.write(payload); +const file = args.includes("ls-files") + ? "ls-files.bin" + : args.includes("--porcelain=v1") + ? "status.bin" + : args.includes("--porcelain=v2") + ? "branch.bin" + : null; +if (!file) process.exit(1); +process.stdout.write(fs.readFileSync(path.join(root, file)), () => { process.exit(0); -} -if (args.includes("--porcelain=v1")) { - process.stdout.write(paths.map((path) => "?? " + path).join("\\0") + "\\0"); - process.exit(0); -} -if (args.includes("--porcelain=v2")) { - process.stdout.write( - [ - "# branch.oid abcdef0123456789", - "# branch.head main", - ...paths.map((path) => "? " + path), - ].join("\\n") + "\\n", - ); - process.exit(0); -} -process.exit(1); +}); `, ); @@ -89,6 +117,7 @@ process.exit(1); const branch = await collectGitBranch(dir, { git }); const listedBytes = Buffer.byteLength(`${files.paths.join("\0")}\0`); + expect(files.kind).toBe("ok"); expect(listedBytes).toBeGreaterThan(1024 * 1024); expect(listedBytes).toBeLessThan(GIT_LISTING_MAX_BYTES); expect(files).toMatchObject({ @@ -97,14 +126,12 @@ process.exit(1); message: null, }); expect(files.paths).toHaveLength(LARGE_PATH_COUNT); - expect(files.paths[0]).toBe(`${LARGE_PATH_PREFIX}00000.txt`); - expect(files.paths.at(-1)).toBe(`${LARGE_PATH_PREFIX}11999.txt`); + expect(files.paths[0]).toBe(paths[0]); + expect(files.paths.at(-1)).toBe(paths.at(-1)); expect(status.kind).toBe("ok"); expect(status.status.size).toBe(LARGE_PATH_COUNT); - expect(status.status.get(`${LARGE_PATH_PREFIX}00000.txt`)).toBe( - "untracked", - ); + expect(status.status.get(paths[0]!)).toBe("untracked"); expect(branch.kind).toBe("ok"); expect(branch.branch).toMatchObject({ @@ -283,7 +310,7 @@ process.exit(0); expect(scanned.paths).toContain("src/app.ts"); expect(emptyIndex.source).toBe("scan"); expect(emptyIndex.kind).toBe("ok"); - expect(emptyIndex.paths).toContain("src/empty"); + expect(emptyIndex.paths).toContain("src/empty/"); expect(emptyIndex.warning).toBeNull(); } finally { await rm(workspace, { recursive: true, force: true }); @@ -357,6 +384,25 @@ process.exit(0); }); }); +function largeListingPaths(): string[] { + const pad = "x".repeat(80); + const paths: string[] = []; + for (let index = 0; index < LARGE_PATH_COUNT; index += 1) { + paths.push( + `${LARGE_PATH_PREFIX}${String(index).padStart(5, "0")}-${pad}.txt`, + ); + } + return paths; +} + +function splitString(value: string, chunkSize: number): string[] { + const chunks: string[] = []; + for (let offset = 0; offset < value.length; offset += chunkSize) { + chunks.push(value.slice(offset, offset + chunkSize)); + } + return chunks; +} + function paddedIndexPath(index: number): string { const stem = `tracked/dir/file-${String(index).padStart(5, "0")}-`; return `${stem}${"x".repeat(64)}.txt`;