perf: reduce transcript and repeated-secret scan allocations

cmd/entire/cli/checkpoint/persistent.go

@@ -2750,7 +2750,7 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t
 			chunkFiles = append([]string{paths.TranscriptFileName}, chunkFiles...)
 		}
 
-		var chunks [][]byte
+		chunks := make([][]byte, 0, len(chunkFiles))
 		for _, chunkFile := range chunkFiles {
 			file, err := tree.File(chunkFile)
 			if err != nil {
@@ -2760,15 +2760,15 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t
 				)
 				continue
 			}
-			content, err := file.Contents()
+			content, err := readTranscriptFile(file)
 			if err != nil {
 				logging.Warn(ctx, "failed to read transcript chunk contents",
 					slog.String("chunk_file", chunkFile),
 					slog.String("error", err.Error()),
 				)
 				continue
 			}
-			chunks = append(chunks, []byte(content))
+			chunks = append(chunks, content)
 		}
 
 		if len(chunks) > 0 {
@@ -2782,21 +2782,48 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t
 
 	// Fall back to reading base file (non-chunked or backwards compatibility)
 	if file, err := tree.File(paths.TranscriptFileName); err == nil {
-		if content, err := file.Contents(); err == nil {
-			return []byte(content), nil
+		if content, err := readTranscriptFile(file); err == nil {
+			return content, nil
 		}
 	}
 
 	// Try legacy filename
 	if file, err := tree.File(paths.TranscriptFileNameLegacy); err == nil {
-		if content, err := file.Contents(); err == nil {
-			return []byte(content), nil
+		if content, err := readTranscriptFile(file); err == nil {
+			return content, nil
 		}
 	}
 
 	return nil, nil
 }
 
+// readTranscriptFile keeps large transcript blobs as bytes throughout the read.
+// File.Contents grows a buffer and copies it to a string, which callers then
+// copy back to bytes. Reserve the known blob size plus ReadFrom's minimum
+// spare capacity so its final EOF probe does not double the whole buffer.
+func readTranscriptFile(file *object.File) (content []byte, err error) {
+	reader, err := file.Reader()
+	if err != nil {
+		return nil, fmt.Errorf("open transcript blob: %w", err)
+	}
+	defer func() {
+		if closeErr := reader.Close(); err == nil && closeErr != nil {
+			err = fmt.Errorf("close transcript blob: %w", closeErr)
+		}
+	}()
+
+	var buf bytes.Buffer
+	// Shadow transcripts can exceed MaxChunkSize without being chunked. Bound
+	// only the upfront allocation hint at 1 GiB; larger blobs still grow incrementally.
+	if file.Size >= 0 && file.Size <= 1<<30 {
+		buf.Grow(int(file.Size) + bytes.MinRead)
+	}
+	if _, err := buf.ReadFrom(reader); err != nil {
+		return nil, fmt.Errorf("read transcript blob: %w", err)
+	}
+	return buf.Bytes(), nil
+}
+
 func transcriptBlobHashesFromTreeEntries(entries []object.TreeEntry) []plumbing.Hash {
 	hashesByName := make(map[string]plumbing.Hash)
 	var chunkFiles []string

cmd/entire/cli/checkpoint/transcript_read_test.go

@@ -0,0 +1,90 @@
+package checkpoint
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/entireio/cli/cmd/entire/cli/paths"
+
+	"github.com/go-git/go-git/v6"
+	"github.com/go-git/go-git/v6/plumbing/filemode"
+	"github.com/go-git/go-git/v6/plumbing/object"
+	"github.com/go-git/go-git/v6/storage/memory"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadTranscriptFromTreeFormats(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name  string
+		files map[string][]byte
+		want  []byte
+	}{
+		{name: "base", files: map[string][]byte{paths.TranscriptFileName: []byte("first\nsecond\n")}, want: []byte("first\nsecond\n")},
+		{name: "legacy", files: map[string][]byte{paths.TranscriptFileNameLegacy: []byte("legacy\n")}, want: []byte("legacy\n")},
+		// A present empty transcript must remain non-nil: ephemeral reads use
+		// nil to decide whether to fall through to their legacy read path.
+		{name: "empty", files: map[string][]byte{paths.TranscriptFileName: {}}, want: []byte{}},
+		{name: "absent", files: map[string][]byte{}, want: nil},
+		{
+			name: "chunk order",
+			files: map[string][]byte{
+				paths.TranscriptFileName:          []byte("zero"),
+				paths.TranscriptFileName + ".010": []byte("ten"),
+				paths.TranscriptFileName + ".002": []byte("two"),
+				paths.TranscriptFileName + ".001": []byte("one"),
+			},
+			want: []byte("zero\none\ntwo\nten"),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			repo, err := git.Init(memory.NewStorage(), nil)
+			require.NoError(t, err)
+			tree := transcriptReadFixture(t, repo, tc.files)
+			got, err := readTranscriptFromTree(t.Context(), NewFetchingTree(t.Context(), tree, repo.Storer, nil), "")
+			require.NoError(t, err)
+			require.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func BenchmarkReadTranscriptFromTree(b *testing.B) {
+	// The 64 MiB case covers unchunked shadow transcripts above MaxChunkSize.
+	for _, size := range []int{1 << 20, 8 << 20, 64 << 20} {
+		b.Run(fmt.Sprintf("%dMiB", size>>20), func(b *testing.B) {
+			// In-memory objects isolate transcript decoding and allocation costs
+			// from filesystem cache state; tree and blob lookup are still real.
+			repo, err := git.Init(memory.NewStorage(), nil)
+			require.NoError(b, err)
+			payload := bytes.Repeat([]byte("x"), size)
+			tree := transcriptReadFixture(b, repo, map[string][]byte{paths.TranscriptFileName: payload})
+			ft := NewFetchingTree(b.Context(), tree, repo.Storer, nil)
+			b.SetBytes(int64(size))
+			b.ReportAllocs()
+			for b.Loop() {
+				got, readErr := readTranscriptFromTree(b.Context(), ft, "")
+				if readErr != nil || !bytes.Equal(got, payload) {
+					b.Fatalf("transcript differs: size=%d, error=%v", len(got), readErr)
+				}
+			}
+		})
+	}
+}
+
+func transcriptReadFixture(t testing.TB, repo *git.Repository, files map[string][]byte) *object.Tree {
+	t.Helper()
+	entries := make([]object.TreeEntry, 0, len(files))
+	for name, content := range files {
+		hash, err := CreateBlobFromContent(repo, content)
+		require.NoError(t, err)
+		entries = append(entries, object.TreeEntry{Name: name, Mode: filemode.Regular, Hash: hash})
+	}
+	sortTreeEntries(entries)
+	hash, err := storeTree(repo, entries)
+	require.NoError(t, err)
+	tree, err := repo.TreeObject(hash)
+	require.NoError(t, err)
+	return tree
+}

redact/redact.go

@@ -213,13 +213,21 @@ func detectAllLayers(s string) []taggedRegion {
 	// via ConfigureScanners; betterleaks-only when unconfigured).
 	if getScanners().betterleaks {
 		if d := getDetector(); d != nil {
+			seenSecrets := make(map[string]struct{})
 			for _, f := range d.DetectString(s) {
 				// Placeholder-valued findings (changeme, secret_here, mask runs)
 				// stay visible — but only on an exact match: splitting a greedy
 				// finding at a placeholder head can leak a real secret in the tail.
 				if isPlaceholderSecretValue(f.Secret) {
 					continue
 				}
+				// A finding reports one occurrence, but the search below already
+				// covers every copy of its secret. Repeated findings must not
+				// rescan the input and accumulate quadratically many regions.
+				if _, seen := seenSecrets[f.Secret]; seen {
+					continue
+				}
+				seenSecrets[f.Secret] = struct{}{}
 				searchFrom := 0
 				for {
 					idx := strings.Index(s[searchFrom:], f.Secret)

redact/redact_bench_test.go

@@ -14,6 +14,28 @@ QyNTUxOQAAACB7ZlJ8tkWCKdRJRGF1BngP3bkNbz8bMF6Yl5xLJp9m1QAAAJj2M3UO9jN1
 DgAAAAtzc2gtZWQyNTUxOQAAACB7ZlJ8tkWCKdRJRGF1BngP3bkNbz8bMF6Yl5xLJp9m1QA
 AAEAGZmFrZS1rZXktZm9yLXJlZGFjdGlvbi1iZW5jaG1hcmstb25seQECAwQF`)
 
+// Repeated credentials can occur in captured tool output, such as request logs.
+// Each matching scanner finding must not trigger another whole-input scan for
+// all copies of the same secret within one String call. Parsed JSONL is redacted
+// per string field, so repeated values in separate fields or lines are not deduped.
+// Compare benchmark results manually against a base ref; CI does not enforce
+// a performance baseline.
+func BenchmarkRedactStringRepeatedSecret(b *testing.B) {
+	for _, repeats := range []int{1, 10, 100, 1000} {
+		b.Run(fmt.Sprintf("Occurrences%d", repeats), func(b *testing.B) {
+			input := strings.Repeat("request key=AKIAYRWQG5EJLPZLBYNP completed\n", repeats)
+			want := strings.Repeat("request key=REDACTED completed\n", repeats)
+			b.ReportAllocs()
+			b.SetBytes(int64(len(input)))
+			for b.Loop() {
+				if got := String(input); got != want {
+					b.Fatal("redacted output did not match expected request log")
+				}
+			}
+		})
+	}
+}
+
 // BenchmarkRedactJSONLBytes gives us a stable redaction performance baseline.
 //
 // To compare against a base ref: