diff --git a/cmd/entire/cli/checkpoint/persistent.go b/cmd/entire/cli/checkpoint/persistent.go index ffda9af1e8..737221dfd6 100644 --- a/cmd/entire/cli/checkpoint/persistent.go +++ b/cmd/entire/cli/checkpoint/persistent.go @@ -2750,7 +2750,7 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t chunkFiles = append([]string{paths.TranscriptFileName}, chunkFiles...) } - var chunks [][]byte + chunks := make([][]byte, 0, len(chunkFiles)) for _, chunkFile := range chunkFiles { file, err := tree.File(chunkFile) if err != nil { @@ -2760,7 +2760,7 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t ) continue } - content, err := file.Contents() + content, err := readTranscriptFile(file) if err != nil { logging.Warn(ctx, "failed to read transcript chunk contents", slog.String("chunk_file", chunkFile), @@ -2768,7 +2768,7 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t ) continue } - chunks = append(chunks, []byte(content)) + chunks = append(chunks, content) } if len(chunks) > 0 { @@ -2782,21 +2782,48 @@ func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType t // Fall back to reading base file (non-chunked or backwards compatibility) if file, err := tree.File(paths.TranscriptFileName); err == nil { - if content, err := file.Contents(); err == nil { - return []byte(content), nil + if content, err := readTranscriptFile(file); err == nil { + return content, nil } } // Try legacy filename if file, err := tree.File(paths.TranscriptFileNameLegacy); err == nil { - if content, err := file.Contents(); err == nil { - return []byte(content), nil + if content, err := readTranscriptFile(file); err == nil { + return content, nil } } return nil, nil } +// readTranscriptFile keeps large transcript blobs as bytes throughout the read. +// File.Contents grows a buffer and copies it to a string, which callers then +// copy back to bytes. Reserve the known blob size plus ReadFrom's minimum +// spare capacity so its final EOF probe does not double the whole buffer. +func readTranscriptFile(file *object.File) (content []byte, err error) { + reader, err := file.Reader() + if err != nil { + return nil, fmt.Errorf("open transcript blob: %w", err) + } + defer func() { + if closeErr := reader.Close(); err == nil && closeErr != nil { + err = fmt.Errorf("close transcript blob: %w", closeErr) + } + }() + + var buf bytes.Buffer + // Shadow transcripts can exceed MaxChunkSize without being chunked. Bound + // only the upfront allocation hint at 1 GiB; larger blobs still grow incrementally. + if file.Size >= 0 && file.Size <= 1<<30 { + buf.Grow(int(file.Size) + bytes.MinRead) + } + if _, err := buf.ReadFrom(reader); err != nil { + return nil, fmt.Errorf("read transcript blob: %w", err) + } + return buf.Bytes(), nil +} + func transcriptBlobHashesFromTreeEntries(entries []object.TreeEntry) []plumbing.Hash { hashesByName := make(map[string]plumbing.Hash) var chunkFiles []string diff --git a/cmd/entire/cli/checkpoint/transcript_read_test.go b/cmd/entire/cli/checkpoint/transcript_read_test.go new file mode 100644 index 0000000000..67dba6951e --- /dev/null +++ b/cmd/entire/cli/checkpoint/transcript_read_test.go @@ -0,0 +1,90 @@ +package checkpoint + +import ( + "bytes" + "fmt" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/paths" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage/memory" + "github.com/stretchr/testify/require" +) + +func TestReadTranscriptFromTreeFormats(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + files map[string][]byte + want []byte + }{ + {name: "base", files: map[string][]byte{paths.TranscriptFileName: []byte("first\nsecond\n")}, want: []byte("first\nsecond\n")}, + {name: "legacy", files: map[string][]byte{paths.TranscriptFileNameLegacy: []byte("legacy\n")}, want: []byte("legacy\n")}, + // A present empty transcript must remain non-nil: ephemeral reads use + // nil to decide whether to fall through to their legacy read path. + {name: "empty", files: map[string][]byte{paths.TranscriptFileName: {}}, want: []byte{}}, + {name: "absent", files: map[string][]byte{}, want: nil}, + { + name: "chunk order", + files: map[string][]byte{ + paths.TranscriptFileName: []byte("zero"), + paths.TranscriptFileName + ".010": []byte("ten"), + paths.TranscriptFileName + ".002": []byte("two"), + paths.TranscriptFileName + ".001": []byte("one"), + }, + want: []byte("zero\none\ntwo\nten"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + repo, err := git.Init(memory.NewStorage(), nil) + require.NoError(t, err) + tree := transcriptReadFixture(t, repo, tc.files) + got, err := readTranscriptFromTree(t.Context(), NewFetchingTree(t.Context(), tree, repo.Storer, nil), "") + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func BenchmarkReadTranscriptFromTree(b *testing.B) { + // The 64 MiB case covers unchunked shadow transcripts above MaxChunkSize. + for _, size := range []int{1 << 20, 8 << 20, 64 << 20} { + b.Run(fmt.Sprintf("%dMiB", size>>20), func(b *testing.B) { + // In-memory objects isolate transcript decoding and allocation costs + // from filesystem cache state; tree and blob lookup are still real. + repo, err := git.Init(memory.NewStorage(), nil) + require.NoError(b, err) + payload := bytes.Repeat([]byte("x"), size) + tree := transcriptReadFixture(b, repo, map[string][]byte{paths.TranscriptFileName: payload}) + ft := NewFetchingTree(b.Context(), tree, repo.Storer, nil) + b.SetBytes(int64(size)) + b.ReportAllocs() + for b.Loop() { + got, readErr := readTranscriptFromTree(b.Context(), ft, "") + if readErr != nil || !bytes.Equal(got, payload) { + b.Fatalf("transcript differs: size=%d, error=%v", len(got), readErr) + } + } + }) + } +} + +func transcriptReadFixture(t testing.TB, repo *git.Repository, files map[string][]byte) *object.Tree { + t.Helper() + entries := make([]object.TreeEntry, 0, len(files)) + for name, content := range files { + hash, err := CreateBlobFromContent(repo, content) + require.NoError(t, err) + entries = append(entries, object.TreeEntry{Name: name, Mode: filemode.Regular, Hash: hash}) + } + sortTreeEntries(entries) + hash, err := storeTree(repo, entries) + require.NoError(t, err) + tree, err := repo.TreeObject(hash) + require.NoError(t, err) + return tree +} diff --git a/redact/redact.go b/redact/redact.go index 8331552c01..0ec5bda386 100644 --- a/redact/redact.go +++ b/redact/redact.go @@ -213,6 +213,7 @@ func detectAllLayers(s string) []taggedRegion { // via ConfigureScanners; betterleaks-only when unconfigured). if getScanners().betterleaks { if d := getDetector(); d != nil { + seenSecrets := make(map[string]struct{}) for _, f := range d.DetectString(s) { // Placeholder-valued findings (changeme, secret_here, mask runs) // stay visible — but only on an exact match: splitting a greedy @@ -220,6 +221,13 @@ func detectAllLayers(s string) []taggedRegion { if isPlaceholderSecretValue(f.Secret) { continue } + // A finding reports one occurrence, but the search below already + // covers every copy of its secret. Repeated findings must not + // rescan the input and accumulate quadratically many regions. + if _, seen := seenSecrets[f.Secret]; seen { + continue + } + seenSecrets[f.Secret] = struct{}{} searchFrom := 0 for { idx := strings.Index(s[searchFrom:], f.Secret) diff --git a/redact/redact_bench_test.go b/redact/redact_bench_test.go index 7d3140620c..5f8426ac83 100644 --- a/redact/redact_bench_test.go +++ b/redact/redact_bench_test.go @@ -14,6 +14,28 @@ QyNTUxOQAAACB7ZlJ8tkWCKdRJRGF1BngP3bkNbz8bMF6Yl5xLJp9m1QAAAJj2M3UO9jN1 DgAAAAtzc2gtZWQyNTUxOQAAACB7ZlJ8tkWCKdRJRGF1BngP3bkNbz8bMF6Yl5xLJp9m1QA AAEAGZmFrZS1rZXktZm9yLXJlZGFjdGlvbi1iZW5jaG1hcmstb25seQECAwQF`) +// Repeated credentials can occur in captured tool output, such as request logs. +// Each matching scanner finding must not trigger another whole-input scan for +// all copies of the same secret within one String call. Parsed JSONL is redacted +// per string field, so repeated values in separate fields or lines are not deduped. +// Compare benchmark results manually against a base ref; CI does not enforce +// a performance baseline. +func BenchmarkRedactStringRepeatedSecret(b *testing.B) { + for _, repeats := range []int{1, 10, 100, 1000} { + b.Run(fmt.Sprintf("Occurrences%d", repeats), func(b *testing.B) { + input := strings.Repeat("request key=AKIAYRWQG5EJLPZLBYNP completed\n", repeats) + want := strings.Repeat("request key=REDACTED completed\n", repeats) + b.ReportAllocs() + b.SetBytes(int64(len(input))) + for b.Loop() { + if got := String(input); got != want { + b.Fatal("redacted output did not match expected request log") + } + } + }) + } +} + // BenchmarkRedactJSONLBytes gives us a stable redaction performance baseline. // // To compare against a base ref: