From cea07a8afd3591d3d7692197ace9d518489f9c82 Mon Sep 17 00:00:00 2001 From: lystopad Date: Mon, 10 Aug 2026 19:07:17 +0200 Subject: [PATCH] execution/commitment: fix wave-BFS livelock when a step budget fills exactly (#23066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `ContractTrunkPreloadParallel.Run` can spin forever in its wave loop, wedging the node. The loop only breaks on `budgetHit`, which `pin()` raises on a *strict* overflow. A wave that pins nothing never raises it, and the frontier is reassigned at the same depth — identical state on re-entry. Two ways a wave pins nothing: - **Whole miss set deferred.** `fileBudget` on re-entry is `stepCap - (usedBytes + dbHitsBytes)`, bit-identical to the value that caused the deferral, because the db-hits consume exactly the bytes the deferral already accounted for. Infinite. - **Capped fetch resolves to nothing.** Every fetched key is absent from the file layer — the normal shape at the BFS fringe, where a set `afterMap` bit names a leaf with no branch record. Re-enters once per chunk: bounded, but quadratic. ## Impact Hit in production on a Gnosis validator (`release/3.6`, 4402a1d18e): ``` sortAndPartitionFrontier preload_parallel.go:121 Run preload_parallel.go:216 runExtensionLocked adaptive_pin.go:354 OnBlockComplete adaptive_pin.go:206 SharedDomains.Commit domain_shared.go:1030 updateForkChoice forkchoice.go:742 ``` The stuck goroutine holds the execution semaphore inside an FCU, so `NewPayload` blocks while holding the fork-choice write mutex. The CL slot ticker stops and ~420 `getAttesterDuties` handlers pile up on the read lock. The node stopped attesting and producing for 17h while burning a core; `eth_syncing` still reported `false`. Two dumps minutes apart showed the same goroutine id, receiver and `dbBranches` pointers — livelock, not a slow walk. 68,732 extension runs over 5.9d uptime before it hit. ## Fix End the step when a wave defers part of its miss set and pinned nothing. Each iteration now either breaks, strictly shrinks the frontier, or increments `nextDepth`. Deferred work resumes on the next `Run`, so there is no throughput cost. Also: - Floor the file fetch at `minEntryBytes` — below one entry's cost the batch is pure waste. - Add the missing `cache == nil` check to `PreloadContractTrunkParallel`: `Run` returns the error, then the wrapper's logger block dereferences `cache.PinnedCount()`, so callers got a panic instead. - Fold two disagreeing `queueEmpty` expressions into one method. ## Testing Differential run of the fixed `Run` against the pre-guard one over 40k configs / 2.3M steps: identical pins, `queueEmpty` and `usedBytes` at every step. The pre-guard code livelocked in 18,306 of those configs. Thanks @awskii. - `ExactBudgetFillTerminates` — the production state; hangs before the fix. - `NoPinWaveEndsStep` — both no-pin paths as two budgets one byte apart, asserting the exact resolver call count. Each case fails against exactly one of the two production changes reverted, so neither is left unpinned by the other. - `DeferWithDbHitsInSameWave` — deferral boundary with db-hits pinned in the same wave. - `StepBudgetSweepTerminates` — budgets straddling one entry's cost. Hang guards abort the binary rather than `t.Fatal`, since `Run` has no cancellation and a spinning goroutine would outlive the failure. `execution/commitment` green under `-race -count=2`; `make lint` clean. ## Scope `preload_parallel.go` is the only site with this shape: the serial `ContractTrunkPreload.Run` pops its queue head unconditionally, `OnBlockComplete` ranges a bounded set, and `execStatusList.drainDeferred` already has a progress net. `release/3.6` needs a backport; `release/3.5` does not have the feature. ## Notes (not fixed here) - `adaptive_pin.go`: `c.misses` entries are never removed, so one `sync.Map` entry accumulates per contract hash ever touched and `OnBlockComplete` ranges the whole set every block. Found by @awskii. - `commitment_trunk_preload_{bytes,duration_seconds}_total` are declared but never written — no metric signal for preload work while this was wedged. Fixed in #23067. - When `PerContractMaxBudgetBytes - usedBytes` drops below one entry's cost, `runExtensionLocked` runs every block and pins nothing. Bounded per-block work, not a hang. --------- Co-authored-by: awskii --- execution/commitment/preload_parallel.go | 55 ++- execution/commitment/preload_parallel_test.go | 400 ++++++++++++++++++ 2 files changed, 439 insertions(+), 16 deletions(-) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 01fb27395aa..f9187a97a1a 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -40,7 +40,8 @@ func estimatedEntryCost(key, value []byte) int { // minEntryBytes: true lower bound on estimatedEntryCost for a storage-trunk // branch (33 = shortest HexToCompact key at depth >= 64; value may be empty). -// Bounds a wave's file fetch so the budget is guaranteed exhausted inside it. +// Bounds a wave's file fetch to what the remaining budget could still pin; a +// wave that pins nothing anyway is ended by the no-progress check in Run. const minEntryBytes = estimatedEntryOverheadBytes + 33 // maxStorageTrunkDepth: 64 (account path) + 64 (keccak256(slot)) = 128. @@ -164,19 +165,19 @@ func (p *ContractTrunkPreloadParallel) Run( return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: resolver is nil") } if stepBudgetBytes <= 0 { - return 0, len(p.frontier) == 0, nil + return 0, p.queueEmpty(), nil } defer p.releaseScratch() stepCap := p.usedBytes + stepBudgetBytes chunkPinned := 0 - budgetHit := false + endStep := false // pin records the entry and queues its children. Returns false on budget hit. pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { cost := estimatedEntryCost(pk.key, v) if p.usedBytes+cost > stepCap { - budgetHit = true + endStep = true return false } // step=0: a storage-trunk branch resolved across merged files has no single @@ -210,13 +211,15 @@ func (p *ContractTrunkPreloadParallel) Run( return true } - for !budgetHit && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { + for !endStep && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { depth := p.nextDepth + wavePinnedBefore := chunkPinned dbHits, dbVals, fileMiss, dbHitsBytes := p.sortAndPartitionFrontier(dbBranches) - // Cap the file fetch by what the budget can absorb after dbHits. + // Cap the file fetch by what the budget can absorb after dbHits. Below + // minEntryBytes no file entry can be pinned, so fetching any is waste. var fileMissDeferred []pathKey - if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { + if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget < minEntryBytes { fileMissDeferred = fileMiss fileMiss = nil } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { @@ -247,8 +250,9 @@ func (p *ContractTrunkPreloadParallel) Run( } p.dbHitsPinned++ } - fileMissStop := len(fileMiss) - if !budgetHit { + fileMissStop := 0 + if !endStep { + fileMissStop = len(fileMiss) for i, pk := range fileMiss { v := fileVals[i] if v == nil { @@ -261,7 +265,16 @@ func (p *ContractTrunkPreloadParallel) Run( } } - if budgetHit { + // Re-entering only helps if the next iteration sees a different budget. + // It does not when the whole miss set was deferred (dbHits consume + // exactly the bytes the deferral already accounted for), nor when a + // capped fetch pinned nothing because its keys were absent from the file + // layer. Either way, end the step; the tail resumes on the next Run. + if len(fileMissDeferred) > 0 && (len(fileMiss) == 0 || chunkPinned == wavePinnedBefore) { + endStep = true + } + + if endStep { // Preserve un-pinned items at current depth; pendingChildren stays // at depth+1 for when this depth is drained on a future Run. rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) @@ -273,8 +286,9 @@ func (p *ContractTrunkPreloadParallel) Run( } if len(fileMissDeferred) > 0 { - // Defensive: !budgetHit should mean no truncation. Clone out of the - // scratch-aliased slice so the next wave's partition can't overwrite it. + // Capped fetch that still pinned something: stay at this depth and + // resume with the tail. Clone out of the scratch-aliased slice so the + // next wave's partition can't overwrite it. p.frontier = slices.Clone(fileMissDeferred) } else { p.frontier = p.pendingChildren @@ -283,7 +297,7 @@ func (p *ContractTrunkPreloadParallel) Run( } } - queueEmpty = (len(p.frontier) == 0 && len(p.pendingChildren) == 0) || p.nextDepth > maxStorageTrunkDepth + queueEmpty = p.queueEmpty() if logger != nil && (chunkPinned > 0 || queueEmpty) { logger.Info("[trunk-preload-parallel] step", "step_budget_mb", stepBudgetBytes/(1<<20), @@ -310,6 +324,12 @@ func (p *ContractTrunkPreloadParallel) QueueRemaining() int { return len(p.frontier) + len(p.pendingChildren) } +// queueEmpty reports that no further Run can pin anything: the walk drained, or +// it reached the depth ceiling with entries still queued below it. +func (p *ContractTrunkPreloadParallel) queueEmpty() bool { + return p.QueueRemaining() == 0 || p.nextDepth > maxStorageTrunkDepth +} + // PinnedPrefixes returns slices aliasing internal storage — do not mutate. func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } @@ -326,13 +346,16 @@ func PreloadContractTrunkParallel( if ramBudgetBytes <= 0 { return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) } - p, err := NewContractTrunkPreloadParallel(contractHash) - if err != nil { - return 0, err + if cache == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: cache is nil") } if resolve == nil { return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") } + p, err := NewContractTrunkPreloadParallel(contractHash) + if err != nil { + return 0, err + } pinned, queueEmpty, err := p.Run(ramBudgetBytes, dbBranches, resolve, cache, logger) if logger != nil { logger.Info("[trunk-preload-parallel] complete", diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go index b9cb886cfc6..8d77f84f44c 100644 --- a/execution/commitment/preload_parallel_test.go +++ b/execution/commitment/preload_parallel_test.go @@ -13,9 +13,14 @@ import ( "cmp" "encoding/binary" "errors" + "fmt" + "runtime/debug" "slices" + "strings" "testing" + "time" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/execution/commitment/nibbles" ) @@ -714,6 +719,22 @@ func TestContractTrunkPreloadParallel_NilCacheError(t *testing.T) { } } +// The wrapper logs unconditionally after Run and dereferences the cache there, +// so a nil cache reached that block and panicked before the caller ever saw +// Run's error. Only a non-nil logger reaches it. +func TestPreloadContractTrunkParallel_NilCacheWithLoggerReturnsError(t *testing.T) { + hash := make([]byte, 32) + resolve := func(keys [][]byte) ([][]byte, error) { return make([][]byte, len(keys)), nil } + + _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, resolve, nil, log.Root()) + if err == nil { + t.Fatal("expected error when cache is nil") + } + if !strings.Contains(err.Error(), "cache is nil") { + t.Fatalf("error %q, want it to name the nil cache", err) + } +} + func TestContractTrunkPreloadParallel_NilResolverError(t *testing.T) { hash := make([]byte, 32) c := NewBranchCache(64) @@ -785,3 +806,382 @@ func TestContractTrunkPreloadParallel_BadHashLengthError(t *testing.T) { t.Fatal("expected error for 33-byte hash") } } + +// panicOnStuck aborts the test binary instead of failing the test: Run has no +// cancellation, so a spinning goroutine would outlive a t.Fatal and burn a core +// for the rest of the package run. +func panicOnStuck(why string) { + debug.SetTraceback("all") + panic("ContractTrunkPreloadParallel.Run did not terminate: " + why) +} + +// TestContractTrunkPreloadParallel_ExactBudgetFillTerminates covers a wave whose +// pins land usedBytes exactly on stepCap, followed by a frontier that misses +// dbBranches entirely. That leaves no budget for a single file entry, so the +// whole miss set is deferred and nothing is pinned — depth, frontier and +// usedBytes all stay put and the wave must not be re-entered. +func TestContractTrunkPreloadParallel_ExactBudgetFillTerminates(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := string(hexNibbles(hash)) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + rootKey := bytes.Clone(nibbles.HexToCompact([]byte(root))) + rootVal := branchVal(tree[root], valSz) + // Shadow only the root, so every wave below depth 64 is a pure file miss. + dbBranches := map[string][]byte{string(rootKey): rootVal} + + c := NewBranchCache(64) + defer c.Close() + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + + // Budget the root pin consumes exactly, landing usedBytes on stepCap. + stepBudget := estimatedEntryCost(rootKey, rootVal) + + type runResult struct { + pinned int + queueEmpty bool + err error + } + res := make(chan runResult, 1) + go func() { + n, done, err := p.Run(stepBudget, dbBranches, resolve, c, nil) + res <- runResult{n, done, err} + }() + + var got runResult + select { + case got = <-res: + case <-time.After(10 * time.Second): + panicOnStuck("a wave with no file budget defers the whole frontier without pinning, so the loop re-enters on identical state") + } + if got.err != nil { + t.Fatal(got.err) + } + if got.queueEmpty { + t.Fatal("queue reported empty, but the root's children were never pinned") + } + if got.pinned != 1 { + t.Fatalf("pinned %d entries, want 1 (the root)", got.pinned) + } + + // The deferred frontier must survive so a later, larger step finishes the tree. + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after a large budget; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("pinned %d entries, want the whole tree (%d)", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_StepBudgetSweepTerminates sweeps step budgets +// straddling one entry's cost, including the values that leave a wave with zero +// file budget. Every Run must return; a budget that can afford the costliest +// entry must additionally finish the tree, since it can always pin at least one +// entry per step. +func TestContractTrunkPreloadParallel_StepBudgetSweepTerminates(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := string(hexNibbles(hash)) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + rootKey := bytes.Clone(nibbles.HexToCompact([]byte(root))) + rootVal := branchVal(tree[root], valSz) + dbBranches := map[string][]byte{string(rootKey): rootVal} + exact := estimatedEntryCost(rootKey, rootVal) + + // Entry cost grows with path depth, so derive the guaranteed-progress budget + // from the costliest entry in the tree rather than scaling the root's cost. + affordable := 0 + for path, afterMap := range tree { + if cost := estimatedEntryCost(nibbles.HexToCompact([]byte(path)), branchVal(afterMap, valSz)); cost > affordable { + affordable = cost + } + } + + const maxSteps = 200 + // Deduped: the derived budgets can coincide, and a repeated value would give + // two subtests the same name. + budgets := []int{1, minEntryBytes, exact - 1, exact, exact + 1, affordable, 1 << 20} + slices.Sort(budgets) + budgets = slices.Compact(budgets) + + type sweepResult struct { + done bool + pinned int + err error + } + + // One subtest per budget: a livelock regression that only bites one of them + // names that budget instead of hanging the sweep behind a shared guard. + for _, budget := range budgets { + t.Run(fmt.Sprintf("budget=%d", budget), func(t *testing.T) { + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + c := NewBranchCache(64) + defer c.Close() + + results := make(chan sweepResult, 1) + go func() { + var r sweepResult + for range maxSteps { + _, done, err := p.Run(budget, dbBranches, resolve, c, nil) + if err != nil { + r.err = err + break + } + if done { + r.done = true + break + } + } + r.pinned = p.PinnedTotal() + results <- r + }() + + var r sweepResult + select { + case r = <-results: + case <-time.After(30 * time.Second): + panicOnStuck(fmt.Sprintf("step budget %d: a wave that pins nothing must end the step, not re-enter on an unchanged frontier", budget)) + } + if r.err != nil { + t.Fatal(r.err) + } + + // A budget below one entry's cost can never pin the root; it must still + // return from every Run, but it cannot make progress. + if budget < exact { + if r.pinned != 0 { + t.Fatalf("pinned %d entries on a sub-entry budget", r.pinned) + } + return + } + if budget < affordable { + return + } + if !r.done { + t.Fatalf("not complete after %d steps (pinned %d/%d)", maxSteps, r.pinned, len(tree)) + } + if r.pinned != len(tree) { + t.Fatalf("pinned %d entries, want the whole tree (%d)", r.pinned, len(tree)) + } + }) + } +} + +// TestContractTrunkPreloadParallel_NoPinWaveEndsStep covers the two ways a wave +// ends a step having pinned nothing. Either the residual budget is below one +// entry, so the miss set is deferred without being fetched, or it affords a +// capped fetch whose keys are all absent from the file layer — the normal shape +// at the BFS fringe, where a set afterMap bit names a leaf with no branch +// record. Neither raises endStep from pin(), so the wave used to grind through +// the rest of the depth inside one Run, reporting the queue drained and issuing +// a resolver batch per chunk. The call count is what pins that down. +func TestContractTrunkPreloadParallel_NoPinWaveEndsStep(t *testing.T) { + hash := make([]byte, 32) + for i := range hash { + hash[i] = 0x55 + } + root := string(hexNibbles(hash)) + // Root only: its 16 children are named by the bitmap but absent from the + // file layer, so every depth-65 key resolves to nil. + tree := syntheticTree{root: 0xffff} + const valSz = 100 + rootKey := nibbles.HexToCompact([]byte(root)) + rootCost := estimatedEntryCost(rootKey, branchVal(0xffff, valSz)) + + for _, tc := range []struct { + name string + stepBudget int + wantCalls int + }{ + // No depth-65 entry fits, so fetching any of them is waste: only the + // root wave reaches the resolver. + {"residual below one entry defers unfetched", rootCost + minEntryBytes - 1, 1}, + // One byte more affords a capped fetch. It resolves, pins nothing, and + // must still end the step: one batch for the root, one for the cap. + {"capped fetch pins nothing", rootCost + minEntryBytes, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + base := fakeResolver(tree, nil, valSz, "") + resolve := func(keys [][]byte) ([][]byte, error) { + calls++ + return base(keys) + } + + c := NewBranchCache(64) + defer c.Close() + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + var pinned int + var queueEmpty bool + go func() { + pinned, queueEmpty, err = p.Run(tc.stepBudget, nil, resolve, c, nil) + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + panicOnStuck("a wave that pins nothing must end the step, not re-enter on the deferred tail") + } + if err != nil { + t.Fatal(err) + } + if pinned != 1 || queueEmpty { + t.Fatalf("pinned %d queueEmpty=%v, want 1 and false (root only, children deferred)", pinned, queueEmpty) + } + if calls != tc.wantCalls { + t.Fatalf("resolver called %d times for a 16-wide wave, want %d; the wave was re-entered per chunk", calls, tc.wantCalls) + } + if p.QueueRemaining() == 0 { + t.Fatal("deferred children were dropped instead of resumed") + } + + // A full budget drains the absent children and completes the preload. + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after a large budget; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal() != 1 { + t.Fatalf("pinned %d, want 1 (only the root exists in the file layer)", p.PinnedTotal()) + } + }) + } +} + +// TestContractTrunkPreloadParallel_DeferWithDbHitsInSameWave covers the shape +// the other termination tests miss: the wave that ends the step also pinned +// db-hits, so the resumed frontier is a concatenation (unpinned db-hits, then +// the deferred misses) at the current depth while pendingChildren already holds +// the pinned hits' children one level down. Deferring the miss unfetched and +// fetching it only to fail the pin leave an identical frontier, so the resolver +// call count is what tells them apart. +func TestContractTrunkPreloadParallel_DeferWithDbHitsInSameWave(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := string(hexNibbles(hash)) + const valSz = 100 + calls := 0 + base := fakeResolver(tree, nil, valSz, "") + resolve := func(keys [][]byte) ([][]byte, error) { + calls++ + return base(keys) + } + + // Shadow R1 only: the depth-65 wave partitions into one db-hit and one file + // miss, and the budget left after pinning R1 cannot afford the miss. + r1 := root + string([]byte{1}) + r1Key := bytes.Clone(nibbles.HexToCompact([]byte(r1))) + dbBranches := map[string][]byte{string(r1Key): branchVal(tree[r1], valSz)} + + c := NewBranchCache(64) + defer c.Close() + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + + rootKey := nibbles.HexToCompact([]byte(root)) + rootCost := estimatedEntryCost(rootKey, branchVal(tree[root], valSz)) + r1Cost := estimatedEntryCost(r1Key, branchVal(tree[r1], valSz)) + // Room for the root and R1, with a remainder below any entry's cost. + stepBudget := rootCost + r1Cost + minEntryBytes - 1 + + done := make(chan struct{}) + var pinned int + var queueEmpty bool + go func() { + pinned, queueEmpty, err = p.Run(stepBudget, dbBranches, resolve, c, nil) + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + panicOnStuck("a wave that defers its misses while pinning db-hits must end the step") + } + if err != nil { + t.Fatal(err) + } + if pinned != 2 || queueEmpty { + t.Fatalf("pinned %d queueEmpty=%v, want 2 (root+R1) and false", pinned, queueEmpty) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("db-hits pinned %d, want 1 (R1 came from dbBranches)", p.DbHitsPinned()) + } + // The remainder is below one entry's cost, so the miss must be deferred + // without being fetched: only the root wave reaches the resolver. + if calls != 1 { + t.Fatalf("resolver called %d times, want 1; the deferred miss was fetched anyway", calls) + } + // R2 stays at depth 65 in the frontier; R1's child is queued at depth 66. + if len(p.frontier) != 1 || len(p.pendingChildren) != 1 { + t.Fatalf("frontier=%d pendingChildren=%d, want 1 and 1", len(p.frontier), len(p.pendingChildren)) + } + if got := string(p.frontier[0].path); got != root+string([]byte{2}) { + t.Fatalf("frontier holds %x, want the deferred R2", got) + } + if got := string(p.pendingChildren[0].path); got != r1+string([]byte{3}) { + t.Fatalf("pendingChildren holds %x, want R1's child", got) + } + + // Nothing was lost: a full budget finishes the tree exactly once each. + if _, done, err := p.Run(1<<20, dbBranches, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after a large budget; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("pinned %d, want the whole tree (%d)", p.PinnedTotal(), len(tree)) + } + seen := map[string]bool{} + for _, pf := range p.PinnedPrefixes() { + if seen[string(pf)] { + t.Fatalf("prefix %x pinned twice across the deferral boundary", pf) + } + seen[string(pf)] = true + } +} + +// A walk stopped at the depth ceiling can pin nothing more even with entries +// still queued, and must report that identically whether or not the step budget +// is spendable — the zero-budget path returns before the loop that would +// otherwise notice the ceiling. +func TestContractTrunkPreloadParallel_DepthCeilingReportsDoneOnAnyBudget(t *testing.T) { + hash := make([]byte, 32) + resolve := func(keys [][]byte) ([][]byte, error) { return make([][]byte, len(keys)), nil } + c := NewBranchCache(64) + defer c.Close() + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + + // Place the walk at the ceiling rather than building a 64-level storage trie. + p.nextDepth = maxStorageTrunkDepth + 1 + if p.QueueRemaining() == 0 { + t.Fatal("queue drained, so the queued-but-finished state is not exercised") + } + + if _, queueEmpty, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !queueEmpty { + t.Error("a spendable budget past the depth ceiling reported work remaining") + } + if _, queueEmpty, err := p.Run(0, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !queueEmpty { + t.Error("a zero budget past the depth ceiling disagreed with the spendable one") + } +}