From 344e183b080f438f6cc9f92f32c1e0dec86a97ec Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:09:05 +0000 Subject: [PATCH 1/6] execution: fix bal warmuper --- common/dbg/experiments.go | 1 + execution/exec/blocks_read_ahead.go | 200 ++++++++++++----------- execution/exec/blocks_read_ahead_test.go | 90 ++++++++++ execution/execmodule/exec_module.go | 20 +-- 4 files changed, 194 insertions(+), 117 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index ecf06414c0a..281cebeccac 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -136,6 +136,7 @@ var ( DisableAdaptivePin = EnvBool("DISABLE_ADAPTIVE_PIN", false) AssertStateCache = EnvBool("ASSERT_STATE_CACHE", false) ReadAhead = EnvBool("READ_AHEAD", true) + ReadAheadWorkers = EnvInt("READ_AHEAD_WORKERS", runtime.NumCPU()) // FilesAsyncIO warms cold state .kv pages via io_uring before the mmap read, so // a would-be blocking page fault becomes a non-blocking read that releases the // goroutine's P. Linux + io_uring only; self-disables (reads use ordinary faults) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index baa7cd78823..b3e7ae24dee 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -137,7 +137,7 @@ func (cpg *cachePopulatingGetter) StepsInFiles(entitySet ...kv.Domain) kv.Step { return cpg.g.StepsInFiles(entitySet...) } -func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body) { +func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, tx kv.Getter, header *types.Header, body *types.Body) { blockHash := header.Hash() bra.headers.Add(blockHash, header) bra.bodies.Add(blockHash, body) @@ -146,8 +146,18 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, h if !bra.warming.CompareAndSwap(false, true) { return } + var bal types.BlockAccessList + balBytes, err := tx.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), blockHash)) + if err != nil { + log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", blockHash, "err", err) + } else if len(balBytes) > 0 { + bal, err = types.DecodeBlockAccessListBytes(balBytes) + if err != nil { + log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", blockHash, "err", err) + } + } bra.warmWg.Go(func() { - bra.warmBody(ctx, db, header, body, 8) // use 8 workers for warming + bra.warmBody(ctx, db, body, bal, dbg.ReadAheadWorkers) }) } } @@ -174,127 +184,127 @@ func (bra *BlockReadAheader) AddSenders(senders []byte, blockHash common.Hash) { bra.senders.Add(blockHash, bytes.Clone(senders)) } -// warmBody warms state for all transactions in a body using multiple workers. -// It reads: To accounts, To account code, To account storage from access lists, -// and block-level access lists. Each worker creates its own transaction. -// Only one warmBody can run at a time - concurrent calls are no-ops. -func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body, workers int) { - defer bra.warming.Store(false) +type balWarmupTaskKind uint8 +const ( + balWarmAccount balWarmupTaskKind = iota + balWarmStorageChanges + balWarmStorageReads +) + +type balWarmupTask struct { + accountIndex int + kind balWarmupTaskKind + slotIndex int +} + +func makeBALWarmupPlan(bal types.BlockAccessList, workers int) ([]balWarmupTask, int) { + taskCount := len(bal) + for _, account := range bal { + taskCount += len(account.StorageChanges) + len(account.StorageReads) + } + tasks := make([]balWarmupTask, 0, taskCount) + for accountIndex, account := range bal { + tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmAccount}) + for slotIndex := range account.StorageChanges { + tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmStorageChanges, slotIndex: slotIndex}) + } + for slotIndex := range account.StorageReads { + tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmStorageReads, slotIndex: slotIndex}) + } + } + return tasks, min(workers, len(tasks)) +} + +func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, body *types.Body, bal types.BlockAccessList, workers int) { + defer bra.warming.Store(false) if !dbg.ReadAhead { return } - if workers <= 0 { workers = 1 } + if len(bal) > 0 { + bra.warmBAL(ctx, db, bal, workers) + return + } + bra.warmTxns(ctx, db, body.Transactions, workers) +} +func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, workers int) { + tasks, balWorkers := makeBALWarmupPlan(bal, workers) + var nextTask atomic.Uint64 var wg errgroup.Group - - // If BAL exists in DB, use BAL warming (more complete) - var bal types.BlockAccessList - if header != nil && db != nil { - tx, err := db.BeginRo(ctx) - if err != nil { - log.Warn("[warmBody] failed to open tx for BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } else { - data, err := tx.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), header.Hash())) + for w := range balWorkers { + workerID := w + wg.Go(func() error { + startTime := time.Now() + tx, err := db.BeginRo(ctx) if err != nil { - log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } else if len(data) > 0 { - bal, err = types.DecodeBlockAccessListBytes(data) - if err != nil { - log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } + return err } - tx.Rollback() - } - } - - balLen := len(bal) - if balLen > 0 { - balWorkers := min(workers, balLen) - - // Pre-divide work: each worker gets a dedicated range of BAL entries - entriesPerWorker := (balLen + balWorkers - 1) / balWorkers - - for w := range balWorkers { - start := w * entriesPerWorker - end := min(start+entriesPerWorker, balLen) - if start >= balLen { - break + defer tx.Rollback() + ttx, ok := tx.(kv.TemporalTx) + if !ok { + return nil } - - // Capture loop variables for closure - workerStart, workerEnd, workerID := start, end, w - wg.Go(func() error { - startTime := time.Now() - tx, err := db.BeginRo(ctx) - if err != nil { - return err - } - defer tx.Rollback() - - ttx, ok := tx.(kv.TemporalTx) - if !ok { - return nil + var getter kv.TemporalGetter = ttx + if bra.stateCache != nil { + getter = newCachePopulatingGetter(ttx, bra.stateCache) + } + stateReader := state.NewReaderV3(getter) + tasksProcessed := 0 + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: } - var getter kv.TemporalGetter = ttx - if bra.stateCache != nil { - getter = newCachePopulatingGetter(ttx, bra.stateCache) + taskIndex := int(nextTask.Add(1) - 1) + if taskIndex >= len(tasks) { + break } - stateReader := state.NewReaderV3(getter) - - for idx := workerStart; idx < workerEnd; idx++ { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - acctChanges := bal[idx] + task := tasks[taskIndex] + acctChanges := bal[task.accountIndex] + switch task.kind { + case balWarmAccount: acct, _ := stateReader.ReadAccountData(acctChanges.Address) - // Warm code if account has code or if there are code changes. - if acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(acctChanges.Address) - } else if len(acctChanges.CodeChanges) > 0 { + if (acct != nil && !acct.CodeHash.IsEmpty()) || len(acctChanges.CodeChanges) > 0 { stateReader.ReadAccountCode(acctChanges.Address) } - for _, slotChanges := range acctChanges.StorageChanges { - stateReader.ReadAccountStorage(acctChanges.Address, slotChanges.Slot) - } - for _, slot := range acctChanges.StorageReads { - stateReader.ReadAccountStorage(acctChanges.Address, slot) - } + case balWarmStorageChanges: + slot := acctChanges.StorageChanges[task.slotIndex].Slot + stateReader.ReadAccountStorage(acctChanges.Address, slot) + case balWarmStorageReads: + slot := acctChanges.StorageReads[task.slotIndex] + stateReader.ReadAccountStorage(acctChanges.Address, slot) } - log.Debug("[warmBody] BAL worker finished", "worker", workerID, "entries", workerEnd-workerStart, "elapsed", time.Since(startTime)) - return nil - }) - } - wg.Wait() - return + tasksProcessed++ + } + log.Debug("[warmBAL] worker finished", "worker", workerID, "tasks", tasksProcessed, "elapsed", time.Since(startTime)) + return nil + }) } - // Fallback: per-transaction warming when no BAL - txns := body.Transactions + wg.Wait() +} + +func (bra *BlockReadAheader) warmTxns(ctx context.Context, db kv.RoDB, txns types.Transactions, workers int) { if len(txns) == 0 { return } - txnLen := len(txns) if workers > txnLen { workers = txnLen } - // Pre-divide work: each worker gets a dedicated range of transactions txnsPerWorker := (txnLen + workers - 1) / workers - + var wg errgroup.Group for w := 0; w < workers; w++ { start := w * txnsPerWorker end := min(start+txnsPerWorker, txnLen) if start >= txnLen { break } - // Capture loop variables for closure workerStart, workerEnd, workerID := start, end, w wg.Go(func() error { @@ -304,28 +314,22 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t return err } defer tx.Rollback() - ttx, ok := tx.(kv.TemporalTx) if !ok { return nil } var getter kv.TemporalGetter = ttx - var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = newCachePopulatingGetter(ttx, bra.stateCache) - getter = cpg + getter = newCachePopulatingGetter(ttx, bra.stateCache) } stateReader := state.NewReaderV3(getter) - for txIdx := workerStart; txIdx < workerEnd; txIdx++ { select { case <-ctx.Done(): return ctx.Err() default: } - txn := txns[txIdx] - // Warm To account and its code if it has one if toAddr := txn.GetTo(); toAddr != nil { to := accounts.InternAddress(*toAddr) @@ -333,7 +337,6 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t stateReader.ReadAccountCode(to) } } - // Warm transaction access list accounts and their code for _, entry := range txn.GetAccessList() { addr := accounts.InternAddress(entry.Address) @@ -345,11 +348,10 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } } } - log.Debug("[warmBody] TX worker finished", "worker", workerID, "txns", workerEnd-workerStart, "elapsed", time.Since(startTime)) + log.Debug("[warmTxns] worker finished", "worker", workerID, "txns", workerEnd-workerStart, "elapsed", time.Since(startTime)) return nil }) } - wg.Wait() } diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 3566bd6bb1e..4b339dd3e90 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -20,11 +20,22 @@ import ( "testing" "github.com/c2h5oh/datasize" + "github.com/holiman/uint256" "github.com/stretchr/testify/require" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/dbg" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/membatchwithdb" + "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/cache" + "github.com/erigontech/erigon/execution/types" + "github.com/erigontech/erigon/execution/types/accounts" ) // stubTemporalGetter stands in for the committed-state snapshot a warmup @@ -49,6 +60,85 @@ func newTestStateCache() *cache.StateCache { return cache.NewStateCache(b, b, b, b) } +func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { + bal := types.BlockAccessList{{ + StorageChanges: make([]*types.SlotChanges, 5), + StorageReads: make([]accounts.StorageKey, 3), + }} + tasks, workers := makeBALWarmupPlan(bal, 4) + require.Equal(t, 4, workers) + require.Equal(t, []balWarmupTask{ + {accountIndex: 0, kind: balWarmAccount}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 0}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 1}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 2}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 3}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 4}, + {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 0}, + {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 1}, + {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 2}, + }, tasks) +} + +func TestBlockReadAheaderWarmsOverlayBlockAccessList(t *testing.T) { + oldReadAhead := dbg.ReadAhead + dbg.SetReadAhead(true) + t.Cleanup(func() { dbg.SetReadAhead(oldReadAhead) }) + ctx := t.Context() + dirs := datadir.New(t.TempDir()) + db := temporaltest.NewTestDB(t, dirs) + address := common.Address{19: 0x42} + account := accounts.Account{ + Nonce: 1, + Balance: *uint256.NewInt(1), + CodeHash: accounts.EmptyCodeHash, + } + accountBytes := accounts.SerialiseV3(&account) + rwTx, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx.Rollback() + domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + require.NoError(t, err) + domains.SetTxNum(1) + require.NoError(t, domains.DomainPut(kv.AccountsDomain, rwTx, address[:], accountBytes, 1, nil)) + require.NoError(t, domains.Commit(ctx, rwTx)) + domains.Close() + bal := types.BlockAccessList{{Address: accounts.InternAddress(address)}} + balBytes, err := types.EncodeBlockAccessListBytes(bal) + require.NoError(t, err) + balHash := bal.Hash() + header := &types.Header{ + Number: *uint256.NewInt(1), + BlockAccessListHash: &balHash, + } + body := new(types.Body) + baseTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer baseTx.Rollback() + overlay, err := membatchwithdb.NewMemoryBatch(baseTx, dirs.Tmp, log.New()) + require.NoError(t, err) + require.NoError(t, rawdb.WriteBlockAccessListBytes(overlay, header.Hash(), header.Number.Uint64(), balBytes)) + // The regression requires the BAL to be present only in BlockOverlay. + require.NoError(t, db.View(ctx, func(tx kv.Tx) error { + stored, err := rawdb.ReadBlockAccessListBytes(tx, header.Hash(), header.Number.Uint64()) + require.NoError(t, err) + require.Empty(t, stored) + return nil + })) + stateCache := newTestStateCache() + readAheader := NewBlockReadAheader() + readAheader.SetStateCache(stateCache) + readAheader.AddHeaderAndBody(ctx, db, overlay, header, body) + // AddHeaderAndBody must have copied the sidecar synchronously; its caller may + // release the overlay before the asynchronous state warming completes. + overlay.Close() + baseTx.Rollback() + readAheader.WaitForWarmup(ctx) + got, ok := stateCache.Get(kv.AccountsDomain, address[:]) + require.True(t, ok, "overlay-only BAL account was not warmed") + require.Equal(t, accountBytes, got) +} + // A warmup read-through must never replace a fresher entry an authoritative // writer (the FCU flush cache-apply) has already put: the warmup reads a // pre-flush snapshot, so a laggard Put landing after the flush would pin stale diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 6bf8ed4f6c7..18a872e9c8f 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -461,7 +461,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b }, nil } defer e.semaphore.Release(1) - e.hook.LastNewBlockSeen(blockNumber) // used by eth_syncing e.currentContext.ResetPendingUpdates() e.forkValidator.ClearWithUnwind() @@ -491,7 +490,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if err != nil { return ValidationResult{}, err } - e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) + e.readAheader.AddHeaderAndBody(ctx, e.db, overlay, header, body) currentBlockNumber = rawdb.ReadCurrentBlockNumber(overlay) } else { if err := e.db.View(ctx, func(tx kv.Tx) error { @@ -499,12 +498,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if err != nil { return err } - body, err = e.blockReader.BodyWithTransactions(ctx, tx, blockHash, blockNumber) if err != nil { return err } - e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) + e.readAheader.AddHeaderAndBody(ctx, e.db, tx, header, body) currentBlockNumber = rawdb.ReadCurrentBlockNumber(tx) return nil }); err != nil { @@ -517,14 +515,12 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b ValidationStatus: ExecutionStatusMissingSegment, }, nil } - if math.AbsoluteDifference(*currentBlockNumber, blockNumber) >= e.syncCfg.MaxReorgDepth { return ValidationResult{ ValidationStatus: ExecutionStatusTooFarAway, LatestValidHash: common.Hash{}, }, nil } - // Use the overlay-as-rwTx pattern: the validation pipeline writes through // a fresh BlockOverlay on a new SharedDomains. This mirrors updateForkChoice // (forkchoice.go:239-251) and is required by the parallel exec path — @@ -538,7 +534,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } defer roTx.Rollback() - doms, err := execctx.NewSharedDomains(ctx, roTx, e.logger) if err != nil { return ValidationResult{}, err @@ -547,13 +542,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // forkValidator.sharedDom inside ValidatePayload and later phases close it, // so we Close explicitly only on the early-return error paths below. doms.SetInMemHistoryReads(inMemHistoryReads) - if err := doms.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { doms.Close() return ValidationResult{}, fmt.Errorf("ValidateChain: init block overlay: %w", err) } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain the validation SD to the canonical generation (e.currentContext) for // any payload with a parent, not just head-extending ones: head-extending // payloads read its not-yet-committed domain state instead of stale MDBX, and @@ -563,7 +556,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if e.currentContext != nil { doms.SetParent(e.currentContext) } - // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into // the validation overlay so unwindToCommonCanonical and ValidatePayload — // and the parallel exec goroutine via NewReadView — see this block data. @@ -577,7 +569,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, fmt.Errorf("ValidateChain: flush overlay to validation tx: %w", err) } } - // Set state cache in SharedDomains for use during state reading doms.SetStateCache(e.stateCache) doms.SetCodeStore(e.codeStore) @@ -585,28 +576,23 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b doms.Close() return ValidationResult{}, err } - status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError } - // No cache invalidation needed on an invalid payload: the state cache is // populated only at flush (committed, fork-agnostic state) and this // validation path never flushes, so a rejected payload leaves nothing // fork-specific in the cache. Reads during validation only add canonical // committed bytes. (Cache invalidation happens solely on unwind.) - // Validation tx is the SD's BlockOverlay; defer doms.Close() above handles // its rollback. By design we do not persist validation-run writes — there // is no Flush/Commit on this path. - validationStatus := ExecutionStatusSuccess if status == engine_types.AcceptedStatus { validationStatus = ExecutionStatusMissingSegment } isInvalidChain := status == engine_types.InvalidStatus || status == engine_types.InvalidBlockHashStatus || validationError != nil - // Only open a second tx when we actually need to write (bad-chain purge). // On the valid-chain path (the common case at tip) opening + empty-committing // a second RwTx just produces no-op commits with openTxs>=2, pinning freelist @@ -617,7 +603,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } defer purgeTx.Rollback() - if (lvh != common.Hash{}) && lvh != blockHash { if err := e.purgeBadChain(ctx, purgeTx, lvh, blockHash); err != nil { return ValidationResult{}, err @@ -637,7 +622,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b e.nextForkActivated = true e.logger.Info(nextForkBanner) } - result := ValidationResult{ ValidationStatus: validationStatus, LatestValidHash: lvh, From 7fa8f7a08f5d204d06f4c2e4e789f898bd998a61 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:22:22 +0000 Subject: [PATCH 2/6] fix warmup --- common/dbg/experiments.go | 1 + execution/exec/blocks_read_ahead.go | 7 +++++++ execution/exec/blocks_read_ahead_test.go | 26 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 281cebeccac..3ea7d60803b 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -137,6 +137,7 @@ var ( AssertStateCache = EnvBool("ASSERT_STATE_CACHE", false) ReadAhead = EnvBool("READ_AHEAD", true) ReadAheadWorkers = EnvInt("READ_AHEAD_WORKERS", runtime.NumCPU()) + ReadAheadWait = EnvBool("READ_AHEAD_WAIT", false) // FilesAsyncIO warms cold state .kv pages via io_uring before the mmap read, so // a would-be blocking page fault becomes a non-blocking read that releases the // goroutine's P. Linux + io_uring only; self-disables (reads use ordinary faults) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index b3e7ae24dee..a0ac4b02e74 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -159,6 +159,7 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, t bra.warmWg.Go(func() { bra.warmBody(ctx, db, body, bal, dbg.ReadAheadWorkers) }) + bra.waitForWarmupIfConfigured(ctx) } } @@ -177,6 +178,12 @@ func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) { } } +func (bra *BlockReadAheader) waitForWarmupIfConfigured(ctx context.Context) { + if dbg.ReadAheadWait { + bra.WaitForWarmup(ctx) + } +} + func (bra *BlockReadAheader) AddSenders(senders []byte, blockHash common.Hash) { if _, ok := bra.bodies.Get(blockHash); !ok { return diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 4b339dd3e90..8117a0eb1f3 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -18,6 +18,7 @@ package exec import ( "testing" + "time" "github.com/c2h5oh/datasize" "github.com/holiman/uint256" @@ -60,6 +61,31 @@ func newTestStateCache() *cache.StateCache { return cache.NewStateCache(b, b, b, b) } +func TestBlockReadAheaderWaitsForConfiguredWarmup(t *testing.T) { + oldReadAheadWait := dbg.ReadAheadWait + dbg.ReadAheadWait = true + t.Cleanup(func() { dbg.ReadAheadWait = oldReadAheadWait }) + readAheader := NewBlockReadAheader() + release := make(chan struct{}) + readAheader.warmWg.Go(func() { <-release }) + done := make(chan struct{}) + go func() { + readAheader.waitForWarmupIfConfigured(t.Context()) + close(done) + }() + select { + case <-done: + t.Fatal("configured warmup wait returned before warmup completed") + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("configured warmup wait did not return after warmup completed") + } +} + func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { bal := types.BlockAccessList{{ StorageChanges: make([]*types.SlotChanges, 5), From fbf4aa13c64c709a6badd15ffedd2ca5c2713780 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 7 Aug 2026 06:54:59 +0000 Subject: [PATCH 3/6] fix unique code lookups --- execution/exec/blocks_read_ahead.go | 37 +++++++++++++++++--- execution/exec/blocks_read_ahead_test.go | 44 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index a0ac4b02e74..8ccc8d82782 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -83,10 +83,13 @@ func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { // (codeHash→bytes) + size-cache layers via PutCodeWithHashIfAbsent, keyed by // the code's own keccak hash so every cached pair is self-consistent. type cachePopulatingGetter struct { - g kv.TemporalGetter - sc *cache.StateCache - progress func(kv.Domain) uint64 // domain progress source for stamping negative fills - stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) + g kv.TemporalGetter + sc *cache.StateCache + progress func(kv.Domain) uint64 // domain progress source for stamping negative fills + stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) + lastCodeAddr common.Address + lastCodeHash common.Hash + lastCodeHashKnown bool } func newCachePopulatingGetter(tx kv.TemporalTx, sc *cache.StateCache) *cachePopulatingGetter { @@ -96,6 +99,16 @@ func newCachePopulatingGetter(tx kv.TemporalTx, sc *cache.StateCache) *cachePopu func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.g.GetLatest(name, k) + if name == kv.AccountsDomain { + cpg.lastCodeHashKnown = false + if err == nil && len(k) == len(cpg.lastCodeAddr) { + if codeHash := accounts.DeserialiseV3CodeHash(v); len(codeHash) == len(cpg.lastCodeHash) { + cpg.lastCodeAddr = common.BytesToAddress(k) + copy(cpg.lastCodeHash[:], codeHash) + cpg.lastCodeHashKnown = true + } + } + } if err == nil && cpg.sc != nil { // If-absent writes only: this runs in a fire-and-forget goroutine over a // committed snapshot, so an unconditional Put racing an FCU flush's @@ -129,6 +142,22 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k return v, step, err } +func (cpg *cachePopulatingGetter) GetCode(addr []byte, _ uint64) ([]byte, bool, error) { + // The hash was captured by a preceding read of this account on this + // getter's committed snapshot. It is safe for an immutable content-cache + // probe; a miss still uses the authoritative address-keyed CodeDomain path. + if cpg.sc != nil && cpg.lastCodeHashKnown && bytes.Equal(addr, cpg.lastCodeAddr[:]) { + if code, ok := cpg.sc.GetCodeByHash(cpg.lastCodeHash[:]); ok { + return code, true, nil + } + } + code, _, err := cpg.GetLatest(kv.CodeDomain, addr) + if err != nil { + return nil, false, err + } + return code, len(code) > 0, nil +} + func (cpg *cachePopulatingGetter) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { return cpg.g.HasPrefix(name, prefix) } diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 8117a0eb1f3..75e3d1455b6 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/cache" + "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/execution/types/accounts" ) @@ -46,6 +47,12 @@ type stubTemporalGetter struct { step kv.Step } +type sharedCodeTemporalGetter struct { + account []byte + code []byte + codeReads int +} + func (s stubTemporalGetter) GetLatest(kv.Domain, []byte) ([]byte, kv.Step, error) { return s.v, s.step, nil } @@ -56,6 +63,23 @@ func (s stubTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, func (s stubTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } +func (s *sharedCodeTemporalGetter) GetLatest(domain kv.Domain, _ []byte) ([]byte, kv.Step, error) { + if domain == kv.AccountsDomain { + return s.account, 0, nil + } + if domain == kv.CodeDomain { + s.codeReads++ + return s.code, 0, nil + } + return nil, 0, nil +} + +func (s *sharedCodeTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, error) { + return nil, nil, false, nil +} + +func (s *sharedCodeTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } + func newTestStateCache() *cache.StateCache { b := 1 * datasize.MB return cache.NewStateCache(b, b, b, b) @@ -206,6 +230,26 @@ func TestCachePopulatingGetterKeepsFresherCodeBinding(t *testing.T) { require.Equal(t, freshCode, got, "warmup must not rebind addr to older code") } +func TestCachePopulatingGetterReusesCodeByHashAcrossGetters(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.Account{Nonce: 1, CodeHash: accounts.InternCodeHash(crypto.Keccak256Hash(code))} + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + stateCache := newTestStateCache() + for _, address := range []accounts.Address{ + accounts.InternAddress(common.Address{19: 1}), + accounts.InternAddress(common.Address{19: 2}), + } { + reader := state.NewReaderV3(&cachePopulatingGetter{g: source, sc: stateCache, stepSize: 16}) + gotAccount, err := reader.ReadAccountData(address) + require.NoError(t, err) + require.Equal(t, account.CodeHash, gotAccount.CodeHash) + gotCode, err := reader.ReadAccountCode(address) + require.NoError(t, err) + require.Equal(t, code, gotCode) + } + require.Equal(t, 1, source.codeReads, "identical code must be loaded from the address-keyed domain only once") +} + // Cold keys must still be warmed — that is the prefetcher's purpose. func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") From 529344bf7a38156f81633b49cc42f4e88e8b9630 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 7 Aug 2026 07:39:11 +0000 Subject: [PATCH 4/6] tidy --- execution/exec/blocks_read_ahead.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 8ccc8d82782..2bda4adfc39 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -143,9 +143,10 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k } func (cpg *cachePopulatingGetter) GetCode(addr []byte, _ uint64) ([]byte, bool, error) { - // The hash was captured by a preceding read of this account on this - // getter's committed snapshot. It is safe for an immutable content-cache - // probe; a miss still uses the authoritative address-keyed CodeDomain path. + // A warmup worker calls ReadAccountData immediately before ReadAccountCode. + // The account read provides the code hash, which lets the code read probe the + // code cache before falling back to the database. This avoids repeated database + // reads for accounts sharing identical code. if cpg.sc != nil && cpg.lastCodeHashKnown && bytes.Equal(addr, cpg.lastCodeAddr[:]) { if code, ok := cpg.sc.GetCodeByHash(cpg.lastCodeHash[:]); ok { return code, true, nil From 893236bfc8e384f9c9a90e1373b1126f0e90db30 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:32:22 +0000 Subject: [PATCH 5/6] add bal code vs tx code vs none env vars for code load experiments --- common/dbg/experiments.go | 2 + execution/exec/blocks_read_ahead.go | 64 +++++++++++++++++++++--- execution/exec/blocks_read_ahead_test.go | 63 ++++++++++++++++++++++- 3 files changed, 122 insertions(+), 7 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 3ea7d60803b..ac08dd4b654 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -138,6 +138,8 @@ var ( ReadAhead = EnvBool("READ_AHEAD", true) ReadAheadWorkers = EnvInt("READ_AHEAD_WORKERS", runtime.NumCPU()) ReadAheadWait = EnvBool("READ_AHEAD_WAIT", false) + ReadAheadBALCode = EnvBool("READ_AHEAD_BAL_CODE", false) + ReadAheadTxCode = EnvBool("READ_AHEAD_TX_CODE", true) // FilesAsyncIO warms cold state .kv pages via io_uring before the mmap read, so // a would-be blocking page fault becomes a non-blocking read that releases the // goroutine's P. Linux + io_uring only; self-disables (reads use ordinary faults) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 2bda4adfc39..57defcb956c 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -227,20 +227,41 @@ const ( balWarmAccount balWarmupTaskKind = iota balWarmStorageChanges balWarmStorageReads + balWarmBALCode + balWarmTxnDestination +) + +type balCodeWarmupMode uint8 + +const ( + balCodeWarmupNone balCodeWarmupMode = iota + balCodeWarmupTxnDestinations + balCodeWarmupAll ) type balWarmupTask struct { accountIndex int kind balWarmupTaskKind slotIndex int + address accounts.Address +} + +func balCodeWarmupModeForFlags(warmBALCode, warmTxCode bool) balCodeWarmupMode { + if warmBALCode { + return balCodeWarmupAll + } + if warmTxCode { + return balCodeWarmupTxnDestinations + } + return balCodeWarmupNone } -func makeBALWarmupPlan(bal types.BlockAccessList, workers int) ([]balWarmupTask, int) { +func makeBALWarmupPlan(bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) ([]balWarmupTask, int) { taskCount := len(bal) for _, account := range bal { taskCount += len(account.StorageChanges) + len(account.StorageReads) } - tasks := make([]balWarmupTask, 0, taskCount) + tasks := make([]balWarmupTask, 0, taskCount+len(bal)+len(txns)) for accountIndex, account := range bal { tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmAccount}) for slotIndex := range account.StorageChanges { @@ -250,6 +271,26 @@ func makeBALWarmupPlan(bal types.BlockAccessList, workers int) ([]balWarmupTask, tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmStorageReads, slotIndex: slotIndex}) } } + switch codeMode { + case balCodeWarmupAll: + for accountIndex := range bal { + tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmBALCode}) + } + case balCodeWarmupTxnDestinations: + txnDestinations := make(map[accounts.Address]struct{}, len(txns)) + for _, txn := range txns { + to := txn.GetTo() + if to == nil { + continue + } + address := accounts.InternAddress(*to) + if _, ok := txnDestinations[address]; ok { + continue + } + txnDestinations[address] = struct{}{} + tasks = append(tasks, balWarmupTask{kind: balWarmTxnDestination, address: address}) + } + } return tasks, min(workers, len(tasks)) } @@ -262,14 +303,15 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, body *typ workers = 1 } if len(bal) > 0 { - bra.warmBAL(ctx, db, bal, workers) + codeMode := balCodeWarmupModeForFlags(dbg.ReadAheadBALCode, dbg.ReadAheadTxCode) + bra.warmBAL(ctx, db, bal, body.Transactions, codeMode, workers) return } bra.warmTxns(ctx, db, body.Transactions, workers) } -func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, workers int) { - tasks, balWorkers := makeBALWarmupPlan(bal, workers) +func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) { + tasks, balWorkers := makeBALWarmupPlan(bal, txns, codeMode, workers) var nextTask atomic.Uint64 var wg errgroup.Group for w := range balWorkers { @@ -302,17 +344,27 @@ func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types. break } task := tasks[taskIndex] - acctChanges := bal[task.accountIndex] switch task.kind { case balWarmAccount: + acctChanges := bal[task.accountIndex] + stateReader.ReadAccountData(acctChanges.Address) + case balWarmBALCode: + acctChanges := bal[task.accountIndex] acct, _ := stateReader.ReadAccountData(acctChanges.Address) if (acct != nil && !acct.CodeHash.IsEmpty()) || len(acctChanges.CodeChanges) > 0 { stateReader.ReadAccountCode(acctChanges.Address) } + case balWarmTxnDestination: + acct, _ := stateReader.ReadAccountData(task.address) + if acct != nil && !acct.CodeHash.IsEmpty() { + stateReader.ReadAccountCode(task.address) + } case balWarmStorageChanges: + acctChanges := bal[task.accountIndex] slot := acctChanges.StorageChanges[task.slotIndex].Slot stateReader.ReadAccountStorage(acctChanges.Address, slot) case balWarmStorageReads: + acctChanges := bal[task.accountIndex] slot := acctChanges.StorageReads[task.slotIndex] stateReader.ReadAccountStorage(acctChanges.Address, slot) } diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 75e3d1455b6..be28c28dfa0 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -115,7 +115,7 @@ func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { StorageChanges: make([]*types.SlotChanges, 5), StorageReads: make([]accounts.StorageKey, 3), }} - tasks, workers := makeBALWarmupPlan(bal, 4) + tasks, workers := makeBALWarmupPlan(bal, nil, balCodeWarmupModeForFlags(false, false), 4) require.Equal(t, 4, workers) require.Equal(t, []balWarmupTask{ {accountIndex: 0, kind: balWarmAccount}, @@ -130,6 +130,67 @@ func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { }, tasks) } +func TestBALCodeWarmupModeForFlags(t *testing.T) { + tests := []struct { + name string + warmBALCode bool + warmTxCode bool + want balCodeWarmupMode + }{ + {name: "all BAL code", warmBALCode: true, want: balCodeWarmupAll}, + {name: "BAL code takes precedence", warmBALCode: true, warmTxCode: true, want: balCodeWarmupAll}, + {name: "transaction destinations", warmTxCode: true, want: balCodeWarmupTxnDestinations}, + {name: "no code", want: balCodeWarmupNone}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, balCodeWarmupModeForFlags(test.warmBALCode, test.warmTxCode)) + }) + } +} + +func TestMakeBALWarmupTasksQueuesUniqueTransactionDestinationsAfterStateData(t *testing.T) { + destinationA := common.Address{19: 0xa1} + destinationB := common.Address{19: 0xb2} + balOnly := common.Address{19: 0xc3} + txns := types.Transactions{ + types.NewTransaction(0, destinationA, nil, 0, nil, nil), + types.NewTransaction(1, destinationA, nil, 0, nil, []byte{0x01}), + types.NewContractCreation(2, nil, 0, nil, []byte{0x02}), + types.NewTransaction(3, destinationB, nil, 0, nil, nil), + } + bal := types.BlockAccessList{ + {Address: accounts.InternAddress(destinationA), StorageReads: []accounts.StorageKey{accounts.InternKey(common.Hash{31: 0x01})}}, + {Address: accounts.InternAddress(balOnly)}, + } + tasks, workers := makeBALWarmupPlan(bal, txns, balCodeWarmupModeForFlags(false, true), 4) + require.Equal(t, 4, workers) + require.Equal(t, []balWarmupTask{ + {accountIndex: 0, kind: balWarmAccount}, + {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 0}, + {accountIndex: 1, kind: balWarmAccount}, + {kind: balWarmTxnDestination, address: accounts.InternAddress(destinationA)}, + {kind: balWarmTxnDestination, address: accounts.InternAddress(destinationB)}, + }, tasks) +} + +func TestMakeBALWarmupTasksQueuesAllCodeAfterStateData(t *testing.T) { + bal := types.BlockAccessList{ + {StorageChanges: make([]*types.SlotChanges, 1)}, + {StorageReads: make([]accounts.StorageKey, 1)}, + } + tasks, workers := makeBALWarmupPlan(bal, nil, balCodeWarmupModeForFlags(true, false), 4) + require.Equal(t, 4, workers) + require.Equal(t, []balWarmupTask{ + {accountIndex: 0, kind: balWarmAccount}, + {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 0}, + {accountIndex: 1, kind: balWarmAccount}, + {accountIndex: 1, kind: balWarmStorageReads, slotIndex: 0}, + {accountIndex: 0, kind: balWarmBALCode}, + {accountIndex: 1, kind: balWarmBALCode}, + }, tasks) +} + func TestBlockReadAheaderWarmsOverlayBlockAccessList(t *testing.T) { oldReadAhead := dbg.ReadAhead dbg.SetReadAhead(true) From 6eb6de572a00f4bc513ef2a44aad774d27c7dea1 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:31:00 +0000 Subject: [PATCH 6/6] address code review findings --- execution/cache/cache_test.go | 38 +++ execution/cache/code_cache.go | 14 +- execution/cache/code_cache_codehash_test.go | 11 + execution/cache/state_cache.go | 22 +- execution/cache/view.go | 41 +++ execution/exec/blocks_read_ahead.go | 286 +++++++++--------- execution/exec/blocks_read_ahead_test.go | 225 +++++++++----- execution/stagedsync/exec3.go | 7 + execution/stagedsync/exec3_read_ahead_test.go | 20 ++ 9 files changed, 446 insertions(+), 218 deletions(-) create mode 100644 execution/stagedsync/exec3_read_ahead_test.go diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 42e0e740eab..f45cf76d355 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -1159,6 +1159,44 @@ func TestStateCache_AccountDeletionGatesStaleCodeFill(t *testing.T) { require.True(t, ok, "unrelated code fills from a current view must stay admitted") } +func TestStateCache_CodeHashHitBindsAddress(t *testing.T) { + b := 1 * datasize.MB + c := NewStateCache(b, b, b, b) + t.Cleanup(c.Close) + firstAddr, secondAddr, code := makeAddr(1), makeAddr(2), makeCode(1) + codeHash := crypto.Keccak256Hash(code) + view := c.View(FrontierFunc(func(kv.Domain) (uint64, bool) { return 100, true })) + view.Fill(kv.CodeDomain, firstAddr, code, 10) + view.SeedAddrCodeHash(secondAddr, codeHash, 11) + _, ok := c.View(nil).Get(kv.CodeDomain, secondAddr) + require.False(t, ok, "the second address must start without an addr-keyed code binding") + got, ok := view.GetCodeByAddressHash(secondAddr) + require.True(t, ok) + require.Equal(t, code, got) + got, ok = c.View(nil).Get(kv.CodeDomain, secondAddr) + require.True(t, ok, "the hash hit must populate the addr-keyed code binding") + require.Equal(t, code, got) + c.Applier().Unwind(11) + _, ok = c.View(nil).Get(kv.CodeDomain, secondAddr) + require.False(t, ok, "the derived binding must keep the mapping stamp for unwind invalidation") +} + +func TestStateCache_EmptyCodeHashUsesViewFrontierStamp(t *testing.T) { + b := 1 * datasize.MB + c := NewStateCache(b, b, b, b) + t.Cleanup(c.Close) + addr := makeAddr(1) + view := c.View(FrontierFunc(func(kv.Domain) (uint64, bool) { return 100, true })) + view.SeedAddrCodeHash(addr, [32]byte{}, 4) + codeHash, txNum, ok := c.getAddrCodeHashWithTxNum(addr) + require.True(t, ok) + require.Zero(t, codeHash) + require.Equal(t, uint64(99), txNum, "a negative mapping reflects the view, not a nonexistent value's reported step") + c.Applier().Unwind(99) + _, _, ok = c.getAddrCodeHashWithTxNum(addr) + require.False(t, ok) +} + // An apply-only cache (STATE_CACHE_FILLS=false) has no fill for a lowered // frontier to poison; wire-up code keys the aggregator forbid on this. func TestApplyOnlyCacheReportsFillsDisabled(t *testing.T) { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 45f7297113e..78a103d3a22 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -364,17 +364,25 @@ func (c *CodeCache) putCodeLocked(addr []byte, code []byte, keyHash [32]byte, co // EVM-known codeHash is already known. Eviction is LRU; freshly seen addrs // replace coldest entries. func (c *CodeCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + h, _, ok := c.GetAddrCodeHashWithTxNum(addr) + return h, ok +} + +// GetAddrCodeHashWithTxNum is GetAddrCodeHash plus the txNum of the +// addr→codeHash mapping. A caller that uses the hash to bind cached code to +// the address must preserve this stamp so unwind invalidation remains correct. +func (c *CodeCache) GetAddrCodeHashWithTxNum(addr []byte) ([32]byte, uint64, bool) { k := common.BytesToAddress(addr) coh := c.coh.Snapshot() e, ok := c.addrToCodeHash.Get(k) if !ok { - return [32]byte{}, false + return [32]byte{}, 0, false } if coh.IsStale(e.txNum, e.epoch) { c.addrToCodeHash.Remove(k) - return [32]byte{}, false + return [32]byte{}, 0, false } - return e.hash, true + return e.hash, e.txNum, true } // PutAddrCodeHash records a committed-state addr → codeHash mapping. An diff --git a/execution/cache/code_cache_codehash_test.go b/execution/cache/code_cache_codehash_test.go index 6572b99ab26..cdea474386b 100644 --- a/execution/cache/code_cache_codehash_test.go +++ b/execution/cache/code_cache_codehash_test.go @@ -62,6 +62,17 @@ func TestCodeCache_PutAddrCodeHashReplacesStaleEntry(t *testing.T) { require.Equal(t, newHash, got) } +func TestCodeCache_GetAddrCodeHashWithTxNumPreservesStamp(t *testing.T) { + c := closeOnCleanup(t, NewCodeCache(1*datasize.MB, 1*datasize.MB)) + addr := makeAddr(1) + codeHash := [32]byte{1} + c.PutAddrCodeHash(addr, codeHash, 42) + gotHash, gotTxNum, ok := c.GetAddrCodeHashWithTxNum(addr) + require.True(t, ok) + require.Equal(t, codeHash, gotHash) + require.Equal(t, uint64(42), gotTxNum) +} + func TestCodeCache_GetByCodeHash_HitAfterPut(t *testing.T) { c := closeOnCleanup(t, NewCodeCache(1*datasize.MB, 1*datasize.MB)) addr := makeAddr(1) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 276871b07cc..ace5fbb9514 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -24,6 +24,7 @@ import ( "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" @@ -204,6 +205,14 @@ func (c *StateCache) getAddrCodeHash(addr []byte) ([32]byte, bool) { return cc.GetAddrCodeHash(addr) } +func (c *StateCache) getAddrCodeHashWithTxNum(addr []byte) ([32]byte, uint64, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return [32]byte{}, 0, false + } + return cc.GetAddrCodeHashWithTxNum(addr) +} + // seedAddrCodeHash conditionally records an addr → codeHash mapping. // The mapping derives from an account record, so admission checks the accounts // frontier even though the mapping lives in the code cache. @@ -271,18 +280,25 @@ func (c *StateCache) fillIfFresh(domain kv.Domain, key []byte, value []byte, rea // negatives are not cached here: "no code" is cached at the addr→codeHash // mapping instead (the zero-hash sentinel seeded by SeedAddrCodeHash). func (c *StateCache) fillCodeIfFresh(key []byte, value []byte, readTxNum, visibleEnd, accountsVisibleEnd uint64) { - codeCache, ok := c.caches[kv.CodeDomain].(*CodeCache) - if !ok || len(value) == 0 { + if len(value) == 0 { return } codeHash := crypto.Keccak256(value) cloned := bytes.Clone(value) + c.fillCodeWithHashIfFresh(key, cloned, codeHash, readTxNum, visibleEnd, accountsVisibleEnd) +} + +func (c *StateCache) fillCodeWithHashIfFresh(key, value, codeHash []byte, readTxNum, visibleEnd, accountsVisibleEnd uint64) { + codeCache, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok || len(value) == 0 || len(codeHash) != len(common.Hash{}) { + return + } c.admissionMu.RLock() defer c.admissionMu.RUnlock() if visibleEnd < c.appliedEnd[kv.CodeDomain] || accountsVisibleEnd < c.appliedEnd[kv.AccountsDomain] { return } - codeCache.PutWithCodeHashIfAbsent(key, cloned, codeHash, readTxNum) + codeCache.PutWithCodeHashIfAbsent(key, value, codeHash, readTxNum) } // deleteKey removes the data for the given domain and key. Authoritative diff --git a/execution/cache/view.go b/execution/cache/view.go index 0de781920d6..b375bb0937d 100644 --- a/execution/cache/view.go +++ b/execution/cache/view.go @@ -105,6 +105,41 @@ func (v ReadView) GetAddrCodeHash(addr []byte) ([32]byte, bool) { return v.c.getAddrCodeHash(addr) } +// GetCodeByAddressHash resolves the durable addr→codeHash mapping and then +// probes the content-addressed code cache. On a hit it also fills the +// addr-keyed code binding with the mapping's original txNum, avoiding both a +// database read and a redundant keccak of the cached code. +func (v ReadView) GetCodeByAddressHash(addr []byte) ([]byte, bool) { + if v.c == nil { + return nil, false + } + codeHash, readTxNum, ok := v.c.getAddrCodeHashWithTxNum(addr) + if !ok || codeHash == ([32]byte{}) { + return nil, false + } + code, ok := v.c.getCodeByHash(codeHash[:]) + if !ok { + return nil, false + } + v.fillCodeWithHash(addr, code, codeHash[:], readTxNum) + return code, true +} + +func (v ReadView) fillCodeWithHash(addr, code, codeHash []byte, readTxNum uint64) { + if v.c == nil || v.c.disableFills || v.frontier == nil { + return + } + visibleEnd, ok := v.frontier.DomainVisibleEnd(kv.CodeDomain) + if !ok { + return + } + accountsEnd, ok := v.frontier.DomainVisibleEnd(kv.AccountsDomain) + if !ok { + return + } + v.c.fillCodeWithHashIfFresh(addr, code, codeHash, readTxNum, visibleEnd, accountsEnd) +} + // CanFill reports whether this view carries a frontier, i.e. Fill and // SeedAddrCodeHash can admit values through it. func (v ReadView) CanFill() bool { return v.c != nil && v.frontier != nil } @@ -145,6 +180,12 @@ func (v ReadView) SeedAddrCodeHash(addr []byte, h [32]byte, txNum uint64) { if !ok { return } + if h == ([32]byte{}) { + txNum = 0 + if visibleEnd > 0 { + txNum = visibleEnd - 1 + } + } v.c.seedAddrCodeHash(addr, h, txNum, visibleEnd) } diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 39f5af7db5e..a16f415a65a 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -3,7 +3,7 @@ package exec import ( "bytes" "context" - "sync" + "errors" "sync/atomic" "time" @@ -32,8 +32,7 @@ type BlockReadAheader struct { bals *lru.Cache[common.Hash, []byte] // this is for warming state - warming atomic.Bool // only one warmBody can run at a time - warmWg sync.WaitGroup + warmDone chan struct{} // contains one token while no warmBody is running // stateCache is the process-global state cache that SharedDomains.GetLatest // consults on the EVM hot path. When set, warmBody routes its prefetches @@ -61,11 +60,14 @@ func NewBlockReadAheader() *BlockReadAheader { if err != nil { panic(err) } + warmDone := make(chan struct{}, 1) + warmDone <- struct{}{} return &BlockReadAheader{ - headers: headers, - bodies: bodies, - senders: senders, - bals: bals, + headers: headers, + bodies: bodies, + senders: senders, + bals: bals, + warmDone: warmDone, } } @@ -87,11 +89,8 @@ func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { // Code reads also populate the content-addressed and size-cache layers. type cachePopulatingGetter struct { kv.TemporalGetter - view cache.ReadView - stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) - lastCodeAddr common.Address - lastCodeHash common.Hash - lastCodeHashKnown bool + view cache.ReadView + stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) } func readAheadGetter(ttx kv.TemporalTx, sc *cache.StateCache) kv.TemporalGetter { @@ -104,32 +103,21 @@ func readAheadGetter(ttx kv.TemporalTx, sc *cache.StateCache) kv.TemporalGetter func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.TemporalGetter.GetLatest(name, k) - if name == kv.AccountsDomain { - cpg.lastCodeHashKnown = false - if err == nil && len(k) == len(cpg.lastCodeAddr) { - if codeHash := accounts.DeserialiseV3CodeHash(v); len(codeHash) == len(cpg.lastCodeHash) { - cpg.lastCodeAddr = common.BytesToAddress(k) - copy(cpg.lastCodeHash[:], codeHash) - cpg.lastCodeHashKnown = true - } - } - } if err == nil { readTxNum := (uint64(step)+1)*cpg.stepSize - 1 cpg.view.Fill(name, k, v, readTxNum) + if name == kv.AccountsDomain { + var codeHash common.Hash + copy(codeHash[:], accounts.DeserialiseV3CodeHash(v)) + cpg.view.SeedAddrCodeHash(k, codeHash, readTxNum) + } } return v, step, err } func (cpg *cachePopulatingGetter) GetCode(addr []byte, _ uint64) ([]byte, bool, error) { - // A warmup worker calls ReadAccountData immediately before ReadAccountCode. - // The account read provides the code hash, which lets the code read probe the - // code cache before falling back to the database. This avoids repeated database - // reads for accounts sharing identical code. - if cpg.lastCodeHashKnown && bytes.Equal(addr, cpg.lastCodeAddr[:]) { - if code, ok := cpg.view.GetCodeByHash(cpg.lastCodeHash[:]); ok { - return code, true, nil - } + if code, ok := cpg.view.GetCodeByAddressHash(addr); ok { + return code, true, nil } code, _, err := cpg.GetLatest(kv.CodeDomain, addr) if err != nil { @@ -148,8 +136,9 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, t if db == nil || ctx == nil || !dbg.ReadAhead { return } - // Only allow one warmBody to run at a time - if !bra.warming.CompareAndSwap(false, true) { + select { + case <-bra.warmDone: + default: return } var balBytes []byte @@ -165,33 +154,23 @@ func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, t } } } - bra.warmWg.Go(func() { + go func() { + defer func() { bra.warmDone <- struct{}{} }() bra.warmBody(ctx, db, header, body, balBytes, dbg.ReadAheadWorkers) - }) - bra.waitForWarmupIfConfigured(ctx) + }() } // WaitForWarmup blocks until any in-flight warmBody goroutine finishes or // the context is cancelled. Call before closing the database to avoid // waitTxsAllDoneOnClose hangs. func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) { - done := make(chan struct{}) - go func() { - bra.warmWg.Wait() - close(done) - }() select { - case <-done: + case <-bra.warmDone: + bra.warmDone <- struct{}{} case <-ctx.Done(): } } -func (bra *BlockReadAheader) waitForWarmupIfConfigured(ctx context.Context) { - if dbg.ReadAheadWait { - bra.WaitForWarmup(ctx) - } -} - func (bra *BlockReadAheader) AddSenders(senders []byte, blockHash common.Hash) { if _, ok := bra.bodies.Get(blockHash); !ok { return @@ -206,15 +185,7 @@ func (bra *BlockReadAheader) AddBlockAccessList(blockHash common.Hash, bal []byt bra.bals.Add(blockHash, bal) } -type balWarmupTaskKind uint8 - -const ( - balWarmAccount balWarmupTaskKind = iota - balWarmStorageChanges - balWarmStorageReads - balWarmBALCode - balWarmTxnDestination -) +const balWarmupStorageChunkSize = 64 type balCodeWarmupMode uint8 @@ -225,10 +196,9 @@ const ( ) type balWarmupTask struct { - accountIndex int - kind balWarmupTaskKind - slotIndex int - address accounts.Address + accountIndex uint32 + slotFrom uint32 + slotTo uint32 } func balCodeWarmupModeForFlags(warmBALCode, warmTxCode bool) balCodeWarmupMode { @@ -241,46 +211,77 @@ func balCodeWarmupModeForFlags(warmBALCode, warmTxCode bool) balCodeWarmupMode { return balCodeWarmupNone } -func makeBALWarmupPlan(bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) ([]balWarmupTask, int) { - taskCount := len(bal) +func makeBALWarmupPlan(bal types.BlockAccessList, workers int) ([]balWarmupTask, int) { + taskCount := 0 for _, account := range bal { - taskCount += len(account.StorageChanges) + len(account.StorageReads) + slots := len(account.StorageChanges) + len(account.StorageReads) + taskCount += max(1, (slots+balWarmupStorageChunkSize-1)/balWarmupStorageChunkSize) } - tasks := make([]balWarmupTask, 0, taskCount+len(bal)+len(txns)) + tasks := make([]balWarmupTask, 0, taskCount) for accountIndex, account := range bal { - tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmAccount}) - for slotIndex := range account.StorageChanges { - tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmStorageChanges, slotIndex: slotIndex}) + slots := len(account.StorageChanges) + len(account.StorageReads) + if slots == 0 { + tasks = append(tasks, balWarmupTask{accountIndex: uint32(accountIndex)}) + continue } - for slotIndex := range account.StorageReads { - tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmStorageReads, slotIndex: slotIndex}) + for slotFrom := 0; slotFrom < slots; slotFrom += balWarmupStorageChunkSize { + tasks = append(tasks, balWarmupTask{accountIndex: uint32(accountIndex), slotFrom: uint32(slotFrom), slotTo: uint32(min(slotFrom+balWarmupStorageChunkSize, slots))}) } } - switch codeMode { - case balCodeWarmupAll: - for accountIndex := range bal { - tasks = append(tasks, balWarmupTask{accountIndex: accountIndex, kind: balWarmBALCode}) + return tasks, min(workers, len(tasks)) +} + +func uniqueTransactionDestinations(txns types.Transactions) map[accounts.Address]struct{} { + destinations := make(map[accounts.Address]struct{}, len(txns)) + for _, txn := range txns { + to := txn.GetTo() + if to == nil { + continue } - case balCodeWarmupTxnDestinations: - txnDestinations := make(map[accounts.Address]struct{}, len(txns)) - for _, txn := range txns { - to := txn.GetTo() - if to == nil { - continue - } - address := accounts.InternAddress(*to) - if _, ok := txnDestinations[address]; ok { - continue - } - txnDestinations[address] = struct{}{} - tasks = append(tasks, balWarmupTask{kind: balWarmTxnDestination, address: address}) + address := accounts.InternAddress(*to) + destinations[address] = struct{}{} + } + return destinations +} + +func warmBALStateTask(stateReader *state.ReaderV3, account *types.AccountChanges, task balWarmupTask, codeMode balCodeWarmupMode, txCodeDestinations map[accounts.Address]struct{}) error { + var accountData *accounts.Account + if task.slotFrom == 0 { + var err error + accountData, err = stateReader.ReadAccountData(account.Address) + if err != nil { + return err } } - return tasks, min(workers, len(tasks)) + storageChanges := uint32(len(account.StorageChanges)) + for slotIndex := task.slotFrom; slotIndex < task.slotTo; slotIndex++ { + var slot accounts.StorageKey + if slotIndex < storageChanges { + slot = account.StorageChanges[slotIndex].Slot + } else { + slot = account.StorageReads[slotIndex-storageChanges] + } + if _, _, err := stateReader.ReadAccountStorage(account.Address, slot); err != nil { + return err + } + } + if task.slotFrom != 0 || codeMode == balCodeWarmupNone { + return nil + } + warmCode := false + if codeMode == balCodeWarmupAll { + warmCode = len(account.CodeChanges) > 0 || (accountData != nil && !accountData.CodeHash.IsEmpty()) + } else if _, ok := txCodeDestinations[account.Address]; ok { + warmCode = accountData != nil && !accountData.CodeHash.IsEmpty() + } + if warmCode { + _, err := stateReader.ReadAccountCode(account.Address) + return err + } + return nil } func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body, balBytes []byte, workers int) { - defer bra.warming.Store(false) if !dbg.ReadAhead { return } @@ -297,35 +298,46 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t } if len(bal) > 0 { codeMode := balCodeWarmupModeForFlags(dbg.ReadAheadBALCode, dbg.ReadAheadTxCode) - bra.warmBAL(ctx, db, bal, body.Transactions, codeMode, workers) + if err := bra.warmBAL(ctx, db, bal, body.Transactions, codeMode, workers); err != nil && !errors.Is(err, context.Canceled) { + log.Warn("[warmBAL] failed", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } return } - bra.warmTxns(ctx, db, body.Transactions, workers) + if err := bra.warmTxns(ctx, db, body.Transactions, workers); err != nil && !errors.Is(err, context.Canceled) { + log.Warn("[warmTxns] failed", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } +} + +func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) error { + var txCodeDestinations map[accounts.Address]struct{} + if codeMode == balCodeWarmupTxnDestinations { + txCodeDestinations = uniqueTransactionDestinations(txns) + } + tasks, balWorkers := makeBALWarmupPlan(bal, workers) + return bra.warmBALState(ctx, db, bal, tasks, codeMode, txCodeDestinations, balWorkers) } -func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) { - tasks, balWorkers := makeBALWarmupPlan(bal, txns, codeMode, workers) +func (bra *BlockReadAheader) warmBALState(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, tasks []balWarmupTask, codeMode balCodeWarmupMode, txCodeDestinations map[accounts.Address]struct{}, workers int) error { var nextTask atomic.Uint64 - var wg errgroup.Group - for w := range balWorkers { - workerID := w + wg, workerCtx := errgroup.WithContext(ctx) + for w := range workers { wg.Go(func() error { startTime := time.Now() - tx, err := db.BeginRo(ctx) + tx, err := db.BeginRo(workerCtx) if err != nil { return err } defer tx.Rollback() ttx, ok := tx.(kv.TemporalTx) if !ok { - return nil + return errors.New("BAL warmup requires a temporal read transaction") } stateReader := state.NewReaderV3(readAheadGetter(ttx, bra.stateCache)) tasksProcessed := 0 for { select { - case <-ctx.Done(): - return ctx.Err() + case <-workerCtx.Done(): + return workerCtx.Err() default: } taskIndex := int(nextTask.Add(1) - 1) @@ -333,42 +345,22 @@ func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types. break } task := tasks[taskIndex] - switch task.kind { - case balWarmAccount: - acctChanges := bal[task.accountIndex] - stateReader.ReadAccountData(acctChanges.Address) - case balWarmBALCode: - acctChanges := bal[task.accountIndex] - acct, _ := stateReader.ReadAccountData(acctChanges.Address) - if (acct != nil && !acct.CodeHash.IsEmpty()) || len(acctChanges.CodeChanges) > 0 { - stateReader.ReadAccountCode(acctChanges.Address) - } - case balWarmTxnDestination: - acct, _ := stateReader.ReadAccountData(task.address) - if acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(task.address) - } - case balWarmStorageChanges: - acctChanges := bal[task.accountIndex] - slot := acctChanges.StorageChanges[task.slotIndex].Slot - stateReader.ReadAccountStorage(acctChanges.Address, slot) - case balWarmStorageReads: - acctChanges := bal[task.accountIndex] - slot := acctChanges.StorageReads[task.slotIndex] - stateReader.ReadAccountStorage(acctChanges.Address, slot) + account := bal[task.accountIndex] + if err := warmBALStateTask(stateReader, account, task, codeMode, txCodeDestinations); err != nil { + return err } tasksProcessed++ } - log.Debug("[warmBAL] worker finished", "worker", workerID, "tasks", tasksProcessed, "elapsed", time.Since(startTime)) + log.Debug("[warmBAL] state worker finished", "worker", w, "tasks", tasksProcessed, "elapsed", time.Since(startTime)) return nil }) } - wg.Wait() + return wg.Wait() } -func (bra *BlockReadAheader) warmTxns(ctx context.Context, db kv.RoDB, txns types.Transactions, workers int) { +func (bra *BlockReadAheader) warmTxns(ctx context.Context, db kv.RoDB, txns types.Transactions, workers int) error { if len(txns) == 0 { - return + return nil } txnLen := len(txns) if workers > txnLen { @@ -376,57 +368,69 @@ func (bra *BlockReadAheader) warmTxns(ctx context.Context, db kv.RoDB, txns type } // Pre-divide work: each worker gets a dedicated range of transactions txnsPerWorker := (txnLen + workers - 1) / workers - var wg errgroup.Group + wg, workerCtx := errgroup.WithContext(ctx) for w := 0; w < workers; w++ { start := w * txnsPerWorker end := min(start+txnsPerWorker, txnLen) if start >= txnLen { break } - // Capture loop variables for closure - workerStart, workerEnd, workerID := start, end, w wg.Go(func() error { startTime := time.Now() - tx, err := db.BeginRo(ctx) + tx, err := db.BeginRo(workerCtx) if err != nil { return err } defer tx.Rollback() ttx, ok := tx.(kv.TemporalTx) if !ok { - return nil + return errors.New("transaction warmup requires a temporal read transaction") } stateReader := state.NewReaderV3(readAheadGetter(ttx, bra.stateCache)) - for txIdx := workerStart; txIdx < workerEnd; txIdx++ { + for txIdx := start; txIdx < end; txIdx++ { select { - case <-ctx.Done(): - return ctx.Err() + case <-workerCtx.Done(): + return workerCtx.Err() default: } txn := txns[txIdx] // Warm To account and its code if it has one if toAddr := txn.GetTo(); toAddr != nil { to := accounts.InternAddress(*toAddr) - if acct, _ := stateReader.ReadAccountData(to); acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(to) + acct, err := stateReader.ReadAccountData(to) + if err != nil { + return err + } + if acct != nil && !acct.CodeHash.IsEmpty() { + if _, err := stateReader.ReadAccountCode(to); err != nil { + return err + } } } // Warm transaction access list accounts and their code for _, entry := range txn.GetAccessList() { addr := accounts.InternAddress(entry.Address) - if acct, _ := stateReader.ReadAccountData(addr); acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(addr) + acct, err := stateReader.ReadAccountData(addr) + if err != nil { + return err + } + if acct != nil && !acct.CodeHash.IsEmpty() { + if _, err := stateReader.ReadAccountCode(addr); err != nil { + return err + } } for _, slot := range entry.StorageKeys { - stateReader.ReadAccountStorage(addr, accounts.InternKey(slot)) + if _, _, err := stateReader.ReadAccountStorage(addr, accounts.InternKey(slot)); err != nil { + return err + } } } } - log.Debug("[warmTxns] worker finished", "worker", workerID, "txns", workerEnd-workerStart, "elapsed", time.Since(startTime)) + log.Debug("[warmTxns] worker finished", "worker", w, "txns", end-start, "elapsed", time.Since(startTime)) return nil }) } - wg.Wait() + return wg.Wait() } func (bra *BlockReadAheader) ReadBodyWithTransactions(blockHash common.Hash) (*types.Body, bool) { diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index d62e9d6d8a1..8c2c83fbe4d 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -49,9 +49,10 @@ type stubTemporalGetter struct { } type sharedCodeTemporalGetter struct { - account []byte - code []byte - codeReads int + account []byte + code []byte + accountReads int + codeReads int } type countingGetter struct { @@ -76,6 +77,7 @@ func (s stubTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } func (s *sharedCodeTemporalGetter) GetLatest(domain kv.Domain, _ []byte) ([]byte, kv.Step, error) { if domain == kv.AccountsDomain { + s.accountReads++ return s.account, 0, nil } if domain == kv.CodeDomain { @@ -96,48 +98,48 @@ func newTestStateCache() *cache.StateCache { return cache.NewStateCache(b, b, b, b) } -func TestBlockReadAheaderWaitsForConfiguredWarmup(t *testing.T) { - oldReadAheadWait := dbg.ReadAheadWait - dbg.ReadAheadWait = true - t.Cleanup(func() { dbg.ReadAheadWait = oldReadAheadWait }) +func TestBlockReadAheaderWaitForWarmup(t *testing.T) { readAheader := NewBlockReadAheader() - release := make(chan struct{}) - readAheader.warmWg.Go(func() { <-release }) - done := make(chan struct{}) - go func() { - readAheader.waitForWarmupIfConfigured(t.Context()) - close(done) - }() - select { - case <-done: - t.Fatal("configured warmup wait returned before warmup completed") - case <-time.After(50 * time.Millisecond): - } - close(release) - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("configured warmup wait did not return after warmup completed") + for range 2 { + <-readAheader.warmDone + done := make(chan struct{}) + go func() { + readAheader.WaitForWarmup(t.Context()) + close(done) + }() + select { + case <-done: + t.Fatal("warmup wait returned before warmup completed") + case <-time.After(50 * time.Millisecond): + } + readAheader.warmDone <- struct{}{} + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("warmup wait did not return after warmup completed") + } } } +func TestBlockReadAheaderIgnoresMissingHeaderOrBody(t *testing.T) { + readAheader := NewBlockReadAheader() + header := &types.Header{Number: *uint256.NewInt(1)} + require.NotPanics(t, func() { readAheader.AddHeaderAndBody(t.Context(), nil, nil, nil, new(types.Body)) }) + require.NotPanics(t, func() { readAheader.AddHeaderAndBody(t.Context(), nil, nil, header, nil) }) + require.Zero(t, readAheader.headers.Len()) + require.Zero(t, readAheader.bodies.Len()) +} + func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { bal := types.BlockAccessList{{ - StorageChanges: make([]*types.SlotChanges, 5), + StorageChanges: make([]*types.SlotChanges, 65), StorageReads: make([]accounts.StorageKey, 3), }} - tasks, workers := makeBALWarmupPlan(bal, nil, balCodeWarmupModeForFlags(false, false), 4) - require.Equal(t, 4, workers) + tasks, workers := makeBALWarmupPlan(bal, 4) + require.Equal(t, 2, workers) require.Equal(t, []balWarmupTask{ - {accountIndex: 0, kind: balWarmAccount}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 0}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 1}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 2}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 3}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 4}, - {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 0}, - {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 1}, - {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 2}, + {accountIndex: 0, slotFrom: 0, slotTo: 64}, + {accountIndex: 0, slotFrom: 64, slotTo: 68}, }, tasks) } @@ -160,48 +162,98 @@ func TestBALCodeWarmupModeForFlags(t *testing.T) { } } -func TestMakeBALWarmupTasksQueuesUniqueTransactionDestinationsAfterStateData(t *testing.T) { +func TestUniqueTransactionDestinations(t *testing.T) { destinationA := common.Address{19: 0xa1} destinationB := common.Address{19: 0xb2} - balOnly := common.Address{19: 0xc3} txns := types.Transactions{ types.NewTransaction(0, destinationA, nil, 0, nil, nil), types.NewTransaction(1, destinationA, nil, 0, nil, []byte{0x01}), types.NewContractCreation(2, nil, 0, nil, []byte{0x02}), types.NewTransaction(3, destinationB, nil, 0, nil, nil), } - bal := types.BlockAccessList{ - {Address: accounts.InternAddress(destinationA), StorageReads: []accounts.StorageKey{accounts.InternKey(common.Hash{31: 0x01})}}, - {Address: accounts.InternAddress(balOnly)}, + require.Equal(t, map[accounts.Address]struct{}{accounts.InternAddress(destinationA): {}, accounts.InternAddress(destinationB): {}}, uniqueTransactionDestinations(txns)) +} + +func TestWarmBALStateTaskLoadsSelectedCode(t *testing.T) { + for _, test := range []struct { + name string + mode balCodeWarmupMode + destination bool + contract bool + codeChanges bool + wantCodeReads int + }{ + {name: "transaction destination contract", mode: balCodeWarmupTxnDestinations, destination: true, contract: true, wantCodeReads: 1}, + {name: "transaction non-destination contract", mode: balCodeWarmupTxnDestinations, contract: true}, + {name: "transaction destination EOA", mode: balCodeWarmupTxnDestinations, destination: true}, + {name: "all contract", mode: balCodeWarmupAll, contract: true, wantCodeReads: 1}, + {name: "all EOA", mode: balCodeWarmupAll}, + {name: "all forced code change", mode: balCodeWarmupAll, codeChanges: true, wantCodeReads: 1}, + {name: "none", destination: true, contract: true}, + } { + t.Run(test.name, func(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.NewAccount() + if test.contract { + account.CodeHash = accounts.InternCodeHash(crypto.Keccak256Hash(code)) + } + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + stateCache := newTestStateCache() + t.Cleanup(stateCache.Close) + frontier := cache.FrontierFunc(func(kv.Domain) (uint64, bool) { return 16, true }) + address := accounts.InternAddress(common.Address{19: 1}) + destinations := make(map[accounts.Address]struct{}) + if test.destination { + destinations[address] = struct{}{} + } + accountChanges := &types.AccountChanges{Address: address} + if test.codeChanges { + accountChanges.CodeChanges = []*types.CodeChange{{Bytecode: code}} + } + getter := &cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16} + reader := state.NewReaderV3(getter) + require.NoError(t, warmBALStateTask(reader, accountChanges, balWarmupTask{}, test.mode, destinations)) + require.Equal(t, 1, source.accountReads) + require.Equal(t, test.wantCodeReads, source.codeReads) + }) } - tasks, workers := makeBALWarmupPlan(bal, txns, balCodeWarmupModeForFlags(false, true), 4) - require.Equal(t, 4, workers) - require.Equal(t, []balWarmupTask{ - {accountIndex: 0, kind: balWarmAccount}, - {accountIndex: 0, kind: balWarmStorageReads, slotIndex: 0}, - {accountIndex: 1, kind: balWarmAccount}, - {kind: balWarmTxnDestination, address: accounts.InternAddress(destinationA)}, - {kind: balWarmTxnDestination, address: accounts.InternAddress(destinationB)}, - }, tasks) } -func TestMakeBALWarmupTasksQueuesAllCodeAfterStateData(t *testing.T) { +func TestWarmBALStateTaskDoesNotRepeatCodeForLaterChunks(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.NewAccount() + account.CodeHash = accounts.InternCodeHash(crypto.Keccak256Hash(code)) + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + address := accounts.InternAddress(common.Address{19: 1}) + accountChanges := &types.AccountChanges{Address: address, StorageReads: make([]accounts.StorageKey, 65)} + reader := state.NewReaderV3(source) + require.NoError(t, warmBALStateTask(reader, accountChanges, balWarmupTask{slotFrom: 64, slotTo: 65}, balCodeWarmupAll, nil)) + require.Zero(t, source.accountReads) + require.Zero(t, source.codeReads) +} + +func TestMakeBALWarmupTasksKeepsSmallAccountsTogether(t *testing.T) { bal := types.BlockAccessList{ {StorageChanges: make([]*types.SlotChanges, 1)}, {StorageReads: make([]accounts.StorageKey, 1)}, } - tasks, workers := makeBALWarmupPlan(bal, nil, balCodeWarmupModeForFlags(true, false), 4) - require.Equal(t, 4, workers) + tasks, workers := makeBALWarmupPlan(bal, 4) + require.Equal(t, 2, workers) require.Equal(t, []balWarmupTask{ - {accountIndex: 0, kind: balWarmAccount}, - {accountIndex: 0, kind: balWarmStorageChanges, slotIndex: 0}, - {accountIndex: 1, kind: balWarmAccount}, - {accountIndex: 1, kind: balWarmStorageReads, slotIndex: 0}, - {accountIndex: 0, kind: balWarmBALCode}, - {accountIndex: 1, kind: balWarmBALCode}, + {accountIndex: 0, slotFrom: 0, slotTo: 1}, + {accountIndex: 1, slotFrom: 0, slotTo: 1}, }, tasks) } +func TestWarmBALPropagatesWorkerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + db := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + bal := types.BlockAccessList{{Address: accounts.InternAddress(common.Address{19: 1})}} + err := NewBlockReadAheader().warmBAL(ctx, db, bal, nil, balCodeWarmupNone, 1) + require.ErrorIs(t, err, context.Canceled) +} + func TestBlockReadAheaderWarmsOverlayBlockAccessList(t *testing.T) { oldReadAhead := dbg.ReadAhead dbg.SetReadAhead(true) @@ -294,6 +346,25 @@ func TestBlockReadAheaderPrefersCachedBlockAccessList(t *testing.T) { require.Zero(t, getter.getOneCalls) } +func TestBlockReadAheaderSkipsBlockAccessListReadWhenDisabledOrAbsent(t *testing.T) { + oldReadAhead := dbg.ReadAhead + dbg.SetReadAhead(false) + t.Cleanup(func() { dbg.SetReadAhead(oldReadAhead) }) + db := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + getter := new(countingGetter) + bal := make(types.BlockAccessList, 0) + balHash := bal.Hash() + headerWithBAL := &types.Header{Number: *uint256.NewInt(1), BlockAccessListHash: &balHash} + bra := NewBlockReadAheader() + bra.AddHeaderAndBody(t.Context(), db, getter, headerWithBAL, new(types.Body)) + require.Zero(t, getter.getOneCalls, "READ_AHEAD=false must skip the BAL lookup") + dbg.SetReadAhead(true) + headerWithoutBAL := &types.Header{Number: *uint256.NewInt(2)} + bra.AddHeaderAndBody(t.Context(), db, getter, headerWithoutBAL, new(types.Body)) + bra.WaitForWarmup(t.Context()) + require.Zero(t, getter.getOneCalls, "pre-Amsterdam blocks must skip the BAL lookup") +} + // seedFill places an entry with an exact txNum stamp through the public fill // API without moving the applied frontier. func seedFill(sc *cache.StateCache, domain kv.Domain, k, v []byte, txNum uint64) { @@ -346,19 +417,31 @@ func TestCachePopulatingGetterReusesCodeByHashAcrossGetters(t *testing.T) { account := accounts.Account{Nonce: 1, CodeHash: accounts.InternCodeHash(crypto.Keccak256Hash(code))} source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} stateCache := newTestStateCache() - for _, address := range []accounts.Address{ - accounts.InternAddress(common.Address{19: 1}), - accounts.InternAddress(common.Address{19: 2}), - } { - reader := state.NewReaderV3(&cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(cache.FrontierFunc(emptyVisibleEnd)), stepSize: 16}) - gotAccount, err := reader.ReadAccountData(address) - require.NoError(t, err) - require.Equal(t, account.CodeHash, gotAccount.CodeHash) - gotCode, err := reader.ReadAccountCode(address) - require.NoError(t, err) - require.Equal(t, code, gotCode) - } + frontier := cache.FrontierFunc(func(kv.Domain) (uint64, bool) { return 16, true }) + firstAddress := accounts.InternAddress(common.Address{19: 1}) + secondAddress := accounts.InternAddress(common.Address{19: 2}) + firstReader := state.NewReaderV3(&cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16}) + gotAccount, err := firstReader.ReadAccountData(firstAddress) + require.NoError(t, err) + require.Equal(t, account.CodeHash, gotAccount.CodeHash) + gotCode, err := firstReader.ReadAccountCode(firstAddress) + require.NoError(t, err) + require.Equal(t, code, gotCode) + accountReader := state.NewReaderV3(&cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16}) + gotAccount, err = accountReader.ReadAccountData(secondAddress) + require.NoError(t, err) + require.Equal(t, account.CodeHash, gotAccount.CodeHash) + codeGetter := &cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16} + codeReader := state.NewReaderV3(codeGetter) + gotCode, err = codeReader.ReadAccountCode(secondAddress) + require.NoError(t, err) + require.Equal(t, code, gotCode) + require.Equal(t, 2, source.accountReads, "the code phase must reuse the account read from the state phase") require.Equal(t, 1, source.codeReads, "identical code must be loaded from the address-keyed domain only once") + secondAddressValue := secondAddress.Value() + boundCode, ok := stateCache.View(nil).Get(kv.CodeDomain, secondAddressValue[:]) + require.True(t, ok, "the code-hash fast path must bind code to the second address") + require.Equal(t, code, boundCode) } // Cold keys must still be warmed — that is the prefetcher's purpose. diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 684560f8c64..7f5c01e1772 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -108,6 +108,10 @@ func restoreTxNum(ctx context.Context, cfg *ExecuteBlockCfg, applyTx kv.Tx, curr return inputTxNum, maxTxNum, offsetFromBlockBeginning, blockNum, nil } +func shouldWaitForReadAhead(isValidatingBlocks bool) bool { + return dbg.ReadAheadWait && isValidatingBlocks +} + func ExecV3(ctx context.Context, execStage *StageState, u Unwinder, cfg ExecuteBlockCfg, doms *execctx.SharedDomains, rwTx kv.TemporalRwTx, @@ -208,6 +212,9 @@ func ExecV3(ctx context.Context, doms.SetDeferCommitmentUpdates(true) } defer doms.SetDeferCommitmentUpdates(false) + if shouldWaitForReadAhead(isForkValidation) && cfg.readAheader != nil { + cfg.readAheader.WaitForWarmup(ctx) + } // snapshots are often stored on chaper drives. don't expect low-read-latency and manually read-ahead. // can't use OS-level ReadAhead - because Data >> RAM // it also warmsup state a bit - by touching senders/coninbase accounts and code diff --git a/execution/stagedsync/exec3_read_ahead_test.go b/execution/stagedsync/exec3_read_ahead_test.go new file mode 100644 index 00000000000..4cbf58682ae --- /dev/null +++ b/execution/stagedsync/exec3_read_ahead_test.go @@ -0,0 +1,20 @@ +package stagedsync + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/dbg" +) + +func TestShouldWaitForReadAhead(t *testing.T) { + oldReadAheadWait := dbg.ReadAheadWait + t.Cleanup(func() { dbg.ReadAheadWait = oldReadAheadWait }) + dbg.ReadAheadWait = false + require.False(t, shouldWaitForReadAhead(false)) + require.False(t, shouldWaitForReadAhead(true)) + dbg.ReadAheadWait = true + require.False(t, shouldWaitForReadAhead(false)) + require.True(t, shouldWaitForReadAhead(true)) +}