diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 0ff10651dc2..f616f616f3c 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -138,6 +138,10 @@ var ( DisableAdaptivePin = EnvBool("DISABLE_ADAPTIVE_PIN", false) AssertStateCache = EnvBool("ASSERT_STATE_CACHE", false) ReadAhead = EnvBool("READ_AHEAD", true) + ReadAheadWorkers = EnvInt("READ_AHEAD_WORKERS", runtime.NumCPU()) + ReadAheadWait = EnvBool("READ_AHEAD_WAIT", false) + ReadAheadBALCode = EnvBool("READ_AHEAD_BAL_CODE", false) + ReadAheadTxCode = EnvBool("READ_AHEAD_TX_CODE", true) // FilesAsyncIO warms cold state .kv pages via io_uring before the mmap read, so // a would-be blocking page fault becomes a non-blocking read that releases the // goroutine's P. Linux + io_uring only; self-disables (reads use ordinary faults) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 42e0e740eab..f45cf76d355 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -1159,6 +1159,44 @@ func TestStateCache_AccountDeletionGatesStaleCodeFill(t *testing.T) { require.True(t, ok, "unrelated code fills from a current view must stay admitted") } +func TestStateCache_CodeHashHitBindsAddress(t *testing.T) { + b := 1 * datasize.MB + c := NewStateCache(b, b, b, b) + t.Cleanup(c.Close) + firstAddr, secondAddr, code := makeAddr(1), makeAddr(2), makeCode(1) + codeHash := crypto.Keccak256Hash(code) + view := c.View(FrontierFunc(func(kv.Domain) (uint64, bool) { return 100, true })) + view.Fill(kv.CodeDomain, firstAddr, code, 10) + view.SeedAddrCodeHash(secondAddr, codeHash, 11) + _, ok := c.View(nil).Get(kv.CodeDomain, secondAddr) + require.False(t, ok, "the second address must start without an addr-keyed code binding") + got, ok := view.GetCodeByAddressHash(secondAddr) + require.True(t, ok) + require.Equal(t, code, got) + got, ok = c.View(nil).Get(kv.CodeDomain, secondAddr) + require.True(t, ok, "the hash hit must populate the addr-keyed code binding") + require.Equal(t, code, got) + c.Applier().Unwind(11) + _, ok = c.View(nil).Get(kv.CodeDomain, secondAddr) + require.False(t, ok, "the derived binding must keep the mapping stamp for unwind invalidation") +} + +func TestStateCache_EmptyCodeHashUsesViewFrontierStamp(t *testing.T) { + b := 1 * datasize.MB + c := NewStateCache(b, b, b, b) + t.Cleanup(c.Close) + addr := makeAddr(1) + view := c.View(FrontierFunc(func(kv.Domain) (uint64, bool) { return 100, true })) + view.SeedAddrCodeHash(addr, [32]byte{}, 4) + codeHash, txNum, ok := c.getAddrCodeHashWithTxNum(addr) + require.True(t, ok) + require.Zero(t, codeHash) + require.Equal(t, uint64(99), txNum, "a negative mapping reflects the view, not a nonexistent value's reported step") + c.Applier().Unwind(99) + _, _, ok = c.getAddrCodeHashWithTxNum(addr) + require.False(t, ok) +} + // An apply-only cache (STATE_CACHE_FILLS=false) has no fill for a lowered // frontier to poison; wire-up code keys the aggregator forbid on this. func TestApplyOnlyCacheReportsFillsDisabled(t *testing.T) { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 45f7297113e..78a103d3a22 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -364,17 +364,25 @@ func (c *CodeCache) putCodeLocked(addr []byte, code []byte, keyHash [32]byte, co // EVM-known codeHash is already known. Eviction is LRU; freshly seen addrs // replace coldest entries. func (c *CodeCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + h, _, ok := c.GetAddrCodeHashWithTxNum(addr) + return h, ok +} + +// GetAddrCodeHashWithTxNum is GetAddrCodeHash plus the txNum of the +// addr→codeHash mapping. A caller that uses the hash to bind cached code to +// the address must preserve this stamp so unwind invalidation remains correct. +func (c *CodeCache) GetAddrCodeHashWithTxNum(addr []byte) ([32]byte, uint64, bool) { k := common.BytesToAddress(addr) coh := c.coh.Snapshot() e, ok := c.addrToCodeHash.Get(k) if !ok { - return [32]byte{}, false + return [32]byte{}, 0, false } if coh.IsStale(e.txNum, e.epoch) { c.addrToCodeHash.Remove(k) - return [32]byte{}, false + return [32]byte{}, 0, false } - return e.hash, true + return e.hash, e.txNum, true } // PutAddrCodeHash records a committed-state addr → codeHash mapping. An diff --git a/execution/cache/code_cache_codehash_test.go b/execution/cache/code_cache_codehash_test.go index 6572b99ab26..cdea474386b 100644 --- a/execution/cache/code_cache_codehash_test.go +++ b/execution/cache/code_cache_codehash_test.go @@ -62,6 +62,17 @@ func TestCodeCache_PutAddrCodeHashReplacesStaleEntry(t *testing.T) { require.Equal(t, newHash, got) } +func TestCodeCache_GetAddrCodeHashWithTxNumPreservesStamp(t *testing.T) { + c := closeOnCleanup(t, NewCodeCache(1*datasize.MB, 1*datasize.MB)) + addr := makeAddr(1) + codeHash := [32]byte{1} + c.PutAddrCodeHash(addr, codeHash, 42) + gotHash, gotTxNum, ok := c.GetAddrCodeHashWithTxNum(addr) + require.True(t, ok) + require.Equal(t, codeHash, gotHash) + require.Equal(t, uint64(42), gotTxNum) +} + func TestCodeCache_GetByCodeHash_HitAfterPut(t *testing.T) { c := closeOnCleanup(t, NewCodeCache(1*datasize.MB, 1*datasize.MB)) addr := makeAddr(1) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 276871b07cc..ace5fbb9514 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -24,6 +24,7 @@ import ( "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" @@ -204,6 +205,14 @@ func (c *StateCache) getAddrCodeHash(addr []byte) ([32]byte, bool) { return cc.GetAddrCodeHash(addr) } +func (c *StateCache) getAddrCodeHashWithTxNum(addr []byte) ([32]byte, uint64, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return [32]byte{}, 0, false + } + return cc.GetAddrCodeHashWithTxNum(addr) +} + // seedAddrCodeHash conditionally records an addr → codeHash mapping. // The mapping derives from an account record, so admission checks the accounts // frontier even though the mapping lives in the code cache. @@ -271,18 +280,25 @@ func (c *StateCache) fillIfFresh(domain kv.Domain, key []byte, value []byte, rea // negatives are not cached here: "no code" is cached at the addr→codeHash // mapping instead (the zero-hash sentinel seeded by SeedAddrCodeHash). func (c *StateCache) fillCodeIfFresh(key []byte, value []byte, readTxNum, visibleEnd, accountsVisibleEnd uint64) { - codeCache, ok := c.caches[kv.CodeDomain].(*CodeCache) - if !ok || len(value) == 0 { + if len(value) == 0 { return } codeHash := crypto.Keccak256(value) cloned := bytes.Clone(value) + c.fillCodeWithHashIfFresh(key, cloned, codeHash, readTxNum, visibleEnd, accountsVisibleEnd) +} + +func (c *StateCache) fillCodeWithHashIfFresh(key, value, codeHash []byte, readTxNum, visibleEnd, accountsVisibleEnd uint64) { + codeCache, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok || len(value) == 0 || len(codeHash) != len(common.Hash{}) { + return + } c.admissionMu.RLock() defer c.admissionMu.RUnlock() if visibleEnd < c.appliedEnd[kv.CodeDomain] || accountsVisibleEnd < c.appliedEnd[kv.AccountsDomain] { return } - codeCache.PutWithCodeHashIfAbsent(key, cloned, codeHash, readTxNum) + codeCache.PutWithCodeHashIfAbsent(key, value, codeHash, readTxNum) } // deleteKey removes the data for the given domain and key. Authoritative diff --git a/execution/cache/view.go b/execution/cache/view.go index 0de781920d6..b375bb0937d 100644 --- a/execution/cache/view.go +++ b/execution/cache/view.go @@ -105,6 +105,41 @@ func (v ReadView) GetAddrCodeHash(addr []byte) ([32]byte, bool) { return v.c.getAddrCodeHash(addr) } +// GetCodeByAddressHash resolves the durable addr→codeHash mapping and then +// probes the content-addressed code cache. On a hit it also fills the +// addr-keyed code binding with the mapping's original txNum, avoiding both a +// database read and a redundant keccak of the cached code. +func (v ReadView) GetCodeByAddressHash(addr []byte) ([]byte, bool) { + if v.c == nil { + return nil, false + } + codeHash, readTxNum, ok := v.c.getAddrCodeHashWithTxNum(addr) + if !ok || codeHash == ([32]byte{}) { + return nil, false + } + code, ok := v.c.getCodeByHash(codeHash[:]) + if !ok { + return nil, false + } + v.fillCodeWithHash(addr, code, codeHash[:], readTxNum) + return code, true +} + +func (v ReadView) fillCodeWithHash(addr, code, codeHash []byte, readTxNum uint64) { + if v.c == nil || v.c.disableFills || v.frontier == nil { + return + } + visibleEnd, ok := v.frontier.DomainVisibleEnd(kv.CodeDomain) + if !ok { + return + } + accountsEnd, ok := v.frontier.DomainVisibleEnd(kv.AccountsDomain) + if !ok { + return + } + v.c.fillCodeWithHashIfFresh(addr, code, codeHash, readTxNum, visibleEnd, accountsEnd) +} + // CanFill reports whether this view carries a frontier, i.e. Fill and // SeedAddrCodeHash can admit values through it. func (v ReadView) CanFill() bool { return v.c != nil && v.frontier != nil } @@ -145,6 +180,12 @@ func (v ReadView) SeedAddrCodeHash(addr []byte, h [32]byte, txNum uint64) { if !ok { return } + if h == ([32]byte{}) { + txNum = 0 + if visibleEnd > 0 { + txNum = visibleEnd - 1 + } + } v.c.seedAddrCodeHash(addr, h, txNum, visibleEnd) } diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 0dd5e11964d..a16f415a65a 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -3,7 +3,7 @@ package exec import ( "bytes" "context" - "sync" + "errors" "sync/atomic" "time" @@ -16,7 +16,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/dbservices" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/kv/dbutils" + "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" @@ -32,8 +32,7 @@ type BlockReadAheader struct { bals *lru.Cache[common.Hash, []byte] // this is for warming state - warming atomic.Bool // only one warmBody can run at a time - warmWg sync.WaitGroup + warmDone chan struct{} // contains one token while no warmBody is running // stateCache is the process-global state cache that SharedDomains.GetLatest // consults on the EVM hot path. When set, warmBody routes its prefetches @@ -61,11 +60,14 @@ func NewBlockReadAheader() *BlockReadAheader { if err != nil { panic(err) } + warmDone := make(chan struct{}, 1) + warmDone <- struct{}{} return &BlockReadAheader{ - headers: headers, - bodies: bodies, - senders: senders, - bals: bals, + headers: headers, + bodies: bodies, + senders: senders, + bals: bals, + warmDone: warmDone, } } @@ -104,36 +106,67 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k if err == nil { readTxNum := (uint64(step)+1)*cpg.stepSize - 1 cpg.view.Fill(name, k, v, readTxNum) + if name == kv.AccountsDomain { + var codeHash common.Hash + copy(codeHash[:], accounts.DeserialiseV3CodeHash(v)) + cpg.view.SeedAddrCodeHash(k, codeHash, readTxNum) + } } return v, step, err } -func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body) { +func (cpg *cachePopulatingGetter) GetCode(addr []byte, _ uint64) ([]byte, bool, error) { + if code, ok := cpg.view.GetCodeByAddressHash(addr); ok { + return code, true, nil + } + code, _, err := cpg.GetLatest(kv.CodeDomain, addr) + if err != nil { + return nil, false, err + } + return code, len(code) > 0, nil +} + +func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, tx kv.Getter, header *types.Header, body *types.Body) { + if header == nil || body == nil { + return + } blockHash := header.Hash() bra.headers.Add(blockHash, header) bra.bodies.Add(blockHash, body) - if db != nil && ctx != nil { - // Only allow one warmBody to run at a time - if !bra.warming.CompareAndSwap(false, true) { - return + if db == nil || ctx == nil || !dbg.ReadAhead { + return + } + select { + case <-bra.warmDone: + default: + return + } + var balBytes []byte + if header.HasBAL() { + var ok bool + balBytes, ok = bra.bals.Get(blockHash) + if !ok { + var err error + balBytes, err = rawdb.ReadBlockAccessListBytes(tx, blockHash, header.Number.Uint64()) + balBytes = bytes.Clone(balBytes) + if err != nil { + log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", blockHash, "err", err) + } } - bra.warmWg.Go(func() { - bra.warmBody(ctx, db, header, body, 8) // use 8 workers for warming - }) } + go func() { + defer func() { bra.warmDone <- struct{}{} }() + bra.warmBody(ctx, db, header, body, balBytes, dbg.ReadAheadWorkers) + }() } // WaitForWarmup blocks until any in-flight warmBody goroutine finishes or // the context is cancelled. Call before closing the database to avoid // waitTxsAllDoneOnClose hangs. func (bra *BlockReadAheader) WaitForWarmup(ctx context.Context) { - done := make(chan struct{}) - go func() { - bra.warmWg.Wait() - close(done) - }() select { - case <-done: + case <-bra.warmDone: + bra.warmDone <- struct{}{} case <-ctx.Done(): } } @@ -152,173 +185,252 @@ func (bra *BlockReadAheader) AddBlockAccessList(blockHash common.Hash, bal []byt bra.bals.Add(blockHash, bal) } -// warmBody warms state for all transactions in a body using multiple workers. -// It reads: To accounts, To account code, To account storage from access lists, -// and block-level access lists. Each worker creates its own transaction. -// Only one warmBody can run at a time - concurrent calls are no-ops. -func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body, workers int) { - defer bra.warming.Store(false) +const balWarmupStorageChunkSize = 64 - if !dbg.ReadAhead { - return +type balCodeWarmupMode uint8 + +const ( + balCodeWarmupNone balCodeWarmupMode = iota + balCodeWarmupTxnDestinations + balCodeWarmupAll +) + +type balWarmupTask struct { + accountIndex uint32 + slotFrom uint32 + slotTo uint32 +} + +func balCodeWarmupModeForFlags(warmBALCode, warmTxCode bool) balCodeWarmupMode { + if warmBALCode { + return balCodeWarmupAll } + if warmTxCode { + return balCodeWarmupTxnDestinations + } + return balCodeWarmupNone +} - if workers <= 0 { - workers = 1 +func makeBALWarmupPlan(bal types.BlockAccessList, workers int) ([]balWarmupTask, int) { + taskCount := 0 + for _, account := range bal { + slots := len(account.StorageChanges) + len(account.StorageReads) + taskCount += max(1, (slots+balWarmupStorageChunkSize-1)/balWarmupStorageChunkSize) + } + tasks := make([]balWarmupTask, 0, taskCount) + for accountIndex, account := range bal { + slots := len(account.StorageChanges) + len(account.StorageReads) + if slots == 0 { + tasks = append(tasks, balWarmupTask{accountIndex: uint32(accountIndex)}) + continue + } + for slotFrom := 0; slotFrom < slots; slotFrom += balWarmupStorageChunkSize { + tasks = append(tasks, balWarmupTask{accountIndex: uint32(accountIndex), slotFrom: uint32(slotFrom), slotTo: uint32(min(slotFrom+balWarmupStorageChunkSize, slots))}) + } } + return tasks, min(workers, len(tasks)) +} - var wg errgroup.Group +func uniqueTransactionDestinations(txns types.Transactions) map[accounts.Address]struct{} { + destinations := make(map[accounts.Address]struct{}, len(txns)) + for _, txn := range txns { + to := txn.GetTo() + if to == nil { + continue + } + address := accounts.InternAddress(*to) + destinations[address] = struct{}{} + } + return destinations +} - // If BAL exists in DB, use BAL warming (more complete) - var bal types.BlockAccessList - if header != nil && db != nil { - tx, err := db.BeginRo(ctx) +func warmBALStateTask(stateReader *state.ReaderV3, account *types.AccountChanges, task balWarmupTask, codeMode balCodeWarmupMode, txCodeDestinations map[accounts.Address]struct{}) error { + var accountData *accounts.Account + if task.slotFrom == 0 { + var err error + accountData, err = stateReader.ReadAccountData(account.Address) if err != nil { - log.Warn("[warmBody] failed to open tx for BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + return err + } + } + storageChanges := uint32(len(account.StorageChanges)) + for slotIndex := task.slotFrom; slotIndex < task.slotTo; slotIndex++ { + var slot accounts.StorageKey + if slotIndex < storageChanges { + slot = account.StorageChanges[slotIndex].Slot } else { - data, err := tx.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), header.Hash())) - if err != nil { - log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } else if len(data) > 0 { - bal, err = types.DecodeBlockAccessListBytes(data) - if err != nil { - log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } - } - tx.Rollback() + slot = account.StorageReads[slotIndex-storageChanges] + } + if _, _, err := stateReader.ReadAccountStorage(account.Address, slot); err != nil { + return err } } + if task.slotFrom != 0 || codeMode == balCodeWarmupNone { + return nil + } + warmCode := false + if codeMode == balCodeWarmupAll { + warmCode = len(account.CodeChanges) > 0 || (accountData != nil && !accountData.CodeHash.IsEmpty()) + } else if _, ok := txCodeDestinations[account.Address]; ok { + warmCode = accountData != nil && !accountData.CodeHash.IsEmpty() + } + if warmCode { + _, err := stateReader.ReadAccountCode(account.Address) + return err + } + return nil +} - balLen := len(bal) - if balLen > 0 { - balWorkers := min(workers, balLen) +func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body, balBytes []byte, workers int) { + if !dbg.ReadAhead { + return + } + if workers <= 0 { + workers = 1 + } + var bal types.BlockAccessList + if len(balBytes) > 0 { + var err error + bal, err = types.DecodeBlockAccessListBytes(balBytes) + if err != nil { + log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } + } + if len(bal) > 0 { + codeMode := balCodeWarmupModeForFlags(dbg.ReadAheadBALCode, dbg.ReadAheadTxCode) + if err := bra.warmBAL(ctx, db, bal, body.Transactions, codeMode, workers); err != nil && !errors.Is(err, context.Canceled) { + log.Warn("[warmBAL] failed", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } + return + } + if err := bra.warmTxns(ctx, db, body.Transactions, workers); err != nil && !errors.Is(err, context.Canceled) { + log.Warn("[warmTxns] failed", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } +} - // Pre-divide work: each worker gets a dedicated range of BAL entries - entriesPerWorker := (balLen + balWorkers - 1) / balWorkers +func (bra *BlockReadAheader) warmBAL(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, txns types.Transactions, codeMode balCodeWarmupMode, workers int) error { + var txCodeDestinations map[accounts.Address]struct{} + if codeMode == balCodeWarmupTxnDestinations { + txCodeDestinations = uniqueTransactionDestinations(txns) + } + tasks, balWorkers := makeBALWarmupPlan(bal, workers) + return bra.warmBALState(ctx, db, bal, tasks, codeMode, txCodeDestinations, balWorkers) +} - for w := range balWorkers { - start := w * entriesPerWorker - end := min(start+entriesPerWorker, balLen) - if start >= balLen { - break +func (bra *BlockReadAheader) warmBALState(ctx context.Context, db kv.RoDB, bal types.BlockAccessList, tasks []balWarmupTask, codeMode balCodeWarmupMode, txCodeDestinations map[accounts.Address]struct{}, workers int) error { + var nextTask atomic.Uint64 + wg, workerCtx := errgroup.WithContext(ctx) + for w := range workers { + wg.Go(func() error { + startTime := time.Now() + tx, err := db.BeginRo(workerCtx) + if err != nil { + return err } - - // Capture loop variables for closure - workerStart, workerEnd, workerID := start, end, w - wg.Go(func() error { - startTime := time.Now() - tx, err := db.BeginRo(ctx) - if err != nil { - return err + defer tx.Rollback() + ttx, ok := tx.(kv.TemporalTx) + if !ok { + return errors.New("BAL warmup requires a temporal read transaction") + } + stateReader := state.NewReaderV3(readAheadGetter(ttx, bra.stateCache)) + tasksProcessed := 0 + for { + select { + case <-workerCtx.Done(): + return workerCtx.Err() + default: } - defer tx.Rollback() - - ttx, ok := tx.(kv.TemporalTx) - if !ok { - return nil + taskIndex := int(nextTask.Add(1) - 1) + if taskIndex >= len(tasks) { + break } - stateReader := state.NewReaderV3(readAheadGetter(ttx, bra.stateCache)) - - for idx := workerStart; idx < workerEnd; idx++ { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - acctChanges := bal[idx] - acct, _ := stateReader.ReadAccountData(acctChanges.Address) - // Warm code if account has code or if there are code changes. - if acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(acctChanges.Address) - } else if len(acctChanges.CodeChanges) > 0 { - stateReader.ReadAccountCode(acctChanges.Address) - } - for _, slotChanges := range acctChanges.StorageChanges { - stateReader.ReadAccountStorage(acctChanges.Address, slotChanges.Slot) - } - for _, slot := range acctChanges.StorageReads { - stateReader.ReadAccountStorage(acctChanges.Address, slot) - } + task := tasks[taskIndex] + account := bal[task.accountIndex] + if err := warmBALStateTask(stateReader, account, task, codeMode, txCodeDestinations); err != nil { + return err } - log.Debug("[warmBody] BAL worker finished", "worker", workerID, "entries", workerEnd-workerStart, "elapsed", time.Since(startTime)) - return nil - }) - } - wg.Wait() - return + tasksProcessed++ + } + log.Debug("[warmBAL] state worker finished", "worker", w, "tasks", tasksProcessed, "elapsed", time.Since(startTime)) + return nil + }) } - // Fallback: per-transaction warming when no BAL - txns := body.Transactions + return wg.Wait() +} + +func (bra *BlockReadAheader) warmTxns(ctx context.Context, db kv.RoDB, txns types.Transactions, workers int) error { if len(txns) == 0 { - return + return nil } - txnLen := len(txns) if workers > txnLen { workers = txnLen } - // Pre-divide work: each worker gets a dedicated range of transactions txnsPerWorker := (txnLen + workers - 1) / workers - + wg, workerCtx := errgroup.WithContext(ctx) for w := 0; w < workers; w++ { start := w * txnsPerWorker end := min(start+txnsPerWorker, txnLen) if start >= txnLen { break } - - // Capture loop variables for closure - workerStart, workerEnd, workerID := start, end, w wg.Go(func() error { startTime := time.Now() - tx, err := db.BeginRo(ctx) + tx, err := db.BeginRo(workerCtx) if err != nil { return err } defer tx.Rollback() - ttx, ok := tx.(kv.TemporalTx) if !ok { - return nil + return errors.New("transaction warmup requires a temporal read transaction") } stateReader := state.NewReaderV3(readAheadGetter(ttx, bra.stateCache)) - - for txIdx := workerStart; txIdx < workerEnd; txIdx++ { + for txIdx := start; txIdx < end; txIdx++ { select { - case <-ctx.Done(): - return ctx.Err() + case <-workerCtx.Done(): + return workerCtx.Err() default: } - txn := txns[txIdx] - // Warm To account and its code if it has one if toAddr := txn.GetTo(); toAddr != nil { to := accounts.InternAddress(*toAddr) - if acct, _ := stateReader.ReadAccountData(to); acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(to) + acct, err := stateReader.ReadAccountData(to) + if err != nil { + return err + } + if acct != nil && !acct.CodeHash.IsEmpty() { + if _, err := stateReader.ReadAccountCode(to); err != nil { + return err + } } } - // Warm transaction access list accounts and their code for _, entry := range txn.GetAccessList() { addr := accounts.InternAddress(entry.Address) - if acct, _ := stateReader.ReadAccountData(addr); acct != nil && !acct.CodeHash.IsEmpty() { - stateReader.ReadAccountCode(addr) + acct, err := stateReader.ReadAccountData(addr) + if err != nil { + return err + } + if acct != nil && !acct.CodeHash.IsEmpty() { + if _, err := stateReader.ReadAccountCode(addr); err != nil { + return err + } } for _, slot := range entry.StorageKeys { - stateReader.ReadAccountStorage(addr, accounts.InternKey(slot)) + if _, _, err := stateReader.ReadAccountStorage(addr, accounts.InternKey(slot)); err != nil { + return err + } } } } - log.Debug("[warmBody] TX worker finished", "worker", workerID, "txns", workerEnd-workerStart, "elapsed", time.Since(startTime)) + log.Debug("[warmTxns] worker finished", "worker", w, "txns", end-start, "elapsed", time.Since(startTime)) return nil }) } - - wg.Wait() + return wg.Wait() } func (bra *BlockReadAheader) ReadBodyWithTransactions(blockHash common.Hash) (*types.Body, bool) { diff --git a/execution/exec/blocks_read_ahead_test.go b/execution/exec/blocks_read_ahead_test.go index 439ac18982e..8c2c83fbe4d 100644 --- a/execution/exec/blocks_read_ahead_test.go +++ b/execution/exec/blocks_read_ahead_test.go @@ -19,6 +19,7 @@ package exec import ( "context" "testing" + "time" "github.com/c2h5oh/datasize" "github.com/holiman/uint256" @@ -26,9 +27,18 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/dbg" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/membatchwithdb" + "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/cache" + "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" + "github.com/erigontech/erigon/execution/types/accounts" ) // stubTemporalGetter stands in for the committed-state read view a warmup @@ -38,6 +48,23 @@ type stubTemporalGetter struct { step kv.Step } +type sharedCodeTemporalGetter struct { + account []byte + code []byte + accountReads int + codeReads int +} + +type countingGetter struct { + kv.Getter + getOneCalls int +} + +func (g *countingGetter) GetOne(string, []byte) ([]byte, error) { + g.getOneCalls++ + return nil, nil +} + func (s stubTemporalGetter) GetLatest(kv.Domain, []byte) ([]byte, kv.Step, error) { return s.v, s.step, nil } @@ -48,11 +75,244 @@ func (s stubTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, func (s stubTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } +func (s *sharedCodeTemporalGetter) GetLatest(domain kv.Domain, _ []byte) ([]byte, kv.Step, error) { + if domain == kv.AccountsDomain { + s.accountReads++ + return s.account, 0, nil + } + if domain == kv.CodeDomain { + s.codeReads++ + return s.code, 0, nil + } + return nil, 0, nil +} + +func (s *sharedCodeTemporalGetter) HasPrefix(kv.Domain, []byte) ([]byte, []byte, bool, error) { + return nil, nil, false, nil +} + +func (s *sharedCodeTemporalGetter) StepsInFiles(...kv.Domain) kv.Step { return 0 } + func newTestStateCache() *cache.StateCache { b := 1 * datasize.MB return cache.NewStateCache(b, b, b, b) } +func TestBlockReadAheaderWaitForWarmup(t *testing.T) { + readAheader := NewBlockReadAheader() + for range 2 { + <-readAheader.warmDone + done := make(chan struct{}) + go func() { + readAheader.WaitForWarmup(t.Context()) + close(done) + }() + select { + case <-done: + t.Fatal("warmup wait returned before warmup completed") + case <-time.After(50 * time.Millisecond): + } + readAheader.warmDone <- struct{}{} + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("warmup wait did not return after warmup completed") + } + } +} + +func TestBlockReadAheaderIgnoresMissingHeaderOrBody(t *testing.T) { + readAheader := NewBlockReadAheader() + header := &types.Header{Number: *uint256.NewInt(1)} + require.NotPanics(t, func() { readAheader.AddHeaderAndBody(t.Context(), nil, nil, nil, new(types.Body)) }) + require.NotPanics(t, func() { readAheader.AddHeaderAndBody(t.Context(), nil, nil, header, nil) }) + require.Zero(t, readAheader.headers.Len()) + require.Zero(t, readAheader.bodies.Len()) +} + +func TestMakeBALWarmupTasksSplitsStorageHeavyAccount(t *testing.T) { + bal := types.BlockAccessList{{ + StorageChanges: make([]*types.SlotChanges, 65), + StorageReads: make([]accounts.StorageKey, 3), + }} + tasks, workers := makeBALWarmupPlan(bal, 4) + require.Equal(t, 2, workers) + require.Equal(t, []balWarmupTask{ + {accountIndex: 0, slotFrom: 0, slotTo: 64}, + {accountIndex: 0, slotFrom: 64, slotTo: 68}, + }, tasks) +} + +func TestBALCodeWarmupModeForFlags(t *testing.T) { + tests := []struct { + name string + warmBALCode bool + warmTxCode bool + want balCodeWarmupMode + }{ + {name: "all BAL code", warmBALCode: true, want: balCodeWarmupAll}, + {name: "BAL code takes precedence", warmBALCode: true, warmTxCode: true, want: balCodeWarmupAll}, + {name: "transaction destinations", warmTxCode: true, want: balCodeWarmupTxnDestinations}, + {name: "no code", want: balCodeWarmupNone}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, balCodeWarmupModeForFlags(test.warmBALCode, test.warmTxCode)) + }) + } +} + +func TestUniqueTransactionDestinations(t *testing.T) { + destinationA := common.Address{19: 0xa1} + destinationB := common.Address{19: 0xb2} + txns := types.Transactions{ + types.NewTransaction(0, destinationA, nil, 0, nil, nil), + types.NewTransaction(1, destinationA, nil, 0, nil, []byte{0x01}), + types.NewContractCreation(2, nil, 0, nil, []byte{0x02}), + types.NewTransaction(3, destinationB, nil, 0, nil, nil), + } + require.Equal(t, map[accounts.Address]struct{}{accounts.InternAddress(destinationA): {}, accounts.InternAddress(destinationB): {}}, uniqueTransactionDestinations(txns)) +} + +func TestWarmBALStateTaskLoadsSelectedCode(t *testing.T) { + for _, test := range []struct { + name string + mode balCodeWarmupMode + destination bool + contract bool + codeChanges bool + wantCodeReads int + }{ + {name: "transaction destination contract", mode: balCodeWarmupTxnDestinations, destination: true, contract: true, wantCodeReads: 1}, + {name: "transaction non-destination contract", mode: balCodeWarmupTxnDestinations, contract: true}, + {name: "transaction destination EOA", mode: balCodeWarmupTxnDestinations, destination: true}, + {name: "all contract", mode: balCodeWarmupAll, contract: true, wantCodeReads: 1}, + {name: "all EOA", mode: balCodeWarmupAll}, + {name: "all forced code change", mode: balCodeWarmupAll, codeChanges: true, wantCodeReads: 1}, + {name: "none", destination: true, contract: true}, + } { + t.Run(test.name, func(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.NewAccount() + if test.contract { + account.CodeHash = accounts.InternCodeHash(crypto.Keccak256Hash(code)) + } + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + stateCache := newTestStateCache() + t.Cleanup(stateCache.Close) + frontier := cache.FrontierFunc(func(kv.Domain) (uint64, bool) { return 16, true }) + address := accounts.InternAddress(common.Address{19: 1}) + destinations := make(map[accounts.Address]struct{}) + if test.destination { + destinations[address] = struct{}{} + } + accountChanges := &types.AccountChanges{Address: address} + if test.codeChanges { + accountChanges.CodeChanges = []*types.CodeChange{{Bytecode: code}} + } + getter := &cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16} + reader := state.NewReaderV3(getter) + require.NoError(t, warmBALStateTask(reader, accountChanges, balWarmupTask{}, test.mode, destinations)) + require.Equal(t, 1, source.accountReads) + require.Equal(t, test.wantCodeReads, source.codeReads) + }) + } +} + +func TestWarmBALStateTaskDoesNotRepeatCodeForLaterChunks(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.NewAccount() + account.CodeHash = accounts.InternCodeHash(crypto.Keccak256Hash(code)) + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + address := accounts.InternAddress(common.Address{19: 1}) + accountChanges := &types.AccountChanges{Address: address, StorageReads: make([]accounts.StorageKey, 65)} + reader := state.NewReaderV3(source) + require.NoError(t, warmBALStateTask(reader, accountChanges, balWarmupTask{slotFrom: 64, slotTo: 65}, balCodeWarmupAll, nil)) + require.Zero(t, source.accountReads) + require.Zero(t, source.codeReads) +} + +func TestMakeBALWarmupTasksKeepsSmallAccountsTogether(t *testing.T) { + bal := types.BlockAccessList{ + {StorageChanges: make([]*types.SlotChanges, 1)}, + {StorageReads: make([]accounts.StorageKey, 1)}, + } + tasks, workers := makeBALWarmupPlan(bal, 4) + require.Equal(t, 2, workers) + require.Equal(t, []balWarmupTask{ + {accountIndex: 0, slotFrom: 0, slotTo: 1}, + {accountIndex: 1, slotFrom: 0, slotTo: 1}, + }, tasks) +} + +func TestWarmBALPropagatesWorkerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + db := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + bal := types.BlockAccessList{{Address: accounts.InternAddress(common.Address{19: 1})}} + err := NewBlockReadAheader().warmBAL(ctx, db, bal, nil, balCodeWarmupNone, 1) + require.ErrorIs(t, err, context.Canceled) +} + +func TestBlockReadAheaderWarmsOverlayBlockAccessList(t *testing.T) { + oldReadAhead := dbg.ReadAhead + dbg.SetReadAhead(true) + t.Cleanup(func() { dbg.SetReadAhead(oldReadAhead) }) + ctx := t.Context() + dirs := datadir.New(t.TempDir()) + db := temporaltest.NewTestDB(t, dirs) + address := common.Address{19: 0x42} + account := accounts.Account{ + Nonce: 1, + Balance: *uint256.NewInt(1), + CodeHash: accounts.EmptyCodeHash, + } + accountBytes := accounts.SerialiseV3(&account) + rwTx, err := db.BeginTemporalRw(ctx) + require.NoError(t, err) + defer rwTx.Rollback() + domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) + require.NoError(t, err) + domains.SetTxNum(1) + require.NoError(t, domains.DomainPut(kv.AccountsDomain, rwTx, address[:], accountBytes, 1, nil)) + require.NoError(t, domains.Commit(ctx, rwTx)) + domains.Close() + bal := types.BlockAccessList{{Address: accounts.InternAddress(address)}} + balBytes, err := types.EncodeBlockAccessListBytes(bal) + require.NoError(t, err) + balHash := bal.Hash() + header := &types.Header{ + Number: *uint256.NewInt(1), + BlockAccessListHash: &balHash, + } + body := new(types.Body) + baseTx, err := db.BeginTemporalRo(ctx) + require.NoError(t, err) + defer baseTx.Rollback() + overlay, err := membatchwithdb.NewMemoryBatch(baseTx, dirs.Tmp, log.New()) + require.NoError(t, err) + require.NoError(t, rawdb.WriteBlockAccessListBytes(overlay, header.Hash(), header.Number.Uint64(), balBytes)) + // The regression requires the BAL to be present only in BlockOverlay. + require.NoError(t, db.View(ctx, func(tx kv.Tx) error { + stored, err := rawdb.ReadBlockAccessListBytes(tx, header.Hash(), header.Number.Uint64()) + require.NoError(t, err) + require.Empty(t, stored) + return nil + })) + stateCache := newTestStateCache() + readAheader := NewBlockReadAheader() + readAheader.SetStateCache(stateCache) + readAheader.AddHeaderAndBody(ctx, db, overlay, header, body) + // AddHeaderAndBody must have copied the sidecar synchronously; its caller may + // release the overlay before the asynchronous state warming completes. + overlay.Close() + baseTx.Rollback() + readAheader.WaitForWarmup(ctx) + got, ok := stateCache.View(nil).Get(kv.AccountsDomain, address[:]) + require.True(t, ok, "overlay-only BAL account was not warmed") + require.Equal(t, accountBytes, got) +} + func TestBlockReadAheaderCarriesBlockAccessList(t *testing.T) { bra := NewBlockReadAheader() header := &types.Header{Number: *uint256.NewInt(1)} @@ -60,7 +320,7 @@ func TestBlockReadAheaderCarriesBlockAccessList(t *testing.T) { blockHash := header.Hash() bal := []byte{0xc0} sender := common.Address{1} - bra.AddHeaderAndBody(context.Background(), nil, header, body) + bra.AddHeaderAndBody(context.Background(), nil, nil, header, body) bra.AddBlockAccessList(blockHash, bal) bra.AddSenders(sender[:], blockHash) block, ok := bra.ReadBlockWithSenders(blockHash) @@ -68,6 +328,43 @@ func TestBlockReadAheaderCarriesBlockAccessList(t *testing.T) { require.Equal(t, bal, block.BlockAccessList()) } +func TestBlockReadAheaderPrefersCachedBlockAccessList(t *testing.T) { + oldReadAhead := dbg.ReadAhead + dbg.SetReadAhead(true) + t.Cleanup(func() { dbg.SetReadAhead(oldReadAhead) }) + db := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + bal := make(types.BlockAccessList, 0) + balBytes, err := types.EncodeBlockAccessListBytes(bal) + require.NoError(t, err) + balHash := bal.Hash() + header := &types.Header{Number: *uint256.NewInt(1), BlockAccessListHash: &balHash} + bra := NewBlockReadAheader() + bra.AddBlockAccessList(header.Hash(), balBytes) + getter := new(countingGetter) + bra.AddHeaderAndBody(t.Context(), db, getter, header, new(types.Body)) + bra.WaitForWarmup(t.Context()) + require.Zero(t, getter.getOneCalls) +} + +func TestBlockReadAheaderSkipsBlockAccessListReadWhenDisabledOrAbsent(t *testing.T) { + oldReadAhead := dbg.ReadAhead + dbg.SetReadAhead(false) + t.Cleanup(func() { dbg.SetReadAhead(oldReadAhead) }) + db := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + getter := new(countingGetter) + bal := make(types.BlockAccessList, 0) + balHash := bal.Hash() + headerWithBAL := &types.Header{Number: *uint256.NewInt(1), BlockAccessListHash: &balHash} + bra := NewBlockReadAheader() + bra.AddHeaderAndBody(t.Context(), db, getter, headerWithBAL, new(types.Body)) + require.Zero(t, getter.getOneCalls, "READ_AHEAD=false must skip the BAL lookup") + dbg.SetReadAhead(true) + headerWithoutBAL := &types.Header{Number: *uint256.NewInt(2)} + bra.AddHeaderAndBody(t.Context(), db, getter, headerWithoutBAL, new(types.Body)) + bra.WaitForWarmup(t.Context()) + require.Zero(t, getter.getOneCalls, "pre-Amsterdam blocks must skip the BAL lookup") +} + // seedFill places an entry with an exact txNum stamp through the public fill // API without moving the applied frontier. func seedFill(sc *cache.StateCache, domain kv.Domain, k, v []byte, txNum uint64) { @@ -115,6 +412,38 @@ func TestCachePopulatingGetterKeepsFresherCodeBinding(t *testing.T) { require.Equal(t, freshCode, got, "warmup must not rebind addr to older code") } +func TestCachePopulatingGetterReusesCodeByHashAcrossGetters(t *testing.T) { + code := []byte{0xaa, 0x01, 0x02, 0x03} + account := accounts.Account{Nonce: 1, CodeHash: accounts.InternCodeHash(crypto.Keccak256Hash(code))} + source := &sharedCodeTemporalGetter{account: accounts.SerialiseV3(&account), code: code} + stateCache := newTestStateCache() + frontier := cache.FrontierFunc(func(kv.Domain) (uint64, bool) { return 16, true }) + firstAddress := accounts.InternAddress(common.Address{19: 1}) + secondAddress := accounts.InternAddress(common.Address{19: 2}) + firstReader := state.NewReaderV3(&cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16}) + gotAccount, err := firstReader.ReadAccountData(firstAddress) + require.NoError(t, err) + require.Equal(t, account.CodeHash, gotAccount.CodeHash) + gotCode, err := firstReader.ReadAccountCode(firstAddress) + require.NoError(t, err) + require.Equal(t, code, gotCode) + accountReader := state.NewReaderV3(&cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16}) + gotAccount, err = accountReader.ReadAccountData(secondAddress) + require.NoError(t, err) + require.Equal(t, account.CodeHash, gotAccount.CodeHash) + codeGetter := &cachePopulatingGetter{TemporalGetter: source, view: stateCache.View(frontier), stepSize: 16} + codeReader := state.NewReaderV3(codeGetter) + gotCode, err = codeReader.ReadAccountCode(secondAddress) + require.NoError(t, err) + require.Equal(t, code, gotCode) + require.Equal(t, 2, source.accountReads, "the code phase must reuse the account read from the state phase") + require.Equal(t, 1, source.codeReads, "identical code must be loaded from the address-keyed domain only once") + secondAddressValue := secondAddress.Value() + boundCode, ok := stateCache.View(nil).Get(kv.CodeDomain, secondAddressValue[:]) + require.True(t, ok, "the code-hash fast path must bind code to the second address") + require.Equal(t, code, boundCode) +} + // Cold keys must still be warmed — that is the prefetcher's purpose. func TestCachePopulatingGetterWarmsColdKeys(t *testing.T) { key := []byte("\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00\x11\x22\x33\x44") diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 9a5b3204633..d999117f9bd 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -473,7 +473,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b }, nil } defer e.semaphore.Release(1) - e.hook.LastNewBlockSeen(blockNumber) // used by eth_syncing e.currentContext.ResetPendingUpdates() e.forkValidator.ClearWithUnwind() @@ -503,7 +502,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if err != nil { return ValidationResult{}, err } - e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) + e.readAheader.AddHeaderAndBody(ctx, e.db, overlay, header, body) currentBlockNumber = rawdb.ReadCurrentBlockNumber(overlay) } else { if err := e.db.View(ctx, func(tx kv.Tx) error { @@ -511,12 +510,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if err != nil { return err } - body, err = e.blockReader.BodyWithTransactions(ctx, tx, blockHash, blockNumber) if err != nil { return err } - e.readAheader.AddHeaderAndBody(ctx, e.db, header, body) + e.readAheader.AddHeaderAndBody(ctx, e.db, tx, header, body) currentBlockNumber = rawdb.ReadCurrentBlockNumber(tx) return nil }); err != nil { @@ -529,14 +527,12 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b ValidationStatus: ExecutionStatusMissingSegment, }, nil } - if math.AbsoluteDifference(*currentBlockNumber, blockNumber) >= e.syncCfg.MaxReorgDepth { return ValidationResult{ ValidationStatus: ExecutionStatusTooFarAway, LatestValidHash: common.Hash{}, }, nil } - // Use the overlay-as-rwTx pattern: the validation pipeline writes through // a fresh BlockOverlay on a new SharedDomains. This mirrors updateForkChoice // (forkchoice.go:239-251) and is required by the parallel exec path — @@ -550,7 +546,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } defer roTx.Rollback() - doms, err := execctx.NewSharedDomains(ctx, roTx, e.logger) if err != nil { return ValidationResult{}, err @@ -559,13 +554,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // forkValidator.sharedDom inside ValidatePayload and later phases close it, // so we Close explicitly only on the early-return error paths below. doms.SetInMemHistoryReads(inMemHistoryReads) - if err := doms.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { doms.Close() return ValidationResult{}, fmt.Errorf("ValidateChain: init block overlay: %w", err) } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain the validation SD to the canonical generation (e.currentContext) for // any payload with a parent, not just head-extending ones: head-extending // payloads read its not-yet-committed domain state instead of stale MDBX, and @@ -575,7 +568,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if e.currentContext != nil { doms.SetParent(e.currentContext) } - // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into // the validation overlay so unwindToCommonCanonical and ValidatePayload — // and the parallel exec goroutine via NewReadView — see this block data. @@ -589,7 +581,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, fmt.Errorf("ValidateChain: flush overlay to validation tx: %w", err) } } - // Set state cache in SharedDomains for use during state reading doms.SetStateCache(e.stateCache) doms.SetCodeStore(e.codeStore) @@ -597,28 +588,23 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b doms.Close() return ValidationResult{}, err } - status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError } - // No cache invalidation needed on an invalid payload: the state cache is // populated only at flush (committed, fork-agnostic state) and this // validation path never flushes, so a rejected payload leaves nothing // fork-specific in the cache. Reads during validation only add canonical // committed bytes. (Cache invalidation happens solely on unwind.) - // Validation tx is the SD's BlockOverlay; defer doms.Close() above handles // its rollback. By design we do not persist validation-run writes — there // is no Flush/Commit on this path. - validationStatus := ExecutionStatusSuccess if status == engine_types.AcceptedStatus { validationStatus = ExecutionStatusMissingSegment } isInvalidChain := status == engine_types.InvalidStatus || status == engine_types.InvalidBlockHashStatus || validationError != nil - // Only open a second tx when we actually need to write (bad-chain purge). // On the valid-chain path (the common case at tip) opening + empty-committing // a second RwTx just produces no-op commits with openTxs>=2, pinning freelist @@ -629,7 +615,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } defer purgeTx.Rollback() - if (lvh != common.Hash{}) && lvh != blockHash { if err := e.purgeBadChain(ctx, purgeTx, lvh, blockHash); err != nil { return ValidationResult{}, err @@ -649,7 +634,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b e.nextForkActivated = true e.logger.Info(nextForkBanner) } - result := ValidationResult{ ValidationStatus: validationStatus, LatestValidHash: lvh, diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 684560f8c64..7f5c01e1772 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -108,6 +108,10 @@ func restoreTxNum(ctx context.Context, cfg *ExecuteBlockCfg, applyTx kv.Tx, curr return inputTxNum, maxTxNum, offsetFromBlockBeginning, blockNum, nil } +func shouldWaitForReadAhead(isValidatingBlocks bool) bool { + return dbg.ReadAheadWait && isValidatingBlocks +} + func ExecV3(ctx context.Context, execStage *StageState, u Unwinder, cfg ExecuteBlockCfg, doms *execctx.SharedDomains, rwTx kv.TemporalRwTx, @@ -208,6 +212,9 @@ func ExecV3(ctx context.Context, doms.SetDeferCommitmentUpdates(true) } defer doms.SetDeferCommitmentUpdates(false) + if shouldWaitForReadAhead(isForkValidation) && cfg.readAheader != nil { + cfg.readAheader.WaitForWarmup(ctx) + } // snapshots are often stored on chaper drives. don't expect low-read-latency and manually read-ahead. // can't use OS-level ReadAhead - because Data >> RAM // it also warmsup state a bit - by touching senders/coninbase accounts and code diff --git a/execution/stagedsync/exec3_read_ahead_test.go b/execution/stagedsync/exec3_read_ahead_test.go new file mode 100644 index 00000000000..4cbf58682ae --- /dev/null +++ b/execution/stagedsync/exec3_read_ahead_test.go @@ -0,0 +1,20 @@ +package stagedsync + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/dbg" +) + +func TestShouldWaitForReadAhead(t *testing.T) { + oldReadAheadWait := dbg.ReadAheadWait + t.Cleanup(func() { dbg.ReadAheadWait = oldReadAheadWait }) + dbg.ReadAheadWait = false + require.False(t, shouldWaitForReadAhead(false)) + require.False(t, shouldWaitForReadAhead(true)) + dbg.ReadAheadWait = true + require.False(t, shouldWaitForReadAhead(false)) + require.True(t, shouldWaitForReadAhead(true)) +}