cl/beacon: prime the execution layer before a slot this node proposes

cl/beacon/handler/block_production.go

@@ -46,6 +46,7 @@ import (
 	"github.com/erigontech/erigon/cl/gossip"
 	"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
 	"github.com/erigontech/erigon/cl/phase1/core/state"
+	"github.com/erigontech/erigon/cl/phase1/execution_client"
 	"github.com/erigontech/erigon/cl/phase1/forkchoice"
 	"github.com/erigontech/erigon/cl/phase1/network/subnets"
 	"github.com/erigontech/erigon/cl/pool"
@@ -208,16 +209,20 @@ func attestationDue(cfg *clparams.BeaconChainConfig, stateVersion clparams.State
 	return time.Duration(cfg.AttestationDueMs(stateVersion.AfterOrEqual(clparams.GloasVersion))) * time.Millisecond
 }
 
-// computeBlockBuilderWindow returns when to first poll for the assembled payload and when to stop,
-// reserving a publication margin before the attestation deadline (see payloadPublicationDivisor).
-func computeBlockBuilderWindow(now, slotStart time.Time, cfg *clparams.BeaconChainConfig, stateVersion clparams.StateVersion) blockBuilderWindow {
+// computeBlockBuilderWindow uses the earlier collection window only for a sufficiently warmed
+// builder. Collecting stops the builder, so the earlier window trades the transactions that would
+// have arrived during the rest of the slot for the time it takes to sign, publish and gossip the
+// block before attesters vote on it.
+func computeBlockBuilderWindow(now, slotStart time.Time, cfg *clparams.BeaconChainConfig, stateVersion clparams.StateVersion, prepared bool) blockBuilderWindow {
 	due := attestationDue(cfg, stateVersion)
-	grabBy := slotStart.Add(due - due/payloadPublicationDivisor)
-	firstGetAt := grabBy.Add(-minPayloadPollingWindow)
+	pollUntil := slotStart.Add(unpreparedGrabOffset(due))
+	firstGetAt := pollUntil.Add(-minPayloadPollingWindow)
+	if prepared {
+		firstGetAt = slotStart.Add(preparedGrabOffset(due)).Add(-minPayloadPollingWindow)
+	}
 	if firstGetAt.Before(now) {
 		firstGetAt = now
 	}
-	pollUntil := grabBy
 	if pollUntil.Before(firstGetAt) {
 		pollUntil = firstGetAt
 	}
@@ -227,6 +232,100 @@ func computeBlockBuilderWindow(now, slotStart time.Time, cfg *clparams.BeaconCha
 	}
 }
 
+// unpreparedGrabOffset is when production starts polling for a payload it did not prime, and
+// preparedGrabOffset is when it may start for one it did. Their difference is the warm-up a
+// primed builder must already have, so the two paths give a builder the same total build time.
+func unpreparedGrabOffset(due time.Duration) time.Duration {
+	return due - due/payloadPublicationDivisor
+}
+
+func preparedGrabOffset(due time.Duration) time.Duration {
+	return due / payloadPublicationDivisor
+}
+
+func preparedPayloadMinimumAge(cfg *clparams.BeaconChainConfig, stateVersion clparams.StateVersion) time.Duration {
+	due := attestationDue(cfg, stateVersion)
+	return max(unpreparedGrabOffset(due)-preparedGrabOffset(due), 0)
+}
+
+const forkChoiceBusyRetryDelay = 100 * time.Millisecond
+
+// forkChoiceUpdateForProposal bounds acquiring a payload id by the moment the payload has to be
+// collected, so neither the retries here nor the assemble retry below them can eat the margin the
+// block needs to reach attesters. One attempt always runs past that bound, because failing a
+// proposal outright is worse than answering late. The head is not re-read between attempts: a
+// proposal is committed to the parent it is being built on.
+func (a *ApiHandler) forkChoiceUpdateForProposal(
+	ctx context.Context,
+	targetSlot uint64,
+	finalized, safe, head common.Hash,
+	attrs *engine_types.PayloadAttributes,
+	stateVersion clparams.StateVersion,
+) ([]byte, error) {
+	collectAt := a.ethClock.GetSlotTime(targetSlot).Add(unpreparedGrabOffset(attestationDue(a.beaconChainCfg, stateVersion)))
+	boundedCtx, cancel := context.WithDeadline(ctx, collectAt)
+	defer cancel()
+
+	for time.Now().Before(collectAt) {
+		idBytes, err := a.engine.ForkChoiceUpdate(boundedCtx, finalized, safe, head, attrs, stateVersion)
+		switch {
+		case err == nil:
+			return idBytes, nil
+		case ctx.Err() != nil:
+			return nil, ctx.Err()
+		case !errors.Is(err, execution_client.ErrForkChoiceBusy) && !errors.Is(err, context.DeadlineExceeded):
+			return nil, err
+		}
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(forkChoiceBusyRetryDelay):
+		}
+	}
+	return a.engine.ForkChoiceUpdate(ctx, finalized, safe, head, attrs, stateVersion)
+}
+
+// payloadAttributes builds the attributes preparation and production must agree on. Withdrawals and
+// the parent beacon block root are set for every fork because the execution layer decides what to do
+// with them from the payload timestamp; gating them on the consensus fork instead puts the two sides
+// on different oracles, which disagree on any chain where the forks are not aligned.
+func payloadAttributes(
+	timestamp hexutil.Uint64,
+	prevRandao common.Hash,
+	feeRecipient common.Address,
+	withdrawals []*types.Withdrawal,
+	parentRoot *common.Hash,
+) *engine_types.PayloadAttributes {
+	return &engine_types.PayloadAttributes{
+		Timestamp:             timestamp,
+		PrevRandao:            prevRandao,
+		SuggestedFeeRecipient: feeRecipient,
+		Withdrawals:           withdrawals,
+		ParentBeaconBlockRoot: parentRoot,
+	}
+}
+
+// gloasWithdrawals reads the withdrawals for a Gloas proposal from the state copy carrying the
+// parent payload when the head is FULL, and from the cached expectation when it is EMPTY.
+func gloasWithdrawals(baseState, withParentPayload *state.CachingBeaconState, epoch uint64) ([]*types.Withdrawal, error) {
+	if withParentPayload != nil {
+		clWithdrawals, err := state.GetExpectedWithdrawals(withParentPayload, epoch)
+		if err != nil {
+			return nil, err
+		}
+		return cltypes.ConvertConsensusWithdrawalsToExecutionWithdrawals(clWithdrawals.Withdrawals), nil
+	}
+	cached := baseState.GetPayloadExpectedWithdrawals()
+	if cached == nil {
+		return nil, nil
+	}
+	consensusWithdrawals := make([]*cltypes.Withdrawal, cached.Len())
+	for i := range consensusWithdrawals {
+		consensusWithdrawals[i] = cached.Get(i)
+	}
+	return cltypes.ConvertConsensusWithdrawalsToExecutionWithdrawals(consensusWithdrawals), nil
+}
+
 func shouldRetryGetPayload(now, deadline time.Time) bool {
 	return now.Before(deadline)
 }
@@ -252,12 +351,30 @@ func pollAssembledPayload(
 	defer deadlineTimer.Stop()
 	retryTicker := time.NewTicker(retryTime)
 	defer retryTicker.Stop()
+	// A slot that fails tends to fail on every poll of the window, so report the first and then a
+	// count rather than hundreds of copies of one line. Contention that clears is not worth an
+	// alert, so the summary is only for a window that never produced anything.
+	var (
+		failures  int
+		lastErr   error
+		collected bool
+	)
+	defer func() {
+		if failures > 1 && !collected {
+			log.Error("BlockProduction: payload polling kept failing", "attempts", failures, "err", lastErr)
+		}
+	}()
 	for {
 		// Grab at least once, even past the deadline, so a late produce request still gets a payload.
 		payload, bundles, requestsBundle, blockValue, err := get()
 		if err != nil {
-			log.Error("BlockProduction: Failed to get payload", "err", err)
+			failures++
+			lastErr = err
+			if failures == 1 {
+				log.Error("BlockProduction: Failed to get payload", "err", err)
+			}
 		} else if payload != nil {
+			collected = true
 			return payload, bundles, requestsBundle, blockValue, true
 		}
 		select {
@@ -473,10 +590,6 @@ func (a *ApiHandler) GetEthV3ValidatorBlock(
 	}); err != nil {
 		return nil, err
 	}
-
-	if err != nil {
-		return nil, err
-	}
 	if baseState == nil {
 		return nil, beaconhttp.NewEndpointError(
 			http.StatusNotFound,
@@ -863,6 +976,9 @@ func (a *ApiHandler) produceBeaconBody(
 			baseBlockSlot,
 		)
 	}
+	a.proposalsInFlight.Add(1)
+	defer a.proposalsInFlight.Add(-1)
+
 	var wg sync.WaitGroup
 	stateVersion := a.beaconChainCfg.GetCurrentStateVersion(
 		targetSlot / a.beaconChainCfg.SlotsPerEpoch,
@@ -924,14 +1040,7 @@ func (a *ApiHandler) produceBeaconBody(
 			head = baseState.GetLatestBlockHash()
 		}
 	}
-	finalizedHash := a.forkchoiceStore.GetFinalizedExecutionHash(baseState.FinalizedCheckpoint().Root)
-	if finalizedHash == (common.Hash{}) {
-		finalizedHash = head
-	}
-	safeHash := a.forkchoiceStore.GetFinalizedExecutionHash(baseState.CurrentJustifiedCheckpoint().Root)
-	if safeHash == (common.Hash{}) {
-		safeHash = head
-	}
+	safeHash, finalizedHash := a.executionCheckpointHashes(baseState, head)
 	proposerIndex, err := baseState.GetBeaconProposerIndexForSlot(targetSlot)
 	if err != nil {
 		return nil, 0, err
@@ -953,8 +1062,8 @@ func (a *ApiHandler) produceBeaconBody(
 			}
 		}
 	}
-	currEpoch := a.ethClock.GetCurrentEpoch()
-	random := baseState.GetRandaoMixes(currEpoch)
+	targetEpoch := targetSlot / a.beaconChainCfg.SlotsPerEpoch
+	random := baseState.GetRandaoMixes(targetEpoch)
 
 	var executionPayload *cltypes.Eth1Block
 	var executionValue uint64
@@ -975,85 +1084,46 @@ func (a *ApiHandler) produceBeaconBody(
 			log.Info("BlockProduction: ForkChoiceUpdate&GetPayload took", "duration", time.Since(start))
 		}()
 		retryTime := 10 * time.Millisecond
-		feeRecipient, _ := a.validatorParams.GetFeeRecipient(proposerIndex)
-		var withdrawals []*types.Withdrawal
-		switch {
-		case gloasWithdrawalsState != nil:
-			// GLOAS FULL: compute withdrawals from the state copy with parent payload applied
-			clWithdrawals, err := state.GetExpectedWithdrawals(
-				gloasWithdrawalsState,
-				targetSlot/a.beaconChainCfg.SlotsPerEpoch,
-			)
+		feeRecipient, registered := a.validatorParams.GetFeeRecipient(proposerIndex)
+		if !registered {
+			// Preparation treats an unregistered proposer as someone else's slot; production
+			// still builds, so make the zero-address fallback visible rather than silent.
+			log.Warn("BlockProduction: no fee recipient registered for proposer, using zero address",
+				"proposer", proposerIndex)
+		}
+		// One branch per fork: the pre-Gloas inputs are the same ones preparation builds, and the
+		// prepared-id match depends on production not deriving its own variant of them.
+		var (
+			fcuHead, fcuSafeHash, fcuFinalizedHash common.Hash
+			attrs                                  *engine_types.PayloadAttributes
+		)
+		if stateVersion.Before(clparams.GloasVersion) {
+			var err error
+			fcuHead, fcuSafeHash, fcuFinalizedHash, attrs, err = a.preGloasForkChoiceInputs(baseState, baseBlockRoot, targetSlot, feeRecipient)
 			if err != nil {
-				log.Error("BlockProduction: GetExpectedWithdrawals (FULL) failed", "err", err)
+				log.Error("BlockProduction: build forkchoice inputs failed", "err", err)
 				return
 			}
-			withdrawals = make([]*types.Withdrawal, 0, len(clWithdrawals.Withdrawals))
-			for _, w := range clWithdrawals.Withdrawals {
-				withdrawals = append(withdrawals, &types.Withdrawal{
-					Index:     w.Index,
-					Amount:    w.Amount,
-					Validator: w.Validator,
-					Address:   w.Address,
-				})
-			}
-		case stateVersion >= clparams.GloasVersion && gloasWithdrawalsState == nil:
-			// GLOAS EMPTY: use cached payload_expected_withdrawals from state
-			cachedWithdrawals := baseState.GetPayloadExpectedWithdrawals()
-			if cachedWithdrawals != nil {
-				withdrawals = make([]*types.Withdrawal, 0, cachedWithdrawals.Len())
-				for i := 0; i < cachedWithdrawals.Len(); i++ {
-					w := cachedWithdrawals.Get(i)
-					withdrawals = append(withdrawals, &types.Withdrawal{
-						Index:     w.Index,
-						Amount:    w.Amount,
-						Validator: w.Validator,
-						Address:   w.Address,
-					})
-				}
-			}
-		default:
-			// Pre-GLOAS: compute withdrawals normally
-			clWithdrawals, err := state.GetExpectedWithdrawals(
-				baseState,
-				targetSlot/a.beaconChainCfg.SlotsPerEpoch,
-			)
+		} else {
+			fcuHead, fcuSafeHash, fcuFinalizedHash = head, safeHash, finalizedHash
+			withdrawals, err := gloasWithdrawals(baseState, gloasWithdrawalsState, targetSlot/a.beaconChainCfg.SlotsPerEpoch)
 			if err != nil {
 				log.Error("BlockProduction: GetExpectedWithdrawals failed", "err", err)
 				return
 			}
-			withdrawals = make([]*types.Withdrawal, 0, len(clWithdrawals.Withdrawals))
-			for _, w := range clWithdrawals.Withdrawals {
-				withdrawals = append(withdrawals, &types.Withdrawal{
-					Index:     w.Index,
-					Amount:    w.Amount,
-					Validator: w.Validator,
-					Address:   w.Address,
-				})
-			}
-		}
-
-		attrs := &engine_types.PayloadAttributes{
-			Timestamp:             hexutil.Uint64(state.ComputeTimestampAtSlot(baseState, targetSlot)),
-			PrevRandao:            random,
-			SuggestedFeeRecipient: feeRecipient,
-			Withdrawals:           withdrawals,
-			ParentBeaconBlockRoot: (*common.Hash)(&blockRoot),
-		}
-		if stateVersion.AfterOrEqual(clparams.GloasVersion) {
-			sn := hexutil.Uint64(targetSlot)
-			attrs.SlotNumber = &sn
+			attrs = payloadAttributes(
+				hexutil.Uint64(state.ComputeTimestampAtSlot(baseState, targetSlot)),
+				random,
+				feeRecipient,
+				withdrawals,
+				(*common.Hash)(&blockRoot),
+			)
+			slotNumber := hexutil.Uint64(targetSlot)
+			attrs.SlotNumber = &slotNumber
 			attrs.TargetGasLimit = targetGasLimit
 		}
 		builderStartedAt := time.Now()
-		idBytes, err := a.engine.ForkChoiceUpdate(
-			ctx,
-			finalizedHash,
-			safeHash,
-			head,
-			attrs,
-			stateVersion,
-		)
+		idBytes, err := a.forkChoiceUpdateForProposal(ctx, targetSlot, fcuFinalizedHash, fcuSafeHash, fcuHead, attrs, stateVersion)
 		if err != nil {
 			log.Error("BlockProduction: Failed to get payload id", "err", err)
 			return
@@ -1063,7 +1133,16 @@ func (a *ApiHandler) produceBeaconBody(
 			return
 		}
 		slotStart := a.ethClock.GetSlotTime(targetSlot)
-		buildWindow := computeBlockBuilderWindow(builderStartedAt, slotStart, a.beaconChainCfg, stateVersion)
+		prepared := canUsePreparedPayload(
+			&a.preparedPayload,
+			a.engine.SupportInsertion(),
+			targetSlot,
+			idBytes,
+			time.Now(),
+			preparedPayloadMinimumAge(a.beaconChainCfg, stateVersion),
+		)
+		log.Info("BlockProduction: payload preparation", "slot", targetSlot, "prepared", prepared)
+		buildWindow := computeBlockBuilderWindow(builderStartedAt, slotStart, a.beaconChainCfg, stateVersion, prepared)
 		payload, bundles, requestsBundle, blockValue, ok := pollAssembledPayload(ctx, buildWindow, retryTime, func() (*cltypes.Eth1Block, *engine_types.BlobsBundle, *typesproto.RequestsBundle, *big.Int, error) {
 			return a.engine.GetAssembledBlock(ctx, idBytes, stateVersion)
 		})
@@ -2592,15 +2671,12 @@ func (a *ApiHandler) cacheExecutionBody(payload *cltypes.Eth1Block) {
 	}
 	var ws []*types.Withdrawal
 	if payload.Withdrawals != nil {
-		payload.Withdrawals.Range(func(idx int, w *cltypes.Withdrawal, total int) bool {
-			ws = append(ws, &types.Withdrawal{
-				Index:     w.Index,
-				Validator: w.Validator,
-				Address:   w.Address,
-				Amount:    w.Amount,
-			})
+		consensusWithdrawals := make([]*cltypes.Withdrawal, 0, payload.Withdrawals.Len())
+		payload.Withdrawals.Range(func(_ int, w *cltypes.Withdrawal, _ int) bool {
+			consensusWithdrawals = append(consensusWithdrawals, w)
 			return true
 		})
+		ws = cltypes.ConvertConsensusWithdrawalsToExecutionWithdrawals(consensusWithdrawals)
 	}
 	a.blockReader.CacheBlockBody(payload.BlockNumber, rawTxs, ws)
 }