diff --git a/packages/beacon-node/src/api/impl/beacon/index.ts b/packages/beacon-node/src/api/impl/beacon/index.ts index 0003e9c97878..9202f86bbd35 100644 --- a/packages/beacon-node/src/api/impl/beacon/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/index.ts @@ -7,7 +7,7 @@ import {getBeaconRewardsApi} from "./rewards/index.js"; import {getBeaconStateApi} from "./state/index.js"; export function getBeaconApi( - modules: Pick + modules: Pick ): ApplicationMethods { const block = getBeaconBlockApi(modules); const pool = getBeaconPoolApi(modules); diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 4b801d3532ad..2128c700a4cc 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -26,11 +26,12 @@ import { export function getBeaconStateApi({ chain, config, -}: Pick): ApplicationMethods { + sync, +}: Pick): ApplicationMethods { async function getState( stateId: routes.beacon.StateId ): Promise<{state: IBeaconStateView; executionOptimistic: boolean; finalized: boolean}> { - const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId); return { state: state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state, diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index 8b881f9f458e..2f17ad766b33 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -14,7 +14,9 @@ import { } from "@lodestar/types"; import {byteArrayEquals, fromHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../../chain/index.js"; +import {IBeaconSync} from "../../../../sync/index.js"; import {ApiError, ValidationError} from "../../errors.js"; +import {notWhileSyncing} from "../../utils.js"; export function resolveStateId( forkChoice: IForkChoice, @@ -51,8 +53,17 @@ export function resolveStateId( export async function getStateResponseWithRegen( chain: IBeaconChain, + sync: IBeaconSync, inStateId: routes.beacon.StateId ): Promise<{state: IBeaconStateView | Uint8Array; executionOptimistic: boolean; finalized: boolean}> { + // "head", "finalized" and "justified" resolve to already-available cached states, and "genesis" to a + // historical DB read - none trigger the forward regen that can walk back past the block-root window + // (SLOTS_PER_HISTORICAL_ROOT) and wedge a far-behind node. Keep serving those (node observability, + // dashboards, validator client checks) even while syncing; guard only the regen-capable lookups. + if (inStateId !== "head" && inStateId !== "finalized" && inStateId !== "justified" && inStateId !== "genesis") { + notWhileSyncing(chain, sync.state); + } + const stateId = resolveStateId(chain.forkChoice, inStateId); const res = diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index fb30bb41d270..237f7e4458c3 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -39,7 +39,8 @@ function toPayloadStatusName(status: PayloadStatus): "pending" | "empty" | "full export function getDebugApi({ chain, config, -}: Pick): ApplicationMethods { + sync, +}: Pick): ApplicationMethods { return { async getDebugChainHeadsV2() { const heads = chain.forkChoice.getHeads(); @@ -132,7 +133,7 @@ export function getDebugApi({ }, async getStateV2({stateId}, context) { - const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId); let slot: number, data: Uint8Array | BeaconState; if (state instanceof Uint8Array) { slot = getStateSlotFromBytes(state); diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index a283c0ca5ab5..c73cc0fc1046 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -217,7 +217,7 @@ export function getLodestarApi({ }, async getHistoricalSummaries({stateId}) { - const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId); const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state; @@ -332,7 +332,7 @@ export function getLodestarApi({ for (const [epoch, attestationsPerEpoch] of attestations) { const slot = computeStartSlotAtEpoch(epoch); - const {state} = await getStateResponseWithRegen(chain, slot); + const {state} = await getStateResponseWithRegen(chain, sync, slot); const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state; const shuffling = stateView.getShufflingAtEpoch(epoch); for (const attestation of attestationsPerEpoch) { diff --git a/packages/beacon-node/src/api/impl/proof/index.ts b/packages/beacon-node/src/api/impl/proof/index.ts index 913114b2f19e..2074c3c8d066 100644 --- a/packages/beacon-node/src/api/impl/proof/index.ts +++ b/packages/beacon-node/src/api/impl/proof/index.ts @@ -8,7 +8,7 @@ import {ApiModules} from "../types.js"; export function getProofApi( opts: ApiOptions, - {chain, config}: Pick + {chain, config, sync}: Pick ): ApplicationMethods { // It's currently possible to request gigantic proofs (eg: a proof of the entire beacon state) // We want some some sort of resistance against this DoS vector. @@ -21,7 +21,7 @@ export function getProofApi( throw new Error("Requested proof is too large."); } - const res = await getStateResponseWithRegen(chain, stateId); + const res = await getStateResponseWithRegen(chain, sync, stateId); const state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state; diff --git a/packages/beacon-node/src/api/impl/utils.ts b/packages/beacon-node/src/api/impl/utils.ts index 86e273c4a041..f69ec1edbf92 100644 --- a/packages/beacon-node/src/api/impl/utils.ts +++ b/packages/beacon-node/src/api/impl/utils.ts @@ -1,4 +1,54 @@ -import {ApiError} from "./errors.js"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import type {IBeaconChain} from "../../chain/index.js"; +import {SyncState} from "../../sync/index.js"; +import {ApiError, NodeIsSyncing} from "./errors.js"; + +/** + * If the node is within this many epochs from the head, we declare it to be synced regardless of + * the network sync state. + * + * This helps prevent attacks where nodes can convince us that we're syncing some non-existent + * finalized head. + * + * TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators + * can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets + * a new head. Then the checkpoint state cache grows unbounded with very different states (because + * they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state + * caches are better. + */ +export const SYNC_TOLERANCE_EPOCHS = 1; + +/** + * Reject any request while the node is syncing. Used by endpoints that must not serve while the + * node is behind — validator duties, and beacon state lookups whose regen could otherwise walk + * back past the block-root window (`SLOTS_PER_HISTORICAL_ROOT`) and wedge the node. Throws + * {@link NodeIsSyncing} (503). + */ +export function notWhileSyncing(chain: IBeaconChain, syncState: SyncState): void { + // Consider node synced before or close to genesis + if (chain.clock.currentSlot < SLOTS_PER_EPOCH) { + return; + } + + switch (syncState) { + case SyncState.SyncingFinalized: + case SyncState.SyncingHead: { + const currentSlot = chain.clock.currentSlot; + const headSlot = chain.forkChoice.getHead().slot; + if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) { + throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`); + } + + return; + } + + case SyncState.Synced: + return; + + case SyncState.Stalled: + throw new NodeIsSyncing("waiting for peers"); + } +} /** * Ensures that the array contains unique values, and throws an ApiError diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 67cb38023c1e..b01099eca952 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -82,7 +82,6 @@ import {ZERO_HASH} from "../../../constants/index.js"; import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js"; import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; import {CommitteeSubscription} from "../../../network/subnets/index.js"; -import {SyncState} from "../../../sync/index.js"; import {callInNextEventLoop} from "../../../util/eventLoop.js"; import {isOptimisticBlock} from "../../../util/forkChoice.js"; import {getBlockGraffiti, toGraffitiBytes} from "../../../util/graffiti.js"; @@ -91,23 +90,9 @@ import {ApiOptions} from "../../options.js"; import {getStateResponseWithRegen} from "../beacon/state/utils.js"; import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js"; import {ApiModules} from "../types.js"; +import {notWhileSyncing} from "../utils.js"; import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js"; -/** - * If the node is within this many epochs from the head, we declare it to be synced regardless of - * the network sync state. - * - * This helps prevent attacks where nodes can convince us that we're syncing some non-existent - * finalized head. - * - * TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators - * can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets - * a new head. Then the checkpoint state cache grows unbounded with very different states (because - * they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state - * caches are better. - */ -export const SYNC_TOLERANCE_EPOCHS = 1; - /** * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve. * Post this time, race execution and builder to pick whatever resolves first. @@ -341,36 +326,6 @@ export function getValidatorApi( return null; } - /** - * Reject any request while the node is syncing - */ - function notWhileSyncing(): void { - // Consider node synced before or close to genesis - if (chain.clock.currentSlot < SLOTS_PER_EPOCH) { - return; - } - - const syncState = sync.state; - switch (syncState) { - case SyncState.SyncingFinalized: - case SyncState.SyncingHead: { - const currentSlot = chain.clock.currentSlot; - const headSlot = chain.forkChoice.getHead().slot; - if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) { - throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`); - } - - return; - } - - case SyncState.Synced: - return; - - case SyncState.Stalled: - throw new NodeIsSyncing("waiting for peers"); - } - } - /** * Post merge, the CL and EL could be out of step in the sync, and could result in * Syncing status of the chain head. To be precise: @@ -573,7 +528,7 @@ export function getValidatorApi( builderBoostFactor?: bigint, {feeRecipient, builderSelection, strictFeeRecipientCheck}: routes.validator.ExtraProduceBlockOpts = {} ): Promise { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); // Must never request for a future slot > currentSlot const parentBlock = chain.getProposerHead(slot); @@ -917,7 +872,7 @@ export function getValidatorApi( throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`); } - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); const parentBlock = chain.getProposerHead(slot); @@ -1035,7 +990,7 @@ export function getValidatorApi( }, async produceAttestationData({committeeIndex, slot}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); // Must never request for a future slot > currentSlot @@ -1116,7 +1071,7 @@ export function getValidatorApi( throw new ApiError(400, `producePayloadAttestationData is not supported before Gloas fork=${fork}`); } - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); const block = chain.forkChoice.getCanonicalBlockAtSlot(slot); @@ -1201,7 +1156,7 @@ export function getValidatorApi( }, async getProposerDuties({epoch}, _context, opts?: {v2?: boolean}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); const currentEpoch = currentEpochWithDisparity(); const nextEpoch = currentEpoch + 1; @@ -1248,7 +1203,7 @@ export function getValidatorApi( // requested epoch is within that range, we can use the head state at current epoch state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties); } else { - const res = await getStateResponseWithRegen(chain, startSlot); + const res = await getStateResponseWithRegen(chain, sync, startSlot); state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state; @@ -1342,7 +1297,7 @@ export function getValidatorApi( }, async getAttesterDuties({epoch, indices}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); if (indices.length === 0) { throw new ApiError(400, "No validator to get attester duties"); @@ -1402,7 +1357,7 @@ export function getValidatorApi( }, async getPtcDuties({epoch, indices}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); if (indices.length === 0) { throw new ApiError(400, "No validator to get PTC duties"); @@ -1464,7 +1419,7 @@ export function getValidatorApi( * @param validatorIndices an array of the validator indices for which to obtain the duties. */ async getSyncCommitteeDuties({epoch, indices}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); if (indices.length === 0) { throw new ApiError(400, "No validator to get attester duties"); @@ -1511,7 +1466,7 @@ export function getValidatorApi( }, async getAggregatedAttestationV2({attestationDataRoot, slot, committeeIndex}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); // Must never request for a future slot > currentSlot @@ -1534,7 +1489,7 @@ export function getValidatorApi( }, async publishAggregateAndProofsV2({signedAggregateAndProofs}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); const seenTimestampSec = Date.now() / 1000; const failures: FailureList = []; @@ -1595,7 +1550,7 @@ export function getValidatorApi( * https://github.com/ethereum/beacon-APIs/pull/137 */ async publishContributionAndProofs({contributionAndProofs}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); const failures: FailureList = []; @@ -1644,7 +1599,7 @@ export function getValidatorApi( }, async prepareBeaconCommitteeSubnet({subscriptions}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await network.prepareBeaconCommitteeSubnets( subscriptions.map(({validatorIndex, slot, isAggregator, committeesAtSlot, committeeIndex}) => ({ @@ -1677,7 +1632,7 @@ export function getValidatorApi( * https://github.com/ethereum/beacon-APIs/pull/136 */ async prepareSyncCommitteeSubnets({subscriptions}) { - notWhileSyncing(); + notWhileSyncing(chain, sync.state); // A `validatorIndex` can be in multiple subnets, so compute the CommitteeSubscription with double for loop const subs: CommitteeSubscription[] = []; @@ -1821,7 +1776,7 @@ export function getValidatorApi( throw new ApiError(400, `getExecutionPayloadEnvelope not supported for pre-gloas fork=${fork}`); } - notWhileSyncing(); + notWhileSyncing(chain, sync.state); await waitForSlot(slot); const blockRootHex = toRootHex(beaconBlockRoot); diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index 6d1aee4581a3..36ff1bd61157 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -1,7 +1,11 @@ import {describe, expect, it} from "vitest"; import {toHexString} from "@chainsafe/ssz"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BeaconStateView} from "@lodestar/state-transition"; -import {getStateValidatorIndex} from "../../../../../../src/api/impl/beacon/state/utils.js"; +import {getStateResponseWithRegen, getStateValidatorIndex} from "../../../../../../src/api/impl/beacon/state/utils.js"; +import {NodeIsSyncing} from "../../../../../../src/api/impl/errors.js"; +import {IBeaconChain} from "../../../../../../src/chain/index.js"; +import {IBeaconSync, SyncState} from "../../../../../../src/sync/index.js"; import {generateCachedAltairState} from "../../../../../utils/state.js"; describe("beacon state api utils", () => { @@ -61,4 +65,35 @@ describe("beacon state api utils", () => { } }); }); + + describe("getStateResponseWithRegen", () => { + // Node far behind head and syncing. Chain getters return a dummy response so the resolved + // "already-available" state ids don't blow up; the regen paths are never reached in these tests. + const servedResponse = {state: {}, executionOptimistic: false, finalized: false}; + const syncingChain = { + clock: {currentSlot: 10 * SLOTS_PER_EPOCH}, + forkChoice: { + getHead: () => ({slot: 0, stateRoot: "0x00"}), + getFinalizedCheckpoint: () => ({rootHex: "0x00", epoch: 0}), + getJustifiedCheckpoint: () => ({rootHex: "0x00", epoch: 0}), + getFinalizedBlock: () => ({slot: 0}), + }, + getStateByStateRoot: () => Promise.resolve(servedResponse), + getStateOrBytesByCheckpoint: () => Promise.resolve(servedResponse), + getStateBySlot: () => Promise.resolve(servedResponse), + getHistoricalStateBySlot: () => Promise.resolve(servedResponse), + } as unknown as IBeaconChain; + const sync = {state: SyncState.SyncingFinalized} as unknown as IBeaconSync; + + it("throws NodeIsSyncing for a regen-triggering slot while the node is behind and syncing", async () => { + // notWhileSyncing runs before any regen, so a minimal clock/forkChoice/sync mock is enough + await expect(getStateResponseWithRegen(syncingChain, sync, 5 * SLOTS_PER_EPOCH)).rejects.toThrow(NodeIsSyncing); + }); + + it("serves head/finalized/justified/genesis while syncing (no regen wedge risk)", async () => { + for (const stateId of ["head", "finalized", "justified", "genesis"] as const) { + await expect(getStateResponseWithRegen(syncingChain, sync, stateId), stateId).resolves.toBeDefined(); + } + }); + }); }); diff --git a/packages/beacon-node/test/unit/api/impl/validator/duties/proposer.test.ts b/packages/beacon-node/test/unit/api/impl/validator/duties/proposer.test.ts index 82eb0485847c..2d257d399d4e 100644 --- a/packages/beacon-node/test/unit/api/impl/validator/duties/proposer.test.ts +++ b/packages/beacon-node/test/unit/api/impl/validator/duties/proposer.test.ts @@ -5,7 +5,8 @@ import {ForkName, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params import {BeaconStateAllForks, BeaconStateView} from "@lodestar/state-transition"; import {Slot} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; -import {SYNC_TOLERANCE_EPOCHS, getValidatorApi} from "../../../../../../src/api/impl/validator/index.js"; +import {SYNC_TOLERANCE_EPOCHS} from "../../../../../../src/api/impl/utils.js"; +import {getValidatorApi} from "../../../../../../src/api/impl/validator/index.js"; import {defaultApiOptions} from "../../../../../../src/api/options.js"; import {FAR_FUTURE_EPOCH} from "../../../../../../src/constants/index.js"; import {SyncState} from "../../../../../../src/sync/interface.js";