Skip to content

fix: reject beacon state requests while syncing to avoid regen wedge #9641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lodekeeper wants to merge 3 commits into
base: unstable
Choose a base branch
from
Open

fix: reject beacon state requests while syncing to avoid regen wedge #9641

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
995c51d
fix: reject beacon state requests while syncing to avoid regen wedge
lodekeeper Jul 10, 2026
d777f85
fix: bypass sync guard for always-available state ids
lodekeeper Jul 10, 2026
075f178
fix: pass sync state to API sync guard
lodekeeper Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/beacon-node/src/api/impl/beacon/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {getBeaconRewardsApi} from "./rewards/index.js";
import {getBeaconStateApi} from "./state/index.js";

export function getBeaconApi(
modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db">
modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db" | "sync">
): ApplicationMethods<routes.beacon.Endpoints> {
const block = getBeaconBlockApi(modules);
const pool = getBeaconPoolApi(modules);
Expand Down
5 changes: 3 additions & 2 deletions packages/beacon-node/src/api/impl/beacon/state/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ import {
export function getBeaconStateApi({
chain,
config,
}: Pick<ApiModules, "chain" | "config">): ApplicationMethods<routes.beacon.state.Endpoints> {
sync,
}: Pick<ApiModules, "chain" | "config" | "sync">): ApplicationMethods<routes.beacon.state.Endpoints> {
async function getState(
stateId: routes.beacon.StateId
): Promise<{state: IBeaconStateView; executionOptimistic: boolean; finalized: boolean}> {
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);

return {
state: state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state,
Expand Down
11 changes: 11 additions & 0 deletions packages/beacon-node/src/api/impl/beacon/state/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ import {
} from "@lodestar/types";
import {byteArrayEquals, fromHex} from "@lodestar/utils";
import {IBeaconChain} from "../../../../chain/index.js";
import {IBeaconSync} from "../../../../sync/index.js";
import {ApiError, ValidationError} from "../../errors.js";
import {notWhileSyncing} from "../../utils.js";

export function resolveStateId(
forkChoice: IForkChoice,
Expand Down Expand Up @@ -51,8 +53,17 @@ export function resolveStateId(

export async function getStateResponseWithRegen(
chain: IBeaconChain,
sync: IBeaconSync,
inStateId: routes.beacon.StateId
): Promise<{state: IBeaconStateView | Uint8Array; executionOptimistic: boolean; finalized: boolean}> {
// "head", "finalized" and "justified" resolve to already-available cached states, and "genesis" to a
// historical DB read - none trigger the forward regen that can walk back past the block-root window
// (SLOTS_PER_HISTORICAL_ROOT) and wedge a far-behind node. Keep serving those (node observability,
// dashboards, validator client checks) even while syncing; guard only the regen-capable lookups.
if (inStateId !== "head" && inStateId !== "finalized" && inStateId !== "justified" && inStateId !== "genesis") {
notWhileSyncing(chain, sync);
}

const stateId = resolveStateId(chain.forkChoice, inStateId);

const res =
Expand Down
5 changes: 3 additions & 2 deletions packages/beacon-node/src/api/impl/debug/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ function toPayloadStatusName(status: PayloadStatus): "pending" | "empty" | "full
export function getDebugApi({
chain,
config,
}: Pick<ApiModules, "chain" | "config" | "db">): ApplicationMethods<routes.debug.Endpoints> {
sync,
}: Pick<ApiModules, "chain" | "config" | "db" | "sync">): ApplicationMethods<routes.debug.Endpoints> {
return {
async getDebugChainHeadsV2() {
const heads = chain.forkChoice.getHeads();
Expand Down Expand Up @@ -132,7 +133,7 @@ export function getDebugApi({
},

async getStateV2({stateId}, context) {
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);
let slot: number, data: Uint8Array | BeaconState;
if (state instanceof Uint8Array) {
slot = getStateSlotFromBytes(state);
Expand Down
4 changes: 2 additions & 2 deletions packages/beacon-node/src/api/impl/lodestar/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ export function getLodestarApi({
},

async getHistoricalSummaries({stateId}) {
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);

const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state;

Expand Down Expand Up @@ -332,7 +332,7 @@ export function getLodestarApi({

for (const [epoch, attestationsPerEpoch] of attestations) {
const slot = computeStartSlotAtEpoch(epoch);
const {state} = await getStateResponseWithRegen(chain, slot);
const {state} = await getStateResponseWithRegen(chain, sync, slot);
const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state;
const shuffling = stateView.getShufflingAtEpoch(epoch);
for (const attestation of attestationsPerEpoch) {
Expand Down
4 changes: 2 additions & 2 deletions packages/beacon-node/src/api/impl/proof/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {ApiModules} from "../types.js";

export function getProofApi(
opts: ApiOptions,
{chain, config}: Pick<ApiModules, "chain" | "config" | "db">
{chain, config, sync}: Pick<ApiModules, "chain" | "config" | "db" | "sync">
): ApplicationMethods<routes.proof.Endpoints> {
// It's currently possible to request gigantic proofs (eg: a proof of the entire beacon state)
// We want some some sort of resistance against this DoS vector.
Expand All @@ -21,7 +21,7 @@ export function getProofApi(
throw new Error("Requested proof is too large.");
}

const res = await getStateResponseWithRegen(chain, stateId);
const res = await getStateResponseWithRegen(chain, sync, stateId);

const state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state;

Expand Down
52 changes: 51 additions & 1 deletion packages/beacon-node/src/api/impl/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,54 @@
import {ApiError} from "./errors.js";
import {SLOTS_PER_EPOCH} from "@lodestar/params";
import type {IBeaconChain} from "../../chain/index.js";
import {type IBeaconSync, SyncState} from "../../sync/index.js";
import {ApiError, NodeIsSyncing} from "./errors.js";

/**
* If the node is within this many epochs from the head, we declare it to be synced regardless of
* the network sync state.
*
* This helps prevent attacks where nodes can convince us that we're syncing some non-existent
* finalized head.
*
* TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators
* can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets
* a new head. Then the checkpoint state cache grows unbounded with very different states (because
* they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state
* caches are better.
*/
export const SYNC_TOLERANCE_EPOCHS = 1;

/**
* Reject any request while the node is syncing. Used by endpoints that must not serve while the
* node is behind — validator duties, and beacon state lookups whose regen could otherwise walk
* back past the block-root window (`SLOTS_PER_HISTORICAL_ROOT`) and wedge the node. Throws
* {@link NodeIsSyncing} (503).
*/
export function notWhileSyncing(chain: IBeaconChain, sync: IBeaconSync): void {

@wemeetagain wemeetagain Jul 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe this should take a sync state instead of the whole sync module?

@lodekeeper lodekeeper Jul 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed — SyncState is cleaner: narrower dep, and it drops the IBeaconSync mock in tests. Will narrow notWhileSyncing to take it.

One sequencing note so this does not get reworked: @twoeths is leaning toward not changing serving behavior at all and instead bounding the regen at the source (closer to the closed #9634 — only fail a lookup that would have to forward-regen past SLOTS_PER_HISTORICAL_ROOT, i.e. the genuinely unserveable case). If we land there, the state endpoints drop notWhileSyncing and it goes back to a validator/index.ts closure, which makes the shared-helper signature moot. So I will fold this in once the approach is settled (pending @nflaig) — good call if the shared helper stays.

@lodekeeper lodekeeper Jul 15, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good call. Updated notWhileSyncing to take SyncState directly and changed the callers to pass sync.state, so the shared API utility no longer depends on the full sync module interface.

Pushed in 075f178897.

// Consider node synced before or close to genesis
if (chain.clock.currentSlot < SLOTS_PER_EPOCH) {
return;
}

switch (sync.state) {
case SyncState.SyncingFinalized:
case SyncState.SyncingHead: {
const currentSlot = chain.clock.currentSlot;
const headSlot = chain.forkChoice.getHead().slot;
if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
}

return;
}

case SyncState.Synced:
return;

case SyncState.Stalled:
throw new NodeIsSyncing("waiting for peers");
}
}

/**
* Ensures that the array contains unique values, and throws an ApiError
Expand Down
77 changes: 16 additions & 61 deletions packages/beacon-node/src/api/impl/validator/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ import {ZERO_HASH} from "../../../constants/index.js";
import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js";
import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js";
import {CommitteeSubscription} from "../../../network/subnets/index.js";
import {SyncState} from "../../../sync/index.js";
import {callInNextEventLoop} from "../../../util/eventLoop.js";
import {isOptimisticBlock} from "../../../util/forkChoice.js";
import {getBlockGraffiti, toGraffitiBytes} from "../../../util/graffiti.js";
Expand All @@ -91,23 +90,9 @@ import {ApiOptions} from "../../options.js";
import {getStateResponseWithRegen} from "../beacon/state/utils.js";
import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js";
import {ApiModules} from "../types.js";
import {notWhileSyncing} from "../utils.js";
import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js";

/**
* If the node is within this many epochs from the head, we declare it to be synced regardless of
* the network sync state.
*
* This helps prevent attacks where nodes can convince us that we're syncing some non-existent
* finalized head.
*
* TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators
* can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets
* a new head. Then the checkpoint state cache grows unbounded with very different states (because
* they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state
* caches are better.
*/
export const SYNC_TOLERANCE_EPOCHS = 1;

/**
* Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
* Post this time, race execution and builder to pick whatever resolves first.
Expand Down Expand Up @@ -341,36 +326,6 @@ export function getValidatorApi(
return null;
}

/**
* Reject any request while the node is syncing
*/
function notWhileSyncing(): void {
// Consider node synced before or close to genesis
if (chain.clock.currentSlot < SLOTS_PER_EPOCH) {
return;
}

const syncState = sync.state;
switch (syncState) {
case SyncState.SyncingFinalized:
case SyncState.SyncingHead: {
const currentSlot = chain.clock.currentSlot;
const headSlot = chain.forkChoice.getHead().slot;
if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
}

return;
}

case SyncState.Synced:
return;

case SyncState.Stalled:
throw new NodeIsSyncing("waiting for peers");
}
}

/**
* Post merge, the CL and EL could be out of step in the sync, and could result in
* Syncing status of the chain head. To be precise:
Expand Down Expand Up @@ -573,7 +528,7 @@ export function getValidatorApi(
builderBoostFactor?: bigint,
{feeRecipient, builderSelection, strictFeeRecipientCheck}: routes.validator.ExtraProduceBlockOpts = {}
): Promise<ProduceBlindedBlockOrBlockContentsRes> {
notWhileSyncing();
notWhileSyncing(chain, sync);
await waitForSlot(slot); // Must never request for a future slot > currentSlot

const parentBlock = chain.getProposerHead(slot);
Expand Down Expand Up @@ -917,7 +872,7 @@ export function getValidatorApi(
throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`);
}

notWhileSyncing();
notWhileSyncing(chain, sync);
await waitForSlot(slot);

const parentBlock = chain.getProposerHead(slot);
Expand Down Expand Up @@ -1035,7 +990,7 @@ export function getValidatorApi(
},

async produceAttestationData({committeeIndex, slot}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

await waitForSlot(slot); // Must never request for a future slot > currentSlot

Expand Down Expand Up @@ -1116,7 +1071,7 @@ export function getValidatorApi(
throw new ApiError(400, `producePayloadAttestationData is not supported before Gloas fork=${fork}`);
}

notWhileSyncing();
notWhileSyncing(chain, sync);
await waitForSlot(slot);

const block = chain.forkChoice.getCanonicalBlockAtSlot(slot);
Expand Down Expand Up @@ -1201,7 +1156,7 @@ export function getValidatorApi(
},

async getProposerDuties({epoch}, _context, opts?: {v2?: boolean}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

const currentEpoch = currentEpochWithDisparity();
const nextEpoch = currentEpoch + 1;
Expand Down Expand Up @@ -1248,7 +1203,7 @@ export function getValidatorApi(
// requested epoch is within that range, we can use the head state at current epoch
state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
} else {
const res = await getStateResponseWithRegen(chain, startSlot);
const res = await getStateResponseWithRegen(chain, sync, startSlot);

state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state;

Expand Down Expand Up @@ -1342,7 +1297,7 @@ export function getValidatorApi(
},

async getAttesterDuties({epoch, indices}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

if (indices.length === 0) {
throw new ApiError(400, "No validator to get attester duties");
Expand Down Expand Up @@ -1402,7 +1357,7 @@ export function getValidatorApi(
},

async getPtcDuties({epoch, indices}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

if (indices.length === 0) {
throw new ApiError(400, "No validator to get PTC duties");
Expand Down Expand Up @@ -1464,7 +1419,7 @@ export function getValidatorApi(
* @param validatorIndices an array of the validator indices for which to obtain the duties.
*/
async getSyncCommitteeDuties({epoch, indices}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

if (indices.length === 0) {
throw new ApiError(400, "No validator to get attester duties");
Expand Down Expand Up @@ -1511,7 +1466,7 @@ export function getValidatorApi(
},

async getAggregatedAttestationV2({attestationDataRoot, slot, committeeIndex}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

await waitForSlot(slot); // Must never request for a future slot > currentSlot

Expand All @@ -1534,7 +1489,7 @@ export function getValidatorApi(
},

async publishAggregateAndProofsV2({signedAggregateAndProofs}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

const seenTimestampSec = Date.now() / 1000;
const failures: FailureList = [];
Expand Down Expand Up @@ -1595,7 +1550,7 @@ export function getValidatorApi(
* https://github.com/ethereum/beacon-APIs/pull/137
*/
async publishContributionAndProofs({contributionAndProofs}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

const failures: FailureList = [];

Expand Down Expand Up @@ -1644,7 +1599,7 @@ export function getValidatorApi(
},

async prepareBeaconCommitteeSubnet({subscriptions}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

await network.prepareBeaconCommitteeSubnets(
subscriptions.map(({validatorIndex, slot, isAggregator, committeesAtSlot, committeeIndex}) => ({
Expand Down Expand Up @@ -1677,7 +1632,7 @@ export function getValidatorApi(
* https://github.com/ethereum/beacon-APIs/pull/136
*/
async prepareSyncCommitteeSubnets({subscriptions}) {
notWhileSyncing();
notWhileSyncing(chain, sync);

// A `validatorIndex` can be in multiple subnets, so compute the CommitteeSubscription with double for loop
const subs: CommitteeSubscription[] = [];
Expand Down Expand Up @@ -1821,7 +1776,7 @@ export function getValidatorApi(
throw new ApiError(400, `getExecutionPayloadEnvelope not supported for pre-gloas fork=${fork}`);
}

notWhileSyncing();
notWhileSyncing(chain, sync);
await waitForSlot(slot);

const blockRootHex = toRootHex(beaconBlockRoot);
Expand Down
Loading
Loading