diff --git a/ethexe/common/src/db.rs b/ethexe/common/src/db.rs index d87f9be1f17..13198a0f078 100644 --- a/ethexe/common/src/db.rs +++ b/ethexe/common/src/db.rs @@ -149,7 +149,8 @@ pub struct CompactMb { #[derive(Debug, Clone, Default, Encode, Decode, TypeInfo, PartialEq, Eq, Hash)] pub struct MbMeta { pub computed: bool, - pub last_advanced_eb: H256, + pub finalized: bool, + pub last_advanced_eb: Option, } #[auto_impl::auto_impl(&, Box)] @@ -267,7 +268,7 @@ mod tests { #[test] fn ensure_types_unchanged() { const EXPECTED_TYPE_INFO_HASH: &str = - "cbf21dc97ec57cc6f653a7808672dc2c086fdfae28d1435e93f0dfe812de21c3"; + "6f7363ba820e33c8b6acfa633b782f998f530fcf6fe15f19afe141996261baf4"; let types = [ meta_type::(), diff --git a/ethexe/common/src/mock.rs b/ethexe/common/src/mock.rs index 183ed86010c..7b8645d0bf1 100644 --- a/ethexe/common/src/mock.rs +++ b/ethexe/common/src/mock.rs @@ -606,7 +606,7 @@ where db.set_mb_outcome(H256::zero(), Vec::new()); db.mutate_mb_meta(H256::zero(), |m| { m.computed = true; - m.last_advanced_eb = H256::zero(); + m.last_advanced_eb = Some(H256::zero()); }); } diff --git a/ethexe/compute/src/compute.rs b/ethexe/compute/src/compute.rs index e7eb682843a..62d6b19e2bb 100644 --- a/ethexe/compute/src/compute.rs +++ b/ethexe/compute/src/compute.rs @@ -244,7 +244,13 @@ pub fn prepare_executable_for_mb( let schedule = db .mb_schedule(parent) .ok_or(ComputeError::ParentMbScheduleMissing(parent))?; - let advanced_block = db.mb_meta(parent).last_advanced_eb; + // `process_mb_proposal` always propagates `last_advanced_eb` when it stores + // an MB (and init seeds the zero genesis parent), so a computed parent + // missing it is a DB-invariant violation. + let advanced_block = db + .mb_meta(parent) + .last_advanced_eb + .expect("parent MB must have last_advanced_eb set"); build_executable_data(db, mb_payload, program_states, schedule, advanced_block) } @@ -252,8 +258,8 @@ pub fn prepare_executable_for_mb( /// Walk the MB's `Operations` list and prepare processor input. /// /// Synthetic block height/timestamp come from `last_advanced_eb` (the latest -/// EB pinned by this MB or any ancestor); if none, fall back to the router's -/// genesis block from [`ConfigStorageRO::config`]. +/// EB pinned by this MB or any ancestor); the zero sentinel falls back to the +/// router's genesis block from [`ConfigStorageRO::config`]. fn build_executable_data( db: &Database, operations: Operations, @@ -265,6 +271,8 @@ fn build_executable_data( let mut injected_transactions = Vec::new(); let mut gas_allowance: Option = None; + // The zero sentinel (MB never advanced past pre-genesis) means "no anchor + // yet" — the genesis-block fallback below stands in. let mut current_anchor = if advanced_block.is_zero() { None } else { @@ -565,7 +573,7 @@ mod tests { /// via the `Some` branch — exactly as the malachite service propagates it. fn seed_genesis_eth(db: &Database) -> H256 { let gen_eb = synthetic_eb(db, 1, vec![]); - db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = gen_eb); + db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = Some(gen_eb)); gen_eb } @@ -595,13 +603,18 @@ mod tests { operations_hash, }, ); + // Mimic `process_mb_proposal`, which always populates `last_advanced_eb` + // before an MB is computed. + db.mutate_mb_meta(mb_hash, |meta| { + meta.last_advanced_eb = Some(H256::zero()); + }); } /// `seed_mb` plus the malachite-side bookkeeping: record the advanced EB /// as this MB's `last_advanced_eb`, so its child walks a depth-1 chain. fn seed_mb_advancing(db: &Database, mb_hash: H256, parent: H256, height: u64, eb_height: u32) { seed_mb(db, mb_hash, parent, height, dummy_ops(db, eb_height)); - db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = eb_hash(eb_height)); + db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = Some(eb_hash(eb_height))); } /// Tail-only queue still computes all uncomputed predecessors. @@ -866,7 +879,7 @@ mod tests { }]; ops.extend(mb_bookend()); seed_mb(db, creator, H256::zero(), 0, Operations::new(ops)); - db.mutate_mb_meta(creator, |m| m.last_advanced_eb = create_eb); + db.mutate_mb_meta(creator, |m| m.last_advanced_eb = Some(create_eb)); mb_hashes.push(creator); // MB #1.. — each injects a single PING into the ping program. @@ -886,7 +899,7 @@ mod tests { i, Operations::new(ops), ); - db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = eb); + db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = Some(eb)); mb_hashes.push(mb_hash); } diff --git a/ethexe/compute/src/service.rs b/ethexe/compute/src/service.rs index e79cbfac70d..d4ff344c39e 100644 --- a/ethexe/compute/src/service.rs +++ b/ethexe/compute/src/service.rs @@ -159,7 +159,7 @@ mod tests { }, ); db.set_block_events(genesis_eb, &[]); - db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = genesis_eb); + db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = Some(genesis_eb)); // The EB this MB advances to, chained onto the genesis Eth block. let eth_block_hash = H256::from_low_u64_be(0xEB01); diff --git a/ethexe/consensus/src/utils.rs b/ethexe/consensus/src/utils.rs index e52c678aca9..c45748867bb 100644 --- a/ethexe/consensus/src/utils.rs +++ b/ethexe/consensus/src/utils.rs @@ -6,18 +6,16 @@ //! This module provides utility functions and data structures for handling batch commitments, //! validation requests, and multi-signature operations in the Ethexe system. -use anyhow::{Result, anyhow}; +use anyhow::Result; use ethexe_common::{ Address, Digest, ToDigest, consensus::BatchCommitmentValidationReply, - db::OnChainStorageRO, ecdsa::{ContractSignature, PublicKey}, gear::BatchCommitment, }; -use gprimitives::H256; use gsigner::secp256k1::{Secp256k1SignerExt, Signer}; use parity_scale_codec::{Decode, Encode}; -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; /// A batch commitment, that has been signed by multiple validators. /// This structure manages the collection of signatures from different validators @@ -105,41 +103,3 @@ impl MultisignedBatchCommitment { (self.batch, self.signatures.into_values().collect()) } } -pub fn has_duplicates(data: &[T]) -> bool { - let mut seen = HashSet::new(); - data.iter().any(|item| !seen.insert(item)) -} - -/// `target` lies on the canonical eth chain ending at `head` — i.e., `head` -/// is `target` itself or one of its descendants reachable via parent links. -/// `target == H256::zero()` is the genesis sentinel and returns `Ok(true)`. -pub fn is_eth_block_canonical_to( - db: &DB, - target: H256, - head: H256, -) -> Result { - if target.is_zero() { - return Ok(true); - } - let target_height = db - .block_header(target) - .ok_or_else(|| anyhow!("eth chain walk: missing header for target {target}"))? - .height; - - let mut current = head; - loop { - if current == target { - return Ok(true); - } - if current.is_zero() { - return Ok(false); - } - let header = db - .block_header(current) - .ok_or_else(|| anyhow!("eth chain walk: missing header for {current}"))?; - if header.height <= target_height { - return Ok(false); - } - current = header.parent_hash; - } -} diff --git a/ethexe/consensus/src/validator/batch/filler.rs b/ethexe/consensus/src/validator/batch/filler.rs index 11dad0c47b6..cc1655892c3 100644 --- a/ethexe/consensus/src/validator/batch/filler.rs +++ b/ethexe/consensus/src/validator/batch/filler.rs @@ -1,7 +1,12 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -use super::types::{BatchLimits, BatchParts, BatchSizeCounter, ValidationRejectReason}; +use std::{mem, num::NonZero}; + +use super::{ + types::{BatchParts, BatchSizeCounter}, + utils, +}; use ethexe_common::gear::{ ChainCommitment, CodeCommitment, RewardsCommitment, ValidatorsCommitment, @@ -29,42 +34,29 @@ pub enum BatchIncludeError { SizeLimitExceeded, } -impl From for ValidationRejectReason { - fn from(value: BatchIncludeError) -> Self { - match value { - BatchIncludeError::SizeLimitExceeded => Self::BatchSizeLimitExceeded, - } - } -} - type FillerResult = Result<(), BatchIncludeError>; impl BatchFiller { - pub fn new(limits: BatchLimits) -> Self { + pub fn new(batch_size_limit: u64) -> Self { Self { parts: BatchParts::default(), - size_counter: BatchSizeCounter::new(limits.batch_size_limit), + size_counter: BatchSizeCounter::new(batch_size_limit), } } pub fn into_parts(mut self) -> BatchParts { - if let Some(chain) = &mut self.parts.chain_commitment { + if let Some((chain, _len)) = &mut self.parts.chain_commitment { chain.transitions = - super::utils::squash_transitions_by_actor(std::mem::take(&mut chain.transitions)); - super::utils::sort_transitions_by_value_to_receive(&mut chain.transitions); + utils::squash_transitions_by_actor(mem::take(&mut chain.transitions)); + utils::sort_transitions_by_value_to_receive(&mut chain.transitions); } self.parts } - pub fn has_chain_commitment(&self) -> bool { - self.parts.chain_commitment.is_some() - } - pub fn include_validators_commitment( &mut self, commitment: ValidatorsCommitment, ) -> FillerResult { - let commitment = Some(commitment); if !self .size_counter .charge_for_validators_commitment(&commitment) @@ -72,30 +64,19 @@ impl BatchFiller { return Err(BatchIncludeError::SizeLimitExceeded); } - self.parts.validators_commitment = commitment; + self.parts.validators_commitment = Some(commitment); Ok(()) } pub fn include_rewards_commitment(&mut self, commitment: RewardsCommitment) -> FillerResult { - let commitment = Some(commitment); if !self.size_counter.charge_for_rewards_commitment(&commitment) { return Err(BatchIncludeError::SizeLimitExceeded); } - self.parts.rewards_commitment = commitment; + self.parts.rewards_commitment = Some(commitment); Ok(()) } - /// Probe whether a hypothetical chain commitment with `transitions` would - /// still fit the remaining batch budget. Used by the producer to grow the - /// chain commitment one MB at a time and stop *before* the size limit is - /// breached, so the call to [`Self::include_chain_commitment`] is - /// guaranteed to succeed. - pub fn would_fit_chain_commitment(&self, candidate: &ChainCommitment) -> bool { - let mut probe = self.size_counter.clone(); - probe.charge_for_chain_commitment(&Some(candidate.clone())) - } - pub fn include_code_commitment(&mut self, commitment: CodeCommitment) -> FillerResult { if !self.size_counter.charge_for_code_commitment(&commitment) { return Err(BatchIncludeError::SizeLimitExceeded); @@ -105,24 +86,33 @@ impl BatchFiller { Ok(()) } - /// Include a freshly aggregated chain commitment in the batch. - /// - /// A commitment with neither transitions nor an Ethereum-anchor - /// advance carries no payload and is dropped — the next coordinator - /// round will re-walk and pick up whatever has finalized since. - /// Empty-transitions checkpoints **with** a non-zero - /// `last_advanced_eth_block` are kept: they exist specifically to push - /// the on-chain Ethereum anchor forward during long quiet stretches. - pub fn include_chain_commitment(&mut self, commitment: ChainCommitment) -> FillerResult { - if commitment.transitions.is_empty() && commitment.last_advanced_eth_block.is_zero() { - return Ok(()); + pub fn append_chain_commitment(&mut self, commitment: ChainCommitment) -> FillerResult { + if let Some((existing, len)) = &mut self.parts.chain_commitment { + let ChainCommitment { + head, + transitions, + last_advanced_eth_block, + } = commitment; + + if !self.size_counter.charge_for_transitions(&transitions) { + return Err(BatchIncludeError::SizeLimitExceeded); + } + + existing.head = head; + existing.transitions.extend(transitions); + existing.last_advanced_eth_block = last_advanced_eth_block; + + *len = len + .checked_add(1) + .expect("u32 chain commitment len overflow"); + } else { + if !self.size_counter.charge_for_chain_commitment(&commitment) { + return Err(BatchIncludeError::SizeLimitExceeded); + } + + self.parts.chain_commitment = Some((commitment, NonZero::new(1).expect("1 != 0"))); } - let commitment = Some(commitment); - if !self.size_counter.charge_for_chain_commitment(&commitment) { - return Err(BatchIncludeError::SizeLimitExceeded); - } - self.parts.chain_commitment = commitment; Ok(()) } } @@ -134,24 +124,36 @@ mod tests { use ethexe_ethereum::abi::Gear; use gprimitives::{CodeId, H256}; - /// Checkpoint chain commitments carry empty transitions but a - /// non-zero `last_advanced_eth_block` — they exist *specifically* - /// to push the on-chain Ethereum anchor forward when the chain has - /// been quiet for a long stretch. The filler must keep them. + const BIG_LIMIT: u64 = u64::MAX; + + /// Appending a single chain commitment seeds the parts with a length of 1, + /// and a subsequent append extends it (head + anchor follow the newest MB). #[test] - fn include_chain_commitment_keeps_checkpoint_with_no_transitions() { - let mut filler = BatchFiller::new(BatchLimits::default()); - let checkpoint = ChainCommitment { + fn append_chain_commitment_seeds_then_extends() { + let mut filler = BatchFiller::new(BIG_LIMIT); + let first = ChainCommitment { head: H256::from_low_u64_be(0xC0DE), transitions: Vec::new(), last_advanced_eth_block: H256::from_low_u64_be(0xEB), }; + let second = ChainCommitment { + head: H256::from_low_u64_be(0xBEEF), + transitions: Vec::new(), + last_advanced_eth_block: H256::from_low_u64_be(0xEC), + }; - filler.include_chain_commitment(checkpoint).unwrap(); - assert!( - filler.has_chain_commitment(), - "checkpoint with empty transitions but a non-zero advanced anchor must \ - be retained — dropping it strands the Ethereum-side anchor advance" + filler.append_chain_commitment(first).unwrap(); + filler.append_chain_commitment(second.clone()).unwrap(); + + let (chain, len) = filler + .into_parts() + .chain_commitment + .expect("chain commitment must be retained"); + assert_eq!(len.get(), 2); + assert_eq!(chain.head, second.head); + assert_eq!( + chain.last_advanced_eth_block, + second.last_advanced_eth_block ); } @@ -166,10 +168,7 @@ mod tests { }; let encoded: Gear::CodeCommitment = first.clone().into(); // Budget fits exactly one commitment; the second include must fail. - let mut filler = BatchFiller::new(BatchLimits { - batch_size_limit: encoded.abi_encoded_size() as u64, - ..BatchLimits::default() - }); + let mut filler = BatchFiller::new(encoded.abi_encoded_size() as u64); filler.include_code_commitment(first.clone()).unwrap(); assert_eq!( diff --git a/ethexe/consensus/src/validator/batch/manager.rs b/ethexe/consensus/src/validator/batch/manager.rs index 28a07a410b4..dd1fd154c5b 100644 --- a/ethexe/consensus/src/validator/batch/manager.rs +++ b/ethexe/consensus/src/validator/batch/manager.rs @@ -1,17 +1,16 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -use super::types::{BatchLimits, ValidationRejectReason, ValidationStatus}; -use crate::validator::{ - batch::{filler::BatchFiller, types::BatchParts, utils}, - core::{ElectionRequest, MiddlewareWrapper}, +use super::{ + filler::BatchFiller, + types::{BatchLimits, ValidationRejectReason, ValidationStatus}, + utils, }; - -use alloy::sol_types::SolValue; +use crate::validator::core::{ElectionRequest, MiddlewareWrapper}; use anyhow::{Context as _, Result, anyhow, bail}; use ethexe_common::{ SimpleBlockData, ToDigest, - consensus::BatchCommitmentValidationRequest, + consensus::{BatchCommitmentValidationRequest, MAX_BATCH_SIZE_LIMIT}, db::{ BlockMetaStorageRO, CodesStorageRO, ConfigStorageRO, GlobalsStorageRO, MbStorageRO, OnChainStorageRO, @@ -21,9 +20,9 @@ use ethexe_common::{ }, }; use ethexe_db::Database; -use ethexe_ethereum::abi::Gear; -use gprimitives::H256; +use gprimitives::{CodeId, H256}; use hashbrown::HashSet; +use std::{collections::VecDeque, num::NonZeroU32}; #[derive(derive_more::Debug, Clone)] pub struct BatchCommitmentManager { @@ -47,77 +46,39 @@ impl BatchCommitmentManager { } } - /// Coordinator-side batch builder. Walks `[last_committed_mb..latest_finalized_mb]` - /// and pairs the chain piece with validators / rewards / code commitments. - /// Returns `Ok(None)` when there's nothing to commit. + /// Coordinator-side batch builder. + /// Creates batch commitment for the given Ethereum `block` used as a reference for the batch. + /// Returns `Ok(None)` if batch commitment is not needed. + /// Returns `Ok(Some(BatchCommitment))` if a batch commitment was successfully created. pub async fn create_batch_commitment( self, block: SimpleBlockData, ) -> Result> { - let mut batch_filler = BatchFiller::new(self.limits.clone()); + let mut batch_filler = BatchFiller::new(self.limits.batch_size_limit); - if let Some(validators_commitment) = self.aggregate_validators_commitment(&block).await? + if let Some(validators_commitment) = self.aggregate_validators_commitment(block).await? && let Err(err) = batch_filler.include_validators_commitment(validators_commitment) { bail!("failed to include validators commitment into batch, err={err}") } - if let Some(rewards_commitment) = self.aggregate_rewards_commitment(&block).await? + if let Some(rewards_commitment) = self.aggregate_rewards_commitment(block).await? && let Err(err) = batch_filler.include_rewards_commitment(rewards_commitment) { bail!("failed to include rewards commitment into batch, err={err}") } - // State transitions before code commitments. - let latest_finalized_mb = self.db.globals().latest_finalized_mb_hash; - if !latest_finalized_mb.is_zero() { - let latest_advanced = self.db.mb_meta(latest_finalized_mb).last_advanced_eb; - if !crate::utils::is_eth_block_canonical_to(&self.db, latest_advanced, block.hash)? { - // Eth reorged deeper than canonical_quarantine past a finalized - // MB; commitments stall until Eth reverts. - tracing::error!( - %latest_finalized_mb, - %latest_advanced, - block = %block.hash, - "coordinator: latest finalized MB advanced to a non-canonical Eth block — \ - refusing to build batch (commitments to Eth are now blocked until recovery)" - ); - return Ok(None); - } + // NOTE: chain commitment must be included before code commitments + utils::try_include_chain_commitment(&self.db, block.hash, &mut batch_filler)?; - // `try_include_chain_commitment` is lenient; only DB-invariant errors propagate. - super::utils::try_include_chain_commitment( - &self.db, - block.hash, - latest_finalized_mb, - &mut batch_filler, - )?; - - // Checkpoint: if no chain commitment fits but the producer's - // `last_advanced_eth_block` is far ahead of `last_committed_eb`, - // emit an empty chain commitment that just bumps the on-chain anchor. - if !batch_filler.has_chain_commitment() { - super::utils::try_include_checkpoint_chain_commitment( - &self.db, - block.hash, - latest_finalized_mb, - self.limits.uncommitted_chain_len_threshold, - &mut batch_filler, - )?; - } - } + utils::aggregate_code_commitments_for_block(&self.db, block.hash, &mut batch_filler)?; - super::utils::aggregate_code_commitments_for_block( - &self.db, - block.hash, - &mut batch_filler, - )?; - - super::utils::create_batch_commitment( + utils::create_batch_commitment( &self.db, &block, batch_filler.into_parts(), self.limits.commitment_delay_limit, + self.limits.checkpoint_threshold, ) } @@ -135,18 +96,24 @@ impl BatchCommitmentManager { validators, rewards, } = &request; - let mut batch_parts = BatchParts::default(); - if crate::utils::has_duplicates(codes.as_slice()) { - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::CodesHaveDuplicates, - }); - } + // NOTE: self.limits.batch_size_limit is used for batch creation, see `create_batch_commitment]`. + // For validation, node allows batch to exceed local limit up to MAX_BATCH_SIZE_LIMIT. + let mut batch_filler = BatchFiller::new(MAX_BATCH_SIZE_LIMIT); if validators { - match self.aggregate_validators_commitment(&block).await? { - Some(commitment) => batch_parts.validators_commitment = Some(commitment), + match self.aggregate_validators_commitment(block).await? { + Some(commitment) => { + if batch_filler + .include_validators_commitment(commitment) + .is_err() + { + return Ok(ValidationStatus::Rejected { + request, + reason: ValidationRejectReason::BatchSizeLimitExceeded, + }); + } + } None => { return Ok(ValidationStatus::Rejected { request, @@ -157,8 +124,15 @@ impl BatchCommitmentManager { } if rewards { - match self.aggregate_rewards_commitment(&block).await? { - Some(commitment) => batch_parts.rewards_commitment = Some(commitment), + match self.aggregate_rewards_commitment(block).await? { + Some(commitment) => { + if batch_filler.include_rewards_commitment(commitment).is_err() { + return Ok(ValidationStatus::Rejected { + request, + reason: ValidationRejectReason::BatchSizeLimitExceeded, + }); + } + } None => { return Ok(ValidationStatus::Rejected { request, @@ -168,156 +142,28 @@ impl BatchCommitmentManager { } } - let waiting_codes = self - .db - .block_meta(block.hash) - .codes_queue - .ok_or_else(|| anyhow!("codes queue not found for block={}", block.hash))? - .into_iter() - .collect::>(); - - if let Some(&code_id) = codes.iter().find(|&id| !waiting_codes.contains(id)) { - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::CodeNotWaitingForCommitment(code_id), - }); + if let Some(head_mb) = head + && let Some(reason) = + self.validate_chain_commitment(block, head_mb, &mut batch_filler)? + { + return Ok(ValidationStatus::Rejected { request, reason }); } - for &id in codes.iter() { - let Some(valid) = self.db.code_valid(id) else { - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::CodeIsNotProcessedYet(id), - }); - }; - batch_parts - .code_commitments - .push(CodeCommitment { id, valid }); + if let Some(reason) = self.validate_code_commitments(block, codes, &mut batch_filler)? { + return Ok(ValidationStatus::Rejected { request, reason }); } - if let Some(head_mb) = head { - // Mirror the coordinator-side guard: refuse to sign anything if our - // own `latest_finalized_mb` advanced to a non-canonical Eth block - // (deep Eth reorg past quarantine). The coordinator's advance must - // also be canonical here for the batch to ever land. - let local_latest_finalized = self.db.globals().latest_finalized_mb_hash; - if !local_latest_finalized.is_zero() { - let latest_advanced = self.db.mb_meta(local_latest_finalized).last_advanced_eb; - if !crate::utils::is_eth_block_canonical_to(&self.db, latest_advanced, block.hash)? - { - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::LatestFinalizedAdvanceNotCanonical( - latest_advanced, - ), - }); - } - } + // Do not restrict coordinator to commit empty batch, even if checkpoint threshold is not reached. + let checkpoint_threshold_for_validation = NonZeroU32::new(1).expect("1 != 0"); - // BFT-safety: any two finalized MBs are linearly ordered, so reachability - // from `latest_finalized_mb` via parents is iff "finalized locally". - let latest_finalized_mb = self.db.globals().latest_finalized_mb_hash; - if !utils::is_finalized_locally(&self.db, head_mb, latest_finalized_mb) { - let head_meta = self.db.mb_meta(head_mb); - tracing::warn!( - %head_mb, - %latest_finalized_mb, - head_computed = head_meta.computed, - "manager: rejecting batch — head_mb not yet finalized locally", - ); - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::HeadMbNotFinalized(head_mb), - }); - } - - let head_meta = self.db.mb_meta(head_mb); - if !head_meta.computed { - tracing::warn!( - %head_mb, - "manager: rejecting batch — head_mb not yet computed locally", - ); - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::HeadMbNotComputed(head_mb), - }); - } - - let last_committed_mb = self - .db - .block_meta(block.hash) - .last_committed_mb - .unwrap_or(H256::zero()); - - // Head must strictly advance past last-committed; genesis = height 0. - let head_height = self - .db - .mb_compact_block(head_mb) - .map(|c| c.height) - .ok_or_else(|| anyhow!("MB {head_mb} marked finalized but has no compact block"))?; - let last_committed_height = if last_committed_mb.is_zero() { - 0 - } else { - self.db - .mb_compact_block(last_committed_mb) - .map(|c| c.height) - .ok_or_else(|| { - anyhow!( - "last_committed_mb {last_committed_mb} not in DB for block {}", - block.hash, - ) - })? - }; - if head_height <= last_committed_height { - tracing::warn!( - %head_mb, - head_height, - %last_committed_mb, - last_committed_height, - "manager: rejecting batch — head_mb at or below last_committed_mb height", - ); - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::HeadMbAlreadyCommitted(head_mb), - }); - } - - // Both endpoints finalized → walk is on canonical chain; only DB-corrupt errors here. - let pending = super::utils::collect_not_committed_mb_predecessors( - &self.db, - last_committed_mb, - head_mb, - )?; - - let mut chain_commitment = ChainCommitment { - transitions: Vec::new(), - head: head_mb, - last_advanced_eth_block: self.db.mb_meta(head_mb).last_advanced_eb, - }; - for mb_hash in pending.into_iter() { - let Some(mb_transitions) = self.db.mb_outcome(mb_hash) else { - anyhow::bail!("Computed MB {mb_hash} outcome not found in db"); - }; - chain_commitment.transitions.extend(mb_transitions); - } - chain_commitment.transitions = super::utils::squash_transitions_by_actor( - std::mem::take(&mut chain_commitment.transitions), - ); - super::utils::sort_transitions_by_value_to_receive(&mut chain_commitment.transitions); - batch_parts.chain_commitment = Some(chain_commitment); - } - - let Some(batch) = super::utils::create_batch_commitment( + let Some(batch) = utils::create_batch_commitment( &self.db, &block, - batch_parts, + batch_filler.into_parts(), self.limits.commitment_delay_limit, + checkpoint_threshold_for_validation, )? else { - tracing::warn!( - "Batch commitment is empty for block({:?}), rejecting batch", - block.hash - ); return Ok(ValidationStatus::Rejected { request, reason: ValidationRejectReason::EmptyBatch, @@ -335,20 +181,217 @@ impl BatchCommitmentManager { }); } - let batch_encoded_size = Gear::BatchCommitment::from(batch).abi_encoded_size() as u64; - if batch_encoded_size > self.limits.batch_size_limit { - return Ok(ValidationStatus::Rejected { - request, - reason: ValidationRejectReason::BatchSizeLimitExceeded, - }); + Ok(ValidationStatus::Accepted(digest)) + } + + fn validate_chain_commitment( + &self, + block: SimpleBlockData, + head_mb_hash: H256, + batch_filler: &mut BatchFiller, + ) -> Result> { + let head_mb_meta = self.db.mb_meta(head_mb_hash); + + // "Finalized locally" is a reachability property from the BFT-finalized + // tip, not just the per-MB `finalized` cache: a freshly-started + // validator learns the prior chain's finality indirectly (sync / + // on-chain `MBCommitted`) without running `process_mb_finalized` for + // every ancestor, so the cache bit can be unset on MBs that are in fact + // finalized. The flag is kept as a fast path. + let latest_finalized_mb = self.db.globals().latest_finalized_mb_hash; + if !head_mb_meta.finalized + && !utils::is_finalized_locally(&self.db, head_mb_hash, latest_finalized_mb) + { + return Ok(Some(ValidationRejectReason::HeadMbNotFinalized( + head_mb_hash, + ))); } - Ok(ValidationStatus::Accepted(digest)) + if !head_mb_meta.computed { + return Ok(Some(ValidationRejectReason::HeadMbNotComputed( + head_mb_hash, + ))); + } + + let head_mb = self + .db + .mb_compact_block(head_mb_hash) + .with_context(|| format!("finalized MB {head_mb_hash} has no compact block in db"))?; + + let last_committed_mb_hash = self + .db + .block_meta(block.hash) + .last_committed_mb + .with_context(|| { + format!( + "prepared block {} has no last_committed_mb in db", + block.hash + ) + })?; + + // Walk the parent chain from `head_mb` down to (exclusive) the on-chain + // committed anchor, terminating on its hash. The anchor's own compact + // block is intentionally never dereferenced: a freshly joined validator + // may know `last_committed_mb_hash` only from the on-chain `MBCommitted` + // event and never have computed that MB. If the walk leaves the + // locally-known chain before reaching the anchor, `head_mb` is not a + // local descendant of it — reject rather than hard-error. + let mut cursor_mb_hash = head_mb_hash; + let mut cursor_mb = head_mb; + let mut not_committed_mbs_chain = VecDeque::new(); + while cursor_mb_hash != last_committed_mb_hash { + // push_front to keep the order from oldest to newest + not_committed_mbs_chain.push_front(cursor_mb_hash); + let parent_hash = cursor_mb.parent; + if parent_hash == last_committed_mb_hash { + cursor_mb_hash = parent_hash; + break; + } + if parent_hash.is_zero() { + // Genesis sentinel (zero MB, seeded with a self-parent in db + // init) reached without hitting the committed anchor: the head + // is not a local descendant of it. Stop instead of spinning. + return Ok(Some( + ValidationRejectReason::HeadMbNotStrictDescendantOfLatestCommittedMb { + head_mb: head_mb_hash, + latest_committed_mb: last_committed_mb_hash, + }, + )); + } + let Some(parent_mb) = self.db.mb_compact_block(parent_hash) else { + return Ok(Some( + ValidationRejectReason::HeadMbNotStrictDescendantOfLatestCommittedMb { + head_mb: head_mb_hash, + latest_committed_mb: last_committed_mb_hash, + }, + )); + }; + cursor_mb_hash = parent_hash; + cursor_mb = parent_mb; + } + + if cursor_mb_hash != last_committed_mb_hash { + return Ok(Some( + ValidationRejectReason::HeadMbNotStrictDescendantOfLatestCommittedMb { + head_mb: head_mb_hash, + latest_committed_mb: last_committed_mb_hash, + }, + )); + } + + let last_advanced_eth_block = self + .db + .mb_meta(head_mb_hash) + .last_advanced_eb + .with_context(|| { + format!("finalized MB {head_mb_hash} has no last_advanced_eb in db") + })?; + + // The committed MB's advanced-EB anchor equals `BlockMeta.last_committed_eb`: + // the Router emits `MBCommitted(head)` and `EBCommitted(lastAdvancedEthBlock)` + // from the same `ChainCommitment` (Router.sol). Unlike `mb_meta` of the + // committed MB, this is available to a freshly joined validator. `None` + // means no EB has been committed yet (genesis anchor → zero). + let last_committed_advanced_eth_block = self + .db + .block_meta(block.hash) + .last_committed_eb + .unwrap_or_default(); + + // This check is not necessary, as soon as this must be guaranteed by ethexe-malachite, + // but we still want to have it just in case, to avoid accepting invalid batch commitments. + if !utils::is_strict_descendant_eth_block( + &self.db, + last_advanced_eth_block, + last_committed_advanced_eth_block, + )? { + tracing::error!( + block = %block.hash, + %head_mb_hash, + %last_committed_mb_hash, + %last_advanced_eth_block, + %last_committed_advanced_eth_block, + "head MB is finalized, but its last advanced EB is not a strict descendant of the last committed advanced EB" + ); + + return Ok(Some( + ValidationRejectReason::LastAdvancedEbNotOnCanonicalChain { + last_advanced_eb: last_advanced_eth_block, + last_committed_advanced_eb: last_committed_advanced_eth_block, + }, + )); + } + + for mb_hash in not_committed_mbs_chain.into_iter() { + let Some(transitions) = self.db.mb_outcome(mb_hash) else { + anyhow::bail!("Computed MB {mb_hash} outcome not found in db"); + }; + + let last_advanced_eth_block = + self.db.mb_meta(mb_hash).last_advanced_eb.with_context(|| { + format!("finalized MB {mb_hash} has no last_advanced_eb in db") + })?; + + let one_mb_commitment = ChainCommitment { + head: mb_hash, + transitions, + last_advanced_eth_block, + }; + + if batch_filler + .append_chain_commitment(one_mb_commitment) + .is_err() + { + return Ok(Some(ValidationRejectReason::BatchSizeLimitExceeded)); + } + } + + Ok(None) } - pub async fn aggregate_validators_commitment( + fn validate_code_commitments( &self, - block: &SimpleBlockData, + block: SimpleBlockData, + codes: &[CodeId], + batch_filler: &mut BatchFiller, + ) -> Result> { + if utils::has_duplicates(codes) { + return Ok(Some(ValidationRejectReason::HaveDuplicates)); + } + + let waiting_codes = self + .db + .block_meta(block.hash) + .codes_queue + .ok_or_else(|| anyhow!("codes queue not found for block={}", block.hash))? + .into_iter() + .collect::>(); + + if let Some(&code_id) = codes.iter().find(|&id| !waiting_codes.contains(id)) { + return Ok(Some(ValidationRejectReason::CodeNotWaitingForCommitment( + code_id, + ))); + } + + for &id in codes.iter() { + let Some(valid) = self.db.code_valid(id) else { + return Ok(Some(ValidationRejectReason::CodeIsNotProcessedYet(id))); + }; + let code_commitment = CodeCommitment { id, valid }; + if batch_filler + .include_code_commitment(code_commitment) + .is_err() + { + return Ok(Some(ValidationRejectReason::BatchSizeLimitExceeded)); + } + } + + Ok(None) + } + + pub(crate) async fn aggregate_validators_commitment( + &self, + block: SimpleBlockData, ) -> Result> { let (timelines, max_validators) = { let config = self.db.config(); @@ -415,13 +458,13 @@ impl BatchCommitmentManager { unreachable!("no other options are possible here"); } - let mut iter_block = *block; + let mut cursor = block; let election_block = loop { - let parent_hash = iter_block.header.parent_hash; + let parent_hash = cursor.header.parent_hash; let Some(parent_header) = self.db.block_header(parent_hash) else { // This case can happen if node is started with fast sync and does not have full blocks history tracing::warn!( - iter_block = %iter_block.hash, + iter_block = %cursor.hash, parent = %parent_hash, "Parent block header not found when searching for election block, skipping validators commitment" ); @@ -430,11 +473,11 @@ impl BatchCommitmentManager { }; if parent_header.timestamp < election_ts { - break iter_block; + break cursor; } - iter_block = SimpleBlockData { - hash: iter_block.header.parent_hash, + cursor = SimpleBlockData { + hash: cursor.header.parent_hash, header: parent_header, } }; @@ -470,9 +513,9 @@ impl BatchCommitmentManager { } // TODO #4742 - pub async fn aggregate_rewards_commitment( + async fn aggregate_rewards_commitment( &self, - _block: &SimpleBlockData, + _block: SimpleBlockData, ) -> Result> { Ok(None) } diff --git a/ethexe/consensus/src/validator/batch/tests.rs b/ethexe/consensus/src/validator/batch/tests.rs index 2a3823c78bc..35e5c470007 100644 --- a/ethexe/consensus/src/validator/batch/tests.rs +++ b/ethexe/consensus/src/validator/batch/tests.rs @@ -14,7 +14,7 @@ use crate::validator::core::MiddlewareWrapper; use ethexe_common::{ Address, Digest, ProgramStates, Schedule, SimpleBlockData, ToDigest, ValidatorsVec, consensus::BatchCommitmentValidationRequest, - db::{BlockMetaStorageRW, CompactMb, GlobalsStorageRW, MbStorageRW, SetConfig}, + db::{BlockMetaStorageRW, CompactMb, GlobalsStorageRW, MbStorageRO, MbStorageRW, SetConfig}, gear::StateTransition, malachite::{Operation, Operations}, mock::*, @@ -81,7 +81,8 @@ fn append_mb(db: &Database, parent: H256, height: u64, outcome: Vec {} + ValidationStatus::Rejected { reason, .. } => { + panic!("expected acceptance via reachability, got rejection: {reason:?}") + } + } +} + +#[tokio::test] +async fn builds_chain_commitment_when_committed_anchor_compact_absent() { + // Regression for the validator-set handover stall: a freshly joined + // validator learns `last_committed_mb` only from the on-chain `MBCommitted` + // event (propagated into `BlockMeta.last_committed_mb`) and never computed + // that MB, so it has no `mb_compact_block` for the anchor. The producer must + // still build a chain commitment for the computed MBs descending from the + // anchor — terminating the parent walk on the anchor hash — instead of + // bailing with "last committed MB is still not synced locally". + let db = Database::memory(); + let chain = test_block_chain(3).setup(&db); + let block = chain.blocks[3].to_simple(); + + // The committed anchor is known only by hash: no compact block, no meta. + let anchor = H256::from([0x87; 32]); + let mb2 = append_mb(&db, anchor, 2, vec![nonempty_transition(2)]); + let mb3 = append_mb(&db, mb2, 3, vec![nonempty_transition(3)]); + db.globals_mutate(|g| g.latest_finalized_mb_hash = mb3); + db.mutate_block_meta(block.hash, |meta| { + meta.last_committed_mb = Some(anchor); + }); + // Sanity: the anchor genuinely has no local compact block. + assert!(db.mb_compact_block(anchor).is_none()); + + let manager = mock_batch_manager(db.clone()); + let batch = manager + .create_batch_commitment(block) + .await + .expect("must not error when the committed anchor compact block is absent") + .expect("expected a non-empty batch"); + + let chain_commitment = batch + .chain_commitment + .expect("computed MBs descend from the anchor → chain commitment expected"); + assert_eq!( + chain_commitment.head, mb3, + "chain commitment head must be the finalized tip" + ); + assert!( + !chain_commitment.transitions.is_empty(), + "transitions from the MBs after the anchor must be committed" + ); +} + +#[tokio::test] +async fn validates_chain_commitment_when_committed_anchor_compact_absent() { + // Validator-side counterpart: a participant that knows the committed anchor + // only by hash (no local compact block) must still accept a well-formed + // request whose head descends from that anchor, rather than hard-erroring. + let db = Database::memory(); + let chain = test_block_chain(3).setup(&db); + let block = chain.blocks[3].to_simple(); + + let anchor = H256::from([0x87; 32]); + let mb2 = append_mb(&db, anchor, 2, vec![nonempty_transition(2)]); + let mb3 = append_mb(&db, mb2, 3, vec![nonempty_transition(3)]); + db.globals_mutate(|g| g.latest_finalized_mb_hash = mb3); + db.mutate_block_meta(block.hash, |meta| { + meta.last_committed_mb = Some(anchor); + }); + assert!(db.mb_compact_block(anchor).is_none()); + + // Build the request from a node that did produce the batch. + let request = { + let manager = mock_batch_manager(db.clone()); + let batch = manager + .create_batch_commitment(block) + .await + .unwrap() + .expect("expected a non-empty batch"); + BatchCommitmentValidationRequest::new(&batch) + }; + + let manager = mock_batch_manager(db); + let status = manager + .validate_batch_commitment(block, request) + .await + .unwrap(); + match status { + ValidationStatus::Accepted(_) => {} + ValidationStatus::Rejected { reason, .. } => { + panic!("expected acceptance with anchor compact absent, got rejection: {reason:?}") + } + } +} + +#[tokio::test] +async fn does_not_hang_when_finalized_chain_does_not_reach_committed_anchor() { + // Reproduces the validator-set handover stall: the new set's finalized MB + // chain is rooted at the seeded genesis sentinel (the zero MB, which db init + // gives a SELF-parent), while `last_committed_mb` still points at the old + // set's on-chain committed MB that is NOT on this chain. The producer walk + // must stop at the zero sentinel and skip — not spin forever dereferencing + // its self-parent (which would wedge the single-threaded runtime). + let db = Database::memory(); + let chain = test_block_chain(3).setup(&db); + let block = chain.blocks[3].to_simple(); + + // Seed the zero MB exactly like db init: a computed MB whose parent is itself. + db.set_mb_compact_block( + H256::zero(), + CompactMb { + parent: H256::zero(), + height: 0, + operations_hash: db.set_operations(Operations::default()), + }, + ); + db.mutate_mb_meta(H256::zero(), |m| { + m.computed = true; + m.finalized = true; + m.last_advanced_eb = Some(H256::zero()); + }); + + // Finalized chain rooted at the zero sentinel (mb1.parent == zero). + setup_mb_chain( + &db, + vec![vec![nonempty_transition(1)], vec![nonempty_transition(2)]], + ); + + // Committed anchor is a nonzero MB that is NOT on the finalized chain. + let foreign_anchor = H256::from([0xDE; 32]); + db.mutate_block_meta(block.hash, |meta| { + meta.last_committed_mb = Some(foreign_anchor); + }); + + // Must return promptly (no hang) and produce no batch — the finalized chain + // does not descend from the committed anchor, so there is nothing to commit. + let manager = mock_batch_manager(db.clone()); + let batch = manager.create_batch_commitment(block).await.unwrap(); + assert!( + batch.is_none(), + "a finalized chain disconnected from the committed anchor must skip, not hang" + ); +} + #[tokio::test] async fn rejects_head_mb_at_or_below_last_committed_mb() { // The coordinator must always advance past `last_committed_mb`. If @@ -337,10 +515,10 @@ async fn rejects_head_mb_at_or_below_last_committed_mb() { .validate_batch_commitment(block, request) .await .unwrap(); - assert_eq!( - unwrap_rejected(status), - ValidationRejectReason::HeadMbAlreadyCommitted(head) - ); + // With head_mb == last_committed_mb there is nothing left to commit: + // the chain walk is empty, no chain/code/validators/rewards commitment + // is produced, so the batch is rejected as empty. + assert_eq!(unwrap_rejected(status), ValidationRejectReason::EmptyBatch); } #[tokio::test] @@ -421,29 +599,29 @@ async fn batch_size_limit_exceeded_is_rejected_on_validation() { let chain = test_block_chain(3).setup(&db); let block = chain.blocks[3].to_simple(); - // Pile up a chain of MBs with many transitions each so the squashed - // batch easily exceeds a tight size limit. - let mut outcomes = Vec::new(); - for mb_idx in 0..5u8 { - let mut o = Vec::new(); - for actor in 0..40u8 { - // distinct actor per transition so squashing keeps them all - o.push(nonempty_transition(mb_idx * 50 + actor + 1)); - } - outcomes.push(o); - } - setup_mb_chain(&db, outcomes); - - // First build under a generous limit, then validate under a tight - // one — that's how the manager catches an oversize batch from a - // misbehaving coordinator. + // Validation tolerates batches above the local `batch_size_limit` up to + // the protocol-wide `MAX_BATCH_SIZE_LIMIT`. To get a rejection, the batch + // must exceed that hard cap — pack one transition with a payload bigger + // than `MAX_BATCH_SIZE_LIMIT`. + let mut oversize = nonempty_transition(1); + oversize.messages = vec![ethexe_common::gear::Message { + id: Default::default(), + destination: ActorId::zero(), + payload: vec![0u8; ethexe_common::consensus::MAX_BATCH_SIZE_LIMIT as usize + 1024], + value: 0, + reply_details: None, + call: false, + }]; + setup_mb_chain(&db, vec![vec![oversize]]); + + // The coordinator builds the oversize batch under a generous local limit; + // the validator must still reject it for breaching the hard cap. let big_manager = mock_batch_manager_with_limits( db.clone(), BatchLimits { commitment_delay_limit: std::num::NonZero::new(100).unwrap(), - batch_size_limit: BLOCK_GAS_LIMIT, // large - // Large enough that the checkpoint path doesn't fire in this size-limit scenario. - uncommitted_chain_len_threshold: NonZero::new(u32::MAX).unwrap(), + batch_size_limit: u64::MAX, + checkpoint_threshold: NonZero::new(u32::MAX).unwrap(), }, ); let batch = big_manager @@ -453,16 +631,8 @@ async fn batch_size_limit_exceeded_is_rejected_on_validation() { .expect("expected non-empty batch"); let request = BatchCommitmentValidationRequest::new(&batch); - let strict_manager = mock_batch_manager_with_limits( - db, - BatchLimits { - commitment_delay_limit: std::num::NonZero::new(100).unwrap(), - batch_size_limit: 256, // intentionally tiny - // Large enough that the checkpoint path doesn't fire in this size-limit scenario. - uncommitted_chain_len_threshold: NonZero::new(u32::MAX).unwrap(), - }, - ); - let status = strict_manager + let manager = mock_batch_manager(db); + let status = manager .validate_batch_commitment(block, request) .await .unwrap(); @@ -559,7 +729,7 @@ async fn idle_chain_below_threshold_yields_no_batch_commitment() { // Anchor advance lands 2 Eth heights past the last committed anchor. let advanced = chain.blocks[4].hash; let last_committed_eb = chain.blocks[2].hash; - db.mutate_mb_meta(head_mb, |m| m.last_advanced_eb = advanced); + db.mutate_mb_meta(head_mb, |m| m.last_advanced_eb = Some(advanced)); db.mutate_block_meta(block.hash, |m| { m.last_committed_eb = Some(last_committed_eb) }); @@ -570,7 +740,7 @@ async fn idle_chain_below_threshold_yields_no_batch_commitment() { BatchLimits { commitment_delay_limit: std::num::NonZero::new(16).unwrap(), batch_size_limit: BLOCK_GAS_LIMIT, - uncommitted_chain_len_threshold: NonZero::new(10).unwrap(), + checkpoint_threshold: NonZero::new(10).unwrap(), }, ); @@ -602,7 +772,7 @@ async fn idle_chain_above_threshold_emits_checkpoint_batch_commitment() { // gap = height(blocks[5]) - height(blocks[1]) = 4 let advanced = chain.blocks[5].hash; let last_committed_eb = chain.blocks[1].hash; - db.mutate_mb_meta(head_mb, |m| m.last_advanced_eb = advanced); + db.mutate_mb_meta(head_mb, |m| m.last_advanced_eb = Some(advanced)); db.mutate_block_meta(block.hash, |m| { m.last_committed_eb = Some(last_committed_eb) }); @@ -613,7 +783,7 @@ async fn idle_chain_above_threshold_emits_checkpoint_batch_commitment() { BatchLimits { commitment_delay_limit: std::num::NonZero::new(16).unwrap(), batch_size_limit: BLOCK_GAS_LIMIT, - uncommitted_chain_len_threshold: threshold, + checkpoint_threshold: threshold, }, ); @@ -701,14 +871,14 @@ async fn test_aggregate_validators_commitment() { // Before election start (era 0, ts < genesis+50) → no commitment. let commitment = manager - .aggregate_validators_commitment(&chain.blocks[4].to_simple()) + .aggregate_validators_commitment(chain.blocks[4].to_simple()) .await .unwrap(); assert!(commitment.is_none(), "expected None before election period"); // Right at election start for era 1 → commits validators1. let commitment = manager - .aggregate_validators_commitment(&chain.blocks[5].to_simple()) + .aggregate_validators_commitment(chain.blocks[5].to_simple()) .await .unwrap() .expect("validators commitment expected"); @@ -720,7 +890,7 @@ async fn test_aggregate_validators_commitment() { // Inside era 1 election period → still validators1. let commitment = manager - .aggregate_validators_commitment(&chain.blocks[7].to_simple()) + .aggregate_validators_commitment(chain.blocks[7].to_simple()) .await .unwrap() .expect("validators commitment expected"); @@ -735,7 +905,7 @@ async fn test_aggregate_validators_commitment() { meta.latest_era_validators_committed = Some(1); }); let commitment = manager - .aggregate_validators_commitment(&chain.blocks[7].to_simple()) + .aggregate_validators_commitment(chain.blocks[7].to_simple()) .await .unwrap(); assert!( @@ -749,7 +919,7 @@ async fn test_aggregate_validators_commitment() { meta.latest_era_validators_committed = Some(0); }); let commitment = manager - .aggregate_validators_commitment(&chain.blocks[15].to_simple()) + .aggregate_validators_commitment(chain.blocks[15].to_simple()) .await .unwrap() .expect("validators commitment expected"); @@ -764,7 +934,7 @@ async fn test_aggregate_validators_commitment() { meta.latest_era_validators_committed = Some(3); }); manager - .aggregate_validators_commitment(&chain.blocks[15].to_simple()) + .aggregate_validators_commitment(chain.blocks[15].to_simple()) .await .unwrap_err(); } diff --git a/ethexe/consensus/src/validator/batch/types.rs b/ethexe/consensus/src/validator/batch/types.rs index 62adcceebc0..6acad950302 100644 --- a/ethexe/consensus/src/validator/batch/types.rs +++ b/ethexe/consensus/src/validator/batch/types.rs @@ -6,7 +6,9 @@ use core::num::NonZero; use ethexe_common::{ DEFAULT_COMMITMENT_DELAY_LIMIT, Digest, consensus::{BatchCommitmentValidationRequest, DEFAULT_BATCH_SIZE_LIMIT}, - gear::{ChainCommitment, CodeCommitment, RewardsCommitment, ValidatorsCommitment}, + gear::{ + ChainCommitment, CodeCommitment, RewardsCommitment, StateTransition, ValidatorsCommitment, + }, }; use ethexe_ethereum::abi::Gear; use gprimitives::{CodeId, H256}; @@ -22,7 +24,7 @@ pub struct BatchLimits { /// Force a checkpoint chain commitment when the producer's view of /// `last_advanced_eth_block` is more than this many blocks ahead of the /// last committed advanced block. - pub uncommitted_chain_len_threshold: NonZero, + pub checkpoint_threshold: NonZero, } impl Default for BatchLimits { @@ -30,7 +32,7 @@ impl Default for BatchLimits { BatchLimits { commitment_delay_limit: DEFAULT_COMMITMENT_DELAY_LIMIT, batch_size_limit: DEFAULT_BATCH_SIZE_LIMIT, - uncommitted_chain_len_threshold: NonZero::new(500).expect("500 != 0"), + checkpoint_threshold: NonZero::new(500).expect("500 != 0"), } } } @@ -56,22 +58,24 @@ impl BatchSizeCounter { Self(max_size) } - pub fn charge_for_validators_commitment( - &mut self, - commitment: &Option, - ) -> bool { - self.charge_optional::(commitment.clone()) + pub fn charge_for_validators_commitment(&mut self, commitment: &ValidatorsCommitment) -> bool { + self.charge_optional::(Some( + commitment.clone(), + )) + } + + pub fn charge_for_rewards_commitment(&mut self, commitment: &RewardsCommitment) -> bool { + self.charge_optional::<_, Gear::RewardsCommitment>(Some(commitment.clone())) } - pub fn charge_for_rewards_commitment( - &mut self, - commitment: &Option, - ) -> bool { - self.charge_optional::<_, Gear::RewardsCommitment>(commitment.clone()) + pub fn charge_for_chain_commitment(&mut self, commitment: &ChainCommitment) -> bool { + self.charge_optional::<_, Gear::ChainCommitment>(Some(commitment.clone())) } - pub fn charge_for_chain_commitment(&mut self, commitment: &Option) -> bool { - self.charge_optional::<_, Gear::ChainCommitment>(commitment.clone()) + pub fn charge_for_transitions(&mut self, transitions: &[StateTransition]) -> bool { + let encoded: Vec = + transitions.iter().cloned().map(Into::into).collect(); + self.charge_value(&encoded) } pub fn charge_for_code_commitment(&mut self, commitment: &CodeCommitment) -> bool { @@ -106,7 +110,7 @@ impl BatchSizeCounter { #[derive(Debug, Clone, Default)] pub struct BatchParts { - pub chain_commitment: Option, + pub chain_commitment: Option<(ChainCommitment, NonZero)>, pub code_commitments: Vec, pub validators_commitment: Option, pub rewards_commitment: Option, @@ -125,34 +129,45 @@ pub enum ValidationStatus { #[derive(Debug, derive_more::Display, Clone, PartialEq, Eq)] pub enum ValidationRejectReason { + // common reasons for batch #[display("batch commitment is empty")] EmptyBatch, - #[display("batch commitment request contains duplicate code ids")] - CodesHaveDuplicates, - #[display("code id {_0} is not waiting for commitment")] - CodeNotWaitingForCommitment(CodeId), - #[display("code id {_0} is not processed yet")] - CodeIsNotProcessedYet(CodeId), + #[display("batch commitment digest mismatch: expected {expected}, found {found}")] + BatchDigestMismatch { expected: Digest, found: Digest }, + #[display("batch size exceeded the maximum size limit")] + BatchSizeLimitExceeded, + + // validators election and rewards distribution + #[display("batch has validators commitment, but it's not time for validators election yet")] + ValidatorsNotReady, + #[display("batch has rewards commitment, but it's not time for rewards distribution yet")] + RewardsNotReady, + + // chain commitment (head MB) #[display("requested head MB {_0} is not finalized locally")] HeadMbNotFinalized(H256), - #[display("requested head MB {_0} is at or below last committed MB")] - HeadMbAlreadyCommitted(H256), - #[display("requested head MB {_0} is not computed by this node")] + #[display("requested head MB {_0} is not computed locally")] HeadMbNotComputed(H256), #[display( - "latest finalized MB advance {_0} is not on the canonical chain ending at the current head" + "requested head MB {head_mb} is not a strict descendant of the latest committed MB {latest_committed_mb}" )] - LatestFinalizedAdvanceNotCanonical(H256), - #[display( - "received batch contains validators commitment, but it's not time for validators election yet" - )] - ValidatorsNotReady, + HeadMbNotStrictDescendantOfLatestCommittedMb { + head_mb: H256, + latest_committed_mb: H256, + }, #[display( - "received batch contains rewards commitment, but it's not time for rewards distribution yet" + "last advanced EB {last_advanced_eb} is not on the canonical chain of the last committed advanced EB {last_committed_advanced_eb}" )] - RewardsNotReady, - #[display("batch commitment digest mismatch: expected {expected}, found {found}")] - BatchDigestMismatch { expected: Digest, found: Digest }, - #[display("batch size exceeded the maximum size limit")] - BatchSizeLimitExceeded, + LastAdvancedEbNotOnCanonicalChain { + last_advanced_eb: H256, + last_committed_advanced_eb: H256, + }, + + // code commitments + #[display("contains duplicate code ids")] + HaveDuplicates, + #[display("code id {_0} is not waiting for commitment")] + CodeNotWaitingForCommitment(CodeId), + #[display("code id {_0} is not processed yet")] + CodeIsNotProcessedYet(CodeId), } diff --git a/ethexe/consensus/src/validator/batch/utils.rs b/ethexe/consensus/src/validator/batch/utils.rs index e23a5f9eec3..b150edfafaa 100644 --- a/ethexe/consensus/src/validator/batch/utils.rs +++ b/ethexe/consensus/src/validator/batch/utils.rs @@ -3,163 +3,101 @@ use crate::validator::batch::{filler::BatchFiller, types::BatchParts}; -use anyhow::{Result, anyhow, bail}; -use core::num::NonZero; +use anyhow::{Context, Result, anyhow}; use ethexe_common::{ - SimpleBlockData, - db::{BlockMetaStorageRO, CodesStorageRO, MbStorageRO, OnChainStorageRO}, + BlockHeader, SimpleBlockData, + db::{ + BlockMetaStorageRO, CodesStorageRO, ConfigStorageRO, GlobalsStorageRO, MbStorageRO, + OnChainStorageRO, + }, gear::{ BatchCommitment, ChainCommitment, CodeCommitment, Message, StateTransition, ValueClaim, }, }; use gprimitives::{ActorId, H256}; -use std::collections::{HashMap, hash_map::Entry}; - -/// MBs in `(last_committed_mb, mb_hash]`, chronological order. Strict: errors -/// if the walk doesn't reach the anchor or any MB along the way is not computed. -/// Used on the participant path; lenient producer counterpart is -/// [`collect_computed_uncommitted_predecessors`]. -pub fn collect_not_committed_mb_predecessors( - db: &DB, - last_committed_mb: H256, - mb_hash: H256, -) -> Result> { - let mut mbs = Vec::new(); - let mut current = mb_hash; - - while current != last_committed_mb { - if current == H256::zero() { - bail!( - "MB chain walk reached genesis without finding last_committed_mb {last_committed_mb}" - ); - } - - let meta = db.mb_meta(current); - if !meta.computed { - bail!("MB {current} in chain is not computed"); - } - - mbs.push(current); - current = db - .mb_compact_block(current) - .ok_or_else(|| anyhow!("MB {current} missing compact block — DB invariant"))? - .parent; - } - - Ok(mbs.into_iter().rev().collect()) -} - -/// Producer-path lenient counterpart: longest computed prefix anchored at -/// `last_committed_mb`. Returns empty when the first successor isn't yet -/// computed or the parent walk doesn't reach the anchor (e.g. fresh restart). -pub fn collect_computed_uncommitted_predecessors( - db: &DB, - last_committed_mb: H256, - mb_head: H256, -) -> Vec { - // Walk the parent chain backward from `mb_head` until we either - // reach `last_committed_mb` or run off the local chain. - let mut chain = Vec::new(); // newest-first - let mut current = mb_head; - while current != last_committed_mb && current != H256::zero() { - let meta = db.mb_meta(current); - chain.push((current, meta.computed)); - current = db - .mb_compact_block(current) - .map(|c| c.parent) - .unwrap_or(H256::zero()); - } - if current != last_committed_mb { - // Walk didn't reach the anchor (fast-restart / sync-lag); caller retries. - tracing::warn!( - %last_committed_mb, - %mb_head, - walk_depth = chain.len(), - "parent walk did not reach last_committed_mb — chain commitment skipped", - ); - return Vec::new(); - } - - chain.reverse(); - - // Longest contiguous computed prefix anchored at `last_committed_mb`. - let mut collected = Vec::with_capacity(chain.len()); - for (hash, computed) in chain.iter().copied() { - if !computed { - break; - } - collected.push(hash); - } - collected -} - -/// `true` iff `candidate` is reachable from `latest_finalized_mb` by walking -/// `parent_mb_hash`. Sound by BFT linear-order; bounded by the height gap. -/// `H256::zero()` is the genesis sentinel. -pub fn is_finalized_locally( - db: &DB, - candidate: H256, - latest_finalized_mb: H256, -) -> bool { - if candidate == H256::zero() || candidate == latest_finalized_mb { - return true; - } - if latest_finalized_mb == H256::zero() { - return false; - } - let mut current = latest_finalized_mb; - while current != H256::zero() { - if current == candidate { - return true; - } - current = db - .mb_compact_block(current) - .map(|c| c.parent) - .unwrap_or(H256::zero()); - } - false -} +use std::{ + collections::{HashMap, HashSet, VecDeque, hash_map::Entry}, + num::NonZero, +}; pub fn create_batch_commitment( db: &DB, block: &SimpleBlockData, batch_parts: BatchParts, - commitment_delay_limit: std::num::NonZero, + commitment_delay_limit: NonZero, + checkpoint_threshold: NonZero, ) -> Result> { let BatchParts { - chain_commitment, + chain_commitment: chain_commitment_with_len, validators_commitment, code_commitments, rewards_commitment, } = batch_parts; - let block_hash = block.hash; - - if chain_commitment.is_none() - && code_commitments.is_empty() - && validators_commitment.is_none() - && rewards_commitment.is_none() - { - tracing::debug!("No commitments for block {block_hash} - skip batch commitment"); - return Ok(None); - } + let SimpleBlockData { + hash: block_hash, + header: BlockHeader { timestamp, .. }, + } = *block; + + let has_other_commitments = !code_commitments.is_empty() + || validators_commitment.is_some() + || rewards_commitment.is_some(); + + let chain_commitment = match chain_commitment_with_len { + Some((commitment, len)) => { + // A chain commitment carrying no transitions only earns its place + // at a genuine checkpoint — advancing the on-chain Ethereum anchor + // after a long quiet stretch (`len >= checkpoint_threshold`). It + // must NOT ride along merely because the batch carries other + // commitments: emitting an empty chain commitment fires + // `MBCommitted`, pinning `last_committed_mb` to an MB that a freshly + // re-synced validator set (e.g. after a validator-set handover) may + // not have on its locally rebuilt chain — after which it can never + // produce a descendant chain commitment and stalls. + if commitment.transitions.is_empty() && len < checkpoint_threshold { + tracing::debug!( + %block_hash, + %len, + %checkpoint_threshold, + has_other_commitments, + "Chain commitment is empty and checkpoint threshold not reached, dropping it" + ); + if !has_other_commitments { + return Ok(None); + } + None + } else { + tracing::debug!( + %block_hash, + %len, + %checkpoint_threshold, + transitions_len = commitment.transitions.len(), + "Including chain commitment into batch" + ); + Some(commitment) + } + } + None => { + if !has_other_commitments { + tracing::debug!(%block_hash, "Nothing to commit, skip batch commitment"); + return Ok(None); + } + None + } + }; let previous_batch = db - .block_meta(block.hash) + .block_meta(block_hash) .last_committed_batch - .ok_or_else( - || anyhow!("Cannot get from db last committed block for block {block_hash}",), - )?; - - let expiry: u8 = commitment_delay_limit.get(); - - tracing::trace!("Batch commitment expiry for block {block_hash} is {expiry:?}",); + .with_context(|| { + format!("Cannot get from db last committed block for block {block_hash}") + })?; Ok(Some(BatchCommitment { block_hash, - timestamp: block.header.timestamp, + timestamp, previous_batch, - expiry, + expiry: commitment_delay_limit.get(), chain_commitment, code_commitments, validators_commitment, @@ -195,140 +133,154 @@ pub fn aggregate_code_commitments_for_block( +/// Producer chain-commitment builder. +pub fn try_include_chain_commitment< + DB: ConfigStorageRO + GlobalsStorageRO + BlockMetaStorageRO + MbStorageRO + OnChainStorageRO, +>( db: &DB, at_block: H256, - mb_head: H256, batch_filler: &mut BatchFiller, -) -> Result { - let last_committed_mb = db - .block_meta(at_block) - .last_committed_mb - .unwrap_or(H256::zero()); +) -> Result<()> { + let latest_finalized_mb = db.globals().latest_finalized_mb_hash; + if latest_finalized_mb.is_zero() { + return Ok(()); + } - let pending = collect_computed_uncommitted_predecessors(db, last_committed_mb, mb_head); + let latest_advanced_eb_hash = db + .mb_meta(latest_finalized_mb) + .last_advanced_eb + .context("latest finalized mb must have latest advanced eb info")?; - if pending.is_empty() { - // Nothing computed in range; producer skips chain commitment this round. - return Ok(last_committed_mb); + if !is_strict_descendant_eth_block(db, at_block, latest_advanced_eb_hash)? { + tracing::error!( + %at_block, + %latest_finalized_mb, + %latest_advanced_eb_hash, + "latest advanced eth block is not strict ancestor of the current chain head, skipping chain commitment" + ); + return Ok(()); } - // Aggregate transitions incrementally; stop when the next MB blows the size budget. - let mut transitions: Vec = Vec::new(); - let mut last_included = last_committed_mb; - for mb_hash in &pending { - let Some(mb_transitions) = db.mb_outcome(*mb_hash) else { - anyhow::bail!("Computed MB {mb_hash} outcome not found in db"); - }; - - // Trial-fit this MB; bail if it pushes us past the batch size budget. - let len_before = transitions.len(); - transitions.extend(mb_transitions); - let trial_commitment = ChainCommitment { - head: *mb_hash, - transitions, - last_advanced_eth_block: db.mb_meta(*mb_hash).last_advanced_eb, + let last_committed_mb_hash = db + .block_meta(at_block) + .last_committed_mb + .with_context(|| format!("at_block {at_block} must be prepared at this moment"))?; + + // `last_committed_mb_hash` is the MB last committed on-chain. A freshly + // joined validator may know it only from the on-chain `MBCommitted` event + // (propagated into `BlockMeta.last_committed_mb`) and never have computed + // that MB, so its compact block can be absent locally. We must therefore + // never dereference the anchor itself: the parent walk below terminates on + // its hash, and running off the locally-known chain before reaching it is a + // lenient skip (sync lag / not yet a local ancestor), retried later. + let mut cursor_mb_hash = latest_finalized_mb; + let mut cursor_mb = db + .mb_compact_block(cursor_mb_hash) + .context("latest finalized MB must have compact block in db")?; + + // Skip the finalized-but-not-yet-computed suffix at the tip, stopping at the + // latest computed MB. + while !db.mb_meta(cursor_mb_hash).computed { + let parent_hash = cursor_mb.parent; + if cursor_mb_hash == last_committed_mb_hash || parent_hash == last_committed_mb_hash { + tracing::debug!( + %at_block, + %latest_finalized_mb, + %last_committed_mb_hash, + "no computed MBs since latest committed MB, skipping chain commitment" + ); + return Ok(()); + } + if parent_hash.is_zero() { + // The genesis sentinel (zero MB) is seeded with a self-parent (see + // db init), so reaching it means the finalized chain does not + // descend from the nonzero committed anchor — stop instead of + // spinning on the self-loop. + tracing::warn!( + %at_block, + %latest_finalized_mb, + %last_committed_mb_hash, + "chain walk reached genesis without finding last committed MB, skipping chain commitment" + ); + return Ok(()); + } + let Some(parent_mb) = db.mb_compact_block(parent_hash) else { + tracing::warn!( + %at_block, + %latest_finalized_mb, + %last_committed_mb_hash, + "chain walk left the local chain before reaching last committed MB, skipping chain commitment" + ); + return Ok(()); }; - let would_fit = batch_filler.would_fit_chain_commitment(&trial_commitment); - transitions = trial_commitment.transitions; + cursor_mb_hash = parent_hash; + cursor_mb = parent_mb; + } - if !would_fit { - let _ = transitions.split_off(len_before); + // Collect computed-but-not-committed MBs down to (exclusive) the committed anchor. + let mut computed_not_committed_mbs = VecDeque::new(); + while cursor_mb_hash != last_committed_mb_hash { + // push_front to maintain chronological order from oldest to newest + computed_not_committed_mbs.push_front(cursor_mb_hash); + let parent_hash = cursor_mb.parent; + if parent_hash == last_committed_mb_hash { break; } - - last_included = *mb_hash; - } - - // Skip the commitment entirely when there are no state transitions - // to carry on-chain. Pushing the Ethereum anchor forward on every - // idle round would spam pointless batches; the dedicated checkpoint - // path ([`try_include_checkpoint_chain_commitment`]) gates that on - // `uncommitted_chain_len_threshold` and emits the empty-transitions - // commitment only after a long quiet stretch. - if transitions.is_empty() { - return Ok(last_committed_mb); - } - - let commitment = ChainCommitment { - head: last_included, - transitions, - last_advanced_eth_block: db.mb_meta(last_included).last_advanced_eb, - }; - - if let Err(err) = batch_filler.include_chain_commitment(commitment) { - tracing::trace!( - "failed to include chain commitment for head MB {mb_head} because of error={err}" - ); - return Ok(last_committed_mb); - } - - Ok(last_included) -} - -/// If `last_advanced_eth_block` of `mb_head` is more than `threshold` Eth blocks -/// past `block.last_committed_eb`, force an empty chain commitment -/// that pins the head MB and the new advanced anchor on-chain. -pub fn try_include_checkpoint_chain_commitment< - DB: BlockMetaStorageRO + MbStorageRO + OnChainStorageRO, ->( - db: &DB, - at_block: H256, - mb_head: H256, - threshold: NonZero, - batch_filler: &mut BatchFiller, -) -> Result<()> { - let advanced = db.mb_meta(mb_head).last_advanced_eb; - if advanced.is_zero() { - return Ok(()); + if parent_hash.is_zero() { + // Genesis sentinel reached without hitting the committed anchor: the + // finalized chain is not a descendant of it. The zero MB is seeded + // with a self-parent (see db init), so stop here rather than spin. + tracing::warn!( + %at_block, + %latest_finalized_mb, + %last_committed_mb_hash, + "chain walk reached genesis without finding last committed MB, skipping chain commitment" + ); + return Ok(()); + } + let Some(parent_mb) = db.mb_compact_block(parent_hash) else { + tracing::warn!( + %at_block, + %latest_finalized_mb, + %last_committed_mb_hash, + "chain walk left the local chain before reaching last committed MB, skipping chain commitment" + ); + return Ok(()); + }; + cursor_mb_hash = parent_hash; + cursor_mb = parent_mb; } - let Some(advanced_header) = db.block_header(advanced) else { - return Ok(()); - }; - // `at_block` is `prepared` by the time the coordinator runs (see - // `Idle`), so the field must be populated. - let last_committed_advanced = db.block_meta(at_block).last_committed_eb.ok_or_else(|| { - anyhow::anyhow!("block_meta({at_block}).last_committed_eb missing despite prepared==true") - })?; - let last_committed_height = if last_committed_advanced.is_zero() { - 0 - } else { - db.block_header(last_committed_advanced) - .ok_or_else(|| { - anyhow::anyhow!( - "block_header({last_committed_advanced}) missing for at_block {at_block}" - ) - })? - .height - }; + // Collect commitment + for cursor in computed_not_committed_mbs.into_iter() { + let transitions = db + .mb_outcome(cursor) + .with_context(|| format!("computed MB {cursor} outcome not found in db"))?; - let gap = advanced_header.height.saturating_sub(last_committed_height); - if gap <= threshold.get() { - return Ok(()); - } + let last_advanced_eth_block = db + .mb_meta(cursor) + .last_advanced_eb + .with_context(|| format!("computed MB {cursor} has no last_advanced_eb in db"))?; - let commitment = ChainCommitment { - head: mb_head, - transitions: Vec::new(), - last_advanced_eth_block: advanced, - }; + let one_block_commitment = ChainCommitment { + head: cursor, + transitions, + last_advanced_eth_block, + }; - if let Err(err) = batch_filler.include_chain_commitment(commitment) { - tracing::trace!( - "checkpoint chain commitment didn't fit (head {mb_head}, advanced {advanced}): {err}" - ); - } else { - tracing::info!( - %mb_head, - %advanced, - gap, - threshold = threshold.get(), - "emitting checkpoint chain commitment" - ); + // Producer is lenient: once an MB would push the batch past the size + // budget, stop here and commit what fits. The next round picks up the + // remainder. + if batch_filler + .append_chain_commitment(one_block_commitment) + .is_err() + { + tracing::debug!( + %cursor, + "chain commitment size limit reached, committing collected MBs only" + ); + break; + } } Ok(()) @@ -466,205 +418,82 @@ pub fn sort_transitions_by_value_to_receive(transitions: &mut [StateTransition]) transitions.sort_by_key(|transition| !transition.value_to_receive_negative_sign); } -#[cfg(test)] -mod tests { - use super::*; - use ethexe_common::{ - Schedule, - db::{CompactMb, MbStorageRW}, - malachite::{Operation, Operations}, - }; - use ethexe_db::Database; - - /// Per-height unique CAS via `AdvanceTillEthereumBlock` salt. - fn empty_ops(height: u64) -> Operations { - Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { - block_hash: H256::from_low_u64_be(0xEB00 + height), - }, - Operation::ProcessQueuesV3 { gas_allowance: 0 }, - ]) - } - - /// Mimics malachite `process_mb_proposal` + executor's `meta.computed` flip. - fn write_mb( - db: &Database, - parent_mb: H256, - height: u64, - outcome: Vec, - ) -> H256 { - let ops = empty_ops(height); - let operations_hash = db.set_operations(ops); - // Synthetic mb_hash; only uniqueness matters here. - let mb_hash = H256::from_low_u64_be(0x1000 + height); - db.set_mb_compact_block( - mb_hash, - CompactMb { - parent: parent_mb, - height, - operations_hash, - }, - ); - db.set_mb_outcome(mb_hash, outcome); - db.set_mb_schedule(mb_hash, Schedule::default()); - db.mutate_mb_meta(mb_hash, |meta| { - meta.computed = true; - meta.last_advanced_eb = H256::zero(); - }); - mb_hash - } - - #[test] - fn collect_predecessors_walks_chain() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - let mb3 = write_mb(&db, mb2, 3, vec![]); - - let walked = collect_not_committed_mb_predecessors(&db, H256::zero(), mb3).unwrap(); - assert_eq!(walked, vec![mb1, mb2, mb3]); - - let from_mb1 = collect_not_committed_mb_predecessors(&db, mb1, mb3).unwrap(); - assert_eq!(from_mb1, vec![mb2, mb3]); - } - - #[test] - fn collect_predecessors_returns_empty_when_at_target() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - - let walked = collect_not_committed_mb_predecessors(&db, mb1, mb1).unwrap(); - assert!(walked.is_empty()); - } - - #[test] - fn collect_predecessors_errors_when_target_not_in_chain() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - - // mb2 cannot trace back to a hash that's not on the chain. - let bogus = H256::from_low_u64_be(0xDEAD); - let err = collect_not_committed_mb_predecessors(&db, bogus, mb2).unwrap_err(); - let msg = format!("{err:#}"); - assert!(msg.contains("genesis"), "got: {msg}"); - } - - #[test] - fn collect_predecessors_errors_on_uncomputed_mb() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - // Force mb2 to look uncomputed. - db.mutate_mb_meta(mb2, |meta| meta.computed = false); - - let err = collect_not_committed_mb_predecessors(&db, H256::zero(), mb2).unwrap_err(); - let msg = format!("{err:#}"); - assert!(msg.contains("not computed"), "got: {msg}"); - } - - #[test] - fn lenient_collect_returns_full_range_when_all_computed() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - let mb3 = write_mb(&db, mb2, 3, vec![]); - - let walked = collect_computed_uncommitted_predecessors(&db, H256::zero(), mb3); - assert_eq!(walked, vec![mb1, mb2, mb3]); - - let from_mb1 = collect_computed_uncommitted_predecessors(&db, mb1, mb3); - assert_eq!(from_mb1, vec![mb2, mb3]); - } - - #[test] - fn lenient_collect_truncates_at_first_uncomputed() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - let mb3 = write_mb(&db, mb2, 3, vec![]); - // Compute is lagging: mb2 hasn't finished yet. - db.mutate_mb_meta(mb2, |meta| meta.computed = false); - - // Only mb1 is contiguous-computed from anchor; mb2 gap blocks the rest. - let walked = collect_computed_uncommitted_predecessors(&db, H256::zero(), mb3); - assert_eq!(walked, vec![mb1]); - } - - #[test] - fn lenient_collect_returns_empty_when_first_successor_uncomputed() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - db.mutate_mb_meta(mb1, |meta| meta.computed = false); - - let walked = collect_computed_uncommitted_predecessors(&db, H256::zero(), mb1); - assert!(walked.is_empty()); - } - - #[test] - fn lenient_collect_returns_empty_when_chain_does_not_reach_anchor() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - - let bogus = H256::from_low_u64_be(0xDEAD); - // Walk doesn't hit `bogus`; producer skips silently instead of erroring. - let walked = collect_computed_uncommitted_predecessors(&db, bogus, mb1); - assert!(walked.is_empty()); - } - - #[test] - fn lenient_collect_returns_empty_when_at_target() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); +pub fn has_duplicates(data: &[T]) -> bool { + let mut seen = HashSet::new(); + data.iter().any(|item| !seen.insert(item)) +} - let walked = collect_computed_uncommitted_predecessors(&db, mb1, mb1); - assert!(walked.is_empty()); +pub fn is_strict_descendant_eth_block( + db: &DB, + block: H256, + ancestor: H256, +) -> Result { + if ancestor.is_zero() { + // The genesis/pre-genesis anchor is an ancestor-or-equal of every + // anchor, including the genesis anchor itself: a chain commitment is + // allowed even when the Eth anchor has not advanced past genesis yet. + return Ok(true); + } + + let ancestor_height = db + .block_header(ancestor) + .ok_or_else(|| anyhow!("eth chain walk: missing header for ancestor {ancestor}"))? + .height; + + let mut current = block; + while current != ancestor { + if current.is_zero() { + return Ok(false); + } + let header = db + .block_header(current) + .ok_or_else(|| anyhow!("eth chain walk: missing header for {current}"))?; + if header.height <= ancestor_height { + return Ok(false); + } + current = header.parent_hash; } - #[test] - fn is_finalized_zero_candidate_is_universally_finalized() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - assert!(is_finalized_locally(&db, H256::zero(), mb1)); - // Even with no local finalization yet, zero is the genesis sentinel. - assert!(is_finalized_locally(&db, H256::zero(), H256::zero())); - } + Ok(true) +} - #[test] - fn is_finalized_self_is_finalized() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - assert!(is_finalized_locally(&db, mb1, mb1)); +/// `true` iff `candidate` is BFT-finalized locally — i.e. it is `H256::zero()` +/// (genesis sentinel) or reachable from `latest_finalized_mb` by walking +/// `CompactMb::parent`. +/// +/// This is the source of truth for "finalized locally": any two finalized MBs +/// are linearly ordered by BFT, so reachability from the finalized tip is an +/// exact membership test, and — unlike the per-MB `MbMeta::finalized` cache — +/// it also covers MBs a node learned about indirectly (sync, on-chain +/// `MBCommitted`) without replaying `process_mb_finalized` for each one. +pub fn is_finalized_locally( + db: &DB, + candidate: H256, + latest_finalized_mb: H256, +) -> bool { + if candidate.is_zero() || candidate == latest_finalized_mb { + return true; } - - #[test] - fn is_finalized_resolves_proper_ancestor_of_finalized_head() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - let mb3 = write_mb(&db, mb2, 3, vec![]); - // Latest finalized is mb3 → mb1 and mb2 are also finalized. - assert!(is_finalized_locally(&db, mb1, mb3)); - assert!(is_finalized_locally(&db, mb2, mb3)); + if latest_finalized_mb.is_zero() { + return false; } - - #[test] - fn is_finalized_returns_false_for_descendant_of_finalized_head() { - // Speculative-but-not-yet-finalized candidate must fail strict check. - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - let mb2 = write_mb(&db, mb1, 2, vec![]); - let mb3 = write_mb(&db, mb2, 3, vec![]); - assert!(!is_finalized_locally(&db, mb3, mb1)); - assert!(!is_finalized_locally(&db, mb2, mb1)); + let mut current = latest_finalized_mb; + while !current.is_zero() { + if current == candidate { + return true; + } + current = db + .mb_compact_block(current) + .map(|c| c.parent) + .unwrap_or(H256::zero()); } + false +} - #[test] - fn is_finalized_returns_false_when_no_local_finalization() { - let db = Database::memory(); - let mb1 = write_mb(&db, H256::zero(), 1, vec![]); - assert!(!is_finalized_locally(&db, mb1, H256::zero())); - } +#[cfg(test)] +mod tests { + use super::*; + use ethexe_db::Database; #[test] fn create_batch_commitment_writes_commitment_delay_limit_into_expiry() { @@ -695,20 +524,23 @@ mod tests { }; let parts = BatchParts { - chain_commitment: Some(ChainCommitment { - transitions: vec![StateTransition { - actor_id: gprimitives::ActorId::from([0xAB; 32]), - new_state_hash: H256::from_low_u64_be(0xDEAD_BEEF), - exited: false, - inheritor: Default::default(), - value_to_receive: 0, - value_to_receive_negative_sign: false, - value_claims: vec![], - messages: vec![], - }], - head: block_hash, - last_advanced_eth_block: H256::zero(), - }), + chain_commitment: Some(( + ChainCommitment { + transitions: vec![StateTransition { + actor_id: gprimitives::ActorId::from([0xAB; 32]), + new_state_hash: H256::from_low_u64_be(0xDEAD_BEEF), + exited: false, + inheritor: Default::default(), + value_to_receive: 0, + value_to_receive_negative_sign: false, + value_claims: vec![], + messages: vec![], + }], + head: block_hash, + last_advanced_eth_block: H256::zero(), + }, + NonZero::new(1).unwrap(), + )), code_commitments: vec![], validators_commitment: None, rewards_commitment: None, @@ -724,6 +556,7 @@ mod tests { &block, parts.clone(), NonZero::new(raw_limit).unwrap(), + NonZero::new(1).unwrap(), ) .unwrap() .expect("non-empty batch commitment"); @@ -733,22 +566,6 @@ mod tests { } } - #[test] - fn is_finalized_returns_false_on_disjoint_chain() { - let db = Database::memory(); - let chain_a = write_mb(&db, H256::zero(), 1, vec![]); - let chain_b_root = H256::from_low_u64_be(0xB001); - db.set_mb_compact_block( - chain_b_root, - CompactMb { - parent: H256::from_low_u64_be(0xB000), // unknown parent - height: 1, - operations_hash: db.set_operations(empty_ops(99)), - }, - ); - assert!(!is_finalized_locally(&db, chain_b_root, chain_a)); - } - #[test] fn test_squash_transitions_by_actor() { use ethexe_common::gear::Message; @@ -1061,4 +878,45 @@ mod tests { assert_eq!(squashed[0].value_to_receive, 0); assert!(!squashed[0].value_to_receive_negative_sign); } + + #[test] + fn is_strict_descendant_eth_block_walks_canonical_chain() { + use ethexe_common::{BlockHeader, db::OnChainStorageRW}; + + let db = Database::memory(); + let header = |height, parent_hash| BlockHeader { + height, + timestamp: height as u64, + parent_hash, + }; + // Linear eth chain b1 <- b2 <- b3, plus a sibling fork at b2's height. + let b1 = H256::from_low_u64_be(0xE1); + let b2 = H256::from_low_u64_be(0xE2); + let b3 = H256::from_low_u64_be(0xE3); + let fork = H256::from_low_u64_be(0xF2); + db.set_block_header(b1, header(1, H256::zero())); + db.set_block_header(b2, header(2, b1)); + db.set_block_header(b3, header(3, b2)); + db.set_block_header(fork, header(2, b1)); + + // Genesis anchor (zero) is ancestor-or-equal of every block, itself included. + assert!(is_strict_descendant_eth_block(&db, H256::zero(), H256::zero()).unwrap()); + assert!(is_strict_descendant_eth_block(&db, b3, H256::zero()).unwrap()); + + // Equal non-zero anchors are accepted: the anchor may stay put between commits. + assert!(is_strict_descendant_eth_block(&db, b2, b2).unwrap()); + + // Proper descendants. + assert!(is_strict_descendant_eth_block(&db, b3, b1).unwrap()); + assert!(is_strict_descendant_eth_block(&db, b3, b2).unwrap()); + + // Non-descendants: sibling fork, or a block at/below the ancestor height. + assert!(!is_strict_descendant_eth_block(&db, fork, b2).unwrap()); + assert!(!is_strict_descendant_eth_block(&db, b1, b2).unwrap()); + assert!(!is_strict_descendant_eth_block(&db, fork, b3).unwrap()); + + // A missing header on the walk surfaces as an error. + let missing = H256::from_low_u64_be(0xDEAD); + assert!(is_strict_descendant_eth_block(&db, missing, b1).is_err()); + } } diff --git a/ethexe/consensus/src/validator/mod.rs b/ethexe/consensus/src/validator/mod.rs index c902f081397..7b3e8733dbc 100644 --- a/ethexe/consensus/src/validator/mod.rs +++ b/ethexe/consensus/src/validator/mod.rs @@ -106,7 +106,7 @@ impl ValidatorService { let limits = BatchLimits { commitment_delay_limit: config.commitment_delay_limit, batch_size_limit: config.batch_size_limit, - uncommitted_chain_len_threshold: config.uncommitted_chain_len_threshold, + checkpoint_threshold: config.uncommitted_chain_len_threshold, }; let middleware = MiddlewareWrapper::from_inner(election_provider); diff --git a/ethexe/db/src/migrations/init.rs b/ethexe/db/src/migrations/init.rs index ec12bb575f8..bd90b996070 100644 --- a/ethexe/db/src/migrations/init.rs +++ b/ethexe/db/src/migrations/init.rs @@ -10,7 +10,9 @@ use anyhow::{Context as _, Result, ensure}; use ethexe_common::{ BlockHeader, ProgramStates, ProtocolTimelines, Schedule, SimpleBlockData, StateHashWithQueueSize, - db::{CodesStorageRO, CodesStorageRW, CompactMb, MbStorageRW, PreparedBlockData}, + db::{ + CodesStorageRO, CodesStorageRW, CompactMb, MbStorageRW, OnChainStorageRW, PreparedBlockData, + }, gear::{GenesisBlockInfo, Timelines}, malachite::Operations, }; @@ -75,7 +77,8 @@ async fn validate_db(config: InitConfig, db: &RawDatabase) -> Result<()> { pub async fn initialize_empty_db(config: InitConfig, db: &RawDatabase) -> Result<()> { let provider = RootProvider::connect(&config.ethereum_rpc).await?; let chain_id = provider.get_chain_id().await?; - let storage_view = RouterQuery::from_provider(config.router_address, provider) + let router_query = RouterQuery::from_provider(config.router_address, provider); + let storage_view = router_query .storage_view_at(alloy::eips::BlockId::latest()) .await .context("Empty db init, failed read router data")?; @@ -125,7 +128,8 @@ pub async fn initialize_empty_db(config: InitConfig, db: &RawDatabase) -> Result db.set_mb_outcome(genesis_parent_mb_hash, Vec::new()); db.mutate_mb_meta(genesis_parent_mb_hash, |m| { m.computed = true; - m.last_advanced_eb = H256::zero(); + m.finalized = true; + m.last_advanced_eb = Some(H256::zero()); }); ethexe_common::setup_block_in_db( @@ -175,6 +179,15 @@ pub async fn initialize_empty_db(config: InitConfig, db: &RawDatabase) -> Result latest_computed_mb_hash: genesis_parent_mb_hash, }; + // Seed the genesis era's on-chain validator set. A synced block must always + // have its era's validators in the db (the consensus resolver relies on it), + // and the genesis block is synced at init — so set it here rather than + // waiting on the first live observer sync. + if let Some(genesis_era) = db_config.timelines.era_from_ts(genesis_eb.header.timestamp) { + let genesis_validators = router_query.validators_at(genesis_eb.hash).await?; + db.set_validators(genesis_era, genesis_validators); + } + db.kv.set_globals(globals); db.kv.set_config(db_config); diff --git a/ethexe/db/src/migrations/mod.rs b/ethexe/db/src/migrations/mod.rs index f6194df3760..daab0bd1c22 100644 --- a/ethexe/db/src/migrations/mod.rs +++ b/ethexe/db/src/migrations/mod.rs @@ -16,12 +16,13 @@ pub use init::initialize_db; mod init; mod migration; mod v1; +mod v2; -pub const LATEST_VERSION: u32 = v1::VERSION; +pub const LATEST_VERSION: u32 = v2::VERSION; pub const OLDEST_SUPPORTED_VERSION: u32 = v1::VERSION; -pub const MIGRATIONS: &[&dyn Migration] = &[]; +pub const MIGRATIONS: &[&dyn Migration] = &[&v2::MigrationFromV1]; const _: () = assert!( (LATEST_VERSION - OLDEST_SUPPORTED_VERSION) as usize == MIGRATIONS.len(), diff --git a/ethexe/db/src/migrations/v1.rs b/ethexe/db/src/migrations/v1.rs index 671d0cbf5df..f0faaa0e9dc 100644 --- a/ethexe/db/src/migrations/v1.rs +++ b/ethexe/db/src/migrations/v1.rs @@ -1,4 +1,16 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 +use gprimitives::H256; +use parity_scale_codec::Decode; + pub const VERSION: u32 = 1; + +/// Frozen v1 on-disk layout of `MbMeta`. Lives here because it describes the +/// v1 database version; the v1 -> v2 migration decodes existing records with +/// it before re-encoding them in the v2 layout. +#[derive(Decode)] +pub struct MbMeta { + pub computed: bool, + pub last_advanced_eb: H256, +} diff --git a/ethexe/db/src/migrations/v2.rs b/ethexe/db/src/migrations/v2.rs new file mode 100644 index 00000000000..c47243c17ef --- /dev/null +++ b/ethexe/db/src/migrations/v2.rs @@ -0,0 +1,223 @@ +// Copyright (C) Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +//! Migration v1 -> v2. +//! +//! `MbMeta` gained a `finalized` flag, and its `last_advanced_eb` changed from +//! `H256` to `Option`. Every stored `MbMeta` record is re-encoded: the +//! old `last_advanced_eb` is wrapped into `Some(..)`, and `finalized` is +//! backfilled to `true` for every MB reachable from `latest_finalized_mb_hash` +//! — mirroring the old reachability-based "finalized locally" semantics. + +use super::{InitConfig, migration::Migration, v1}; +use crate::RawDatabase; +use anyhow::{Context, Result}; +use ethexe_common::db::{CompactMb, DBConfig, DBGlobals}; +use gprimitives::H256; +use parity_scale_codec::{Decode, Encode}; +use std::{collections::HashSet, pin::Pin}; + +pub const VERSION: u32 = 2; + +// Discriminants frozen from `database::Key` at the time of this migration. +// They must not be changed even if the live `Key` enum is renumbered later. +const MB_META: u64 = 22; +const MB_COMPACT_BLOCK: u64 = 25; +const GLOBALS: u64 = 14; +const CONFIG: u64 = 15; + +fn prefix(discriminant: u64) -> [u8; 32] { + H256::from_low_u64_be(discriminant).to_fixed_bytes() +} + +fn hash_key(discriminant: u64, hash: H256) -> Vec { + let mut key = prefix(discriminant).to_vec(); + key.extend_from_slice(hash.as_ref()); + key +} + +fn singleton_key(discriminant: u64) -> Vec { + let mut key = prefix(discriminant).to_vec(); + key.extend_from_slice(&[0u8; 8]); + key +} + +/// Frozen v2 layout of `MbMeta`. +#[derive(Encode)] +struct MbMetaV2 { + computed: bool, + finalized: bool, + last_advanced_eb: Option, +} + +pub struct MigrationFromV1; + +impl Migration for MigrationFromV1 { + fn source_version(&self) -> u32 { + VERSION - 1 + } + + fn migrate<'a>( + &'a self, + _config: &'a InitConfig, + db: &'a RawDatabase, + ) -> Pin> + 'a>> { + Box::pin(async move { + let finalized = collect_finalized_mbs(db)?; + + let meta_prefix = prefix(MB_META); + let entries: Vec<(Vec, Vec)> = db.kv.iter_prefix(&meta_prefix).collect(); + for (key, value) in entries { + let mb_hash = H256::from_slice(&key[meta_prefix.len()..]); + let old = v1::MbMeta::decode(&mut value.as_slice()) + .with_context(|| format!("failed to decode v1 MbMeta for {mb_hash}"))?; + let new = MbMetaV2 { + computed: old.computed, + finalized: finalized.contains(&mb_hash), + last_advanced_eb: Some(old.last_advanced_eb), + }; + db.kv.put(&key, new.encode()); + } + + // Bump the persisted schema version to v2. + let mut config = DBConfig::decode( + &mut db + .kv + .get(&singleton_key(CONFIG)) + .context("config not found during v1 -> v2 migration")? + .as_slice(), + ) + .context("failed to decode config during v1 -> v2 migration")?; + config.version = VERSION; + db.kv.put(&singleton_key(CONFIG), config.encode()); + + Ok(()) + }) + } +} + +/// Collect every MB hash reachable from `latest_finalized_mb_hash` by walking +/// `CompactMb::parent`, plus the genesis zero MB. The walk terminates at the +/// zero sentinel, a missing compact block, or a cycle. +fn collect_finalized_mbs(db: &RawDatabase) -> Result> { + let globals = DBGlobals::decode( + &mut db + .kv + .get(&singleton_key(GLOBALS)) + .context("globals not found during v1 -> v2 migration")? + .as_slice(), + ) + .context("failed to decode globals during v1 -> v2 migration")?; + + let mut finalized = HashSet::new(); + let mut current = globals.latest_finalized_mb_hash; + while !current.is_zero() && finalized.insert(current) { + let Some(raw) = db.kv.get(&hash_key(MB_COMPACT_BLOCK, current)) else { + break; + }; + let compact = CompactMb::decode(&mut raw.as_slice()) + .with_context(|| format!("failed to decode CompactMb for {current}"))?; + current = compact.parent; + } + // The genesis zero MB has its own `MbMeta` row and is finalized by definition. + finalized.insert(H256::zero()); + Ok(finalized) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{MemDb, RawDatabase}; + use ethexe_common::{Address, ProtocolTimelines, SimpleBlockData}; + + fn put_v1_meta(db: &RawDatabase, hash: H256, computed: bool, last_advanced_eb: H256) { + // SCALE-encodes identically to the frozen v1 `MbMeta` struct. + db.kv.put( + &hash_key(MB_META, hash), + (computed, last_advanced_eb).encode(), + ); + } + + fn put_compact(db: &RawDatabase, hash: H256, parent: H256) { + let compact = CompactMb { + parent, + height: 0, + operations_hash: H256::zero(), + }; + db.kv + .put(&hash_key(MB_COMPACT_BLOCK, hash), compact.encode()); + } + + #[test] + fn migrate_reencodes_mb_meta_and_backfills_finalized() { + let db = RawDatabase::from_one(&MemDb::default()); + + // Chain: zero <- mb1 <- mb2 (finalized head) ; mb3 is a computed, + // not-yet-finalized tip above mb2. + let mb1 = H256::from_low_u64_be(0x1); + let mb2 = H256::from_low_u64_be(0x2); + let mb3 = H256::from_low_u64_be(0x3); + put_compact(&db, mb1, H256::zero()); + put_compact(&db, mb2, mb1); + put_compact(&db, mb3, mb2); + + put_v1_meta(&db, H256::zero(), true, H256::zero()); + put_v1_meta(&db, mb1, true, H256::from_low_u64_be(0xE1)); + put_v1_meta(&db, mb2, true, H256::from_low_u64_be(0xE2)); + put_v1_meta(&db, mb3, true, H256::from_low_u64_be(0xE3)); + + let globals = DBGlobals { + start_block_hash: H256::zero(), + latest_synced_eb: SimpleBlockData::default(), + latest_prepared_eb_hash: H256::zero(), + latest_finalized_mb_hash: mb2, + latest_computed_mb_hash: mb3, + }; + db.kv.put(&singleton_key(GLOBALS), globals.encode()); + + let config = DBConfig { + version: 1, + chain_id: 0, + router_address: Address([0; 20]), + timelines: ProtocolTimelines { + genesis_ts: 0, + era: 1.try_into().unwrap(), + election: 0, + slot: 1.try_into().unwrap(), + }, + genesis_block_hash: H256::zero(), + max_validators: 10, + }; + db.kv.put(&singleton_key(CONFIG), config.encode()); + + let init_config = InitConfig { + ethereum_rpc: String::new(), + router_address: Address([0; 20]), + slot_duration_secs: 1, + genesis_initializer: None, + }; + futures::executor::block_on(MigrationFromV1.migrate(&init_config, &db)).unwrap(); + + let decode_meta = |hash: H256| { + let raw = db.kv.get(&hash_key(MB_META, hash)).unwrap(); + ethexe_common::db::MbMeta::decode(&mut raw.as_slice()).unwrap() + }; + + // Finalized chain (zero, mb1, mb2) is backfilled true; the tip mb3 false. + assert!(decode_meta(H256::zero()).finalized); + assert!(decode_meta(mb1).finalized); + assert!(decode_meta(mb2).finalized); + assert!(!decode_meta(mb3).finalized); + + // `last_advanced_eb` is wrapped into `Some(..)`. + assert_eq!( + decode_meta(mb1).last_advanced_eb, + Some(H256::from_low_u64_be(0xE1)) + ); + + // Version bumped to v2. + let migrated_config = + DBConfig::decode(&mut db.kv.get(&singleton_key(CONFIG)).unwrap().as_slice()).unwrap(); + assert_eq!(migrated_config.version, VERSION); + } +} diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 4e7d06bc656..8a11022d385 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -30,14 +30,17 @@ use crate::{ codec::{decode_value, encode_value}, - context::{Height, MalachiteCtx, ProposalPart, ValueId}, + context::{ + EQUAL_VOTING_POWER, Height, MalachiteCtx, ProposalPart, Validator, ValidatorSet, ValueId, + }, externalities::Externalities, + signing::public_key_from_gsigner, state::State, store::BlockEntry, streaming::ProposalParts, types::{Address, Block, CommitCertificate, H256}, }; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result, anyhow, ensure}; use bytes::Bytes; use ethexe_common::Acceptance; use malachitebft_app_channel::{ @@ -130,8 +133,15 @@ where match msg { // ConsensusReady AppMsg::ConsensusReady { reply } => { + let start_height = self.state.current_height; + info!(%start_height, "Consensus ready"); + let params = HeightParams::new( + self.validator_set_for_height(start_height).await?, + self.state.get_timeouts(start_height), + None, + ); reply - .send(self.process_consensus_ready()) + .send((start_height, params)) .map_err(|e| anyhow!("failed to send ConsensusReady reply: {e:?}"))?; } @@ -245,32 +255,26 @@ where evidence = ?evidence, "Finalized" ); - let next = match self.process_finalized(certificate).await { - Ok(()) => { - let h = self.state.current_height; - Next::Start( - h, - HeightParams::new( - self.state.get_validator_set(h), - self.state.get_timeouts(h), - None, - ), - ) - } - Err(FinalizationError::NonFatal(e)) => { - let h = self.state.current_height; - error!(?e, height = %h, "Finalized: commit failed — restarting height"); - Next::Restart( - h, - HeightParams::new( - self.state.get_validator_set(h), - self.state.get_timeouts(h), - None, - ), - ) - } + + let res = match self.process_finalized(certificate).await { + Ok(()) => Ok(()), + Err(FinalizationError::NonFatal(e)) => Err(e), Err(FinalizationError::Fatal(e)) => { - return Err(anyhow!("Fatal error during finalization: {e:?}")); + Err(anyhow!("Fatal error during finalization: {e:?}"))? + } + }; + + let h = self.state.current_height; + let validators_set = self.validator_set_for_height(h).await.with_context(|| { + format!("FATAL: failed to resolve validator set for height {h}") + })?; + let params = HeightParams::new(validators_set, self.state.get_timeouts(h), None); + + let next = match res { + Ok(()) => Next::Start(h, params), + Err(err) => { + error!(%err, height = %h, "Finalized: commit failed — restarting height"); + Next::Restart(h, params) } }; reply @@ -334,20 +338,35 @@ where Ok(()) } - // --------------------------- processors --------------------------- - - /// Infallible: the start height was resolved at [`State::new`] - /// and lives in `self.state.current_height`. Nothing here touches - /// the store, so this can never fail at message-handling time. - fn process_consensus_ready(&self) -> (Height, HeightParams) { - let start_height = self.state.current_height; - info!(%start_height, "Consensus ready"); - let params = HeightParams::new( - self.state.get_validator_set(start_height), - self.state.get_timeouts(start_height), - None, + async fn validator_set_for_height(&self, height: Height) -> Result { + let parent_mb_hash = if height.as_u64() <= 1 { + // The parent of the genesis MB is the zero hash. + H256::zero() + } else { + let parent_height = height.as_u64() - 1; + self.state + .store + .finalized_block_at(parent_height)? + .with_context(|| format!("no finalized MB at height {parent_height}"))? + }; + + let public_keys = self + .externalities + .validators_for_child_of(parent_mb_hash) + .await?; + + ensure!( + !public_keys.is_empty(), + "empty validator set resolved for height {height}" ); - (start_height, params) + + let mut validators = Vec::with_capacity(public_keys.len()); + for public_key in &public_keys { + let pk = public_key_from_gsigner(public_key) + .context("converting gsigner pub key to malachite key")?; + validators.push(Validator::new(pk, EQUAL_VOTING_POWER)); + } + Ok(ValidatorSet::new(validators)) } async fn process_started_round( @@ -767,9 +786,8 @@ fn compute_value_id_from_parts(parts: &ProposalParts) -> ValueId { mod tests { use super::*; use crate::{ - context::{ProposalData, ProposalInit, Validator, ValidatorSet, Value}, + context::{ProposalData, ProposalInit, Value}, signing::{MalachiteSigner, libp2p_peer_id, private_key_from_bytes}, - state::SharedValidatorSet, store::Store, types::BlockPayload, }; @@ -806,6 +824,12 @@ mod tests { ) -> Result> { Ok(Acceptance::Accepted(())) } + async fn validators_for_child_of( + &self, + _: H256, + ) -> Result> { + Err(anyhow!("test mock does not resolve era validators")) + } } fn test_signer(byte: u8) -> MalachiteSigner { @@ -859,18 +883,7 @@ mod tests { let store = Store::open(dir.path()).unwrap(); let signer = test_signer(1); let address = Address::from_public_key(&signer.public_key()); - let validator_set = SharedValidatorSet::new(ValidatorSet::new(vec![Validator::new( - signer.public_key(), - 1, - )])); - let mut state = State::new( - signer, - validator_set, - address, - store, - Duration::from_secs(1), - ) - .unwrap(); + let mut state = State::new(signer, address, store, Duration::from_secs(1)).unwrap(); state.current_height = Height::new(current_height); let (_consensus_tx, consensus_rx) = mpsc::channel::>(1); @@ -993,6 +1006,12 @@ mod tests { ) -> Result> { Ok(Acceptance::Accepted(())) } + async fn validators_for_child_of( + &self, + _: H256, + ) -> Result> { + Err(anyhow!("test mock does not resolve era validators")) + } } /// Same shape as [`make_handler`] but with a caller-supplied @@ -1006,18 +1025,7 @@ mod tests { let store = Store::open(dir.path()).unwrap(); let signer = test_signer(1); let address = Address::from_public_key(&signer.public_key()); - let validator_set = SharedValidatorSet::new(ValidatorSet::new(vec![Validator::new( - signer.public_key(), - 1, - )])); - let mut state = State::new( - signer, - validator_set, - address, - store, - Duration::from_secs(1), - ) - .unwrap(); + let mut state = State::new(signer, address, store, Duration::from_secs(1)).unwrap(); state.current_height = Height::new(current_height); let (_consensus_tx, consensus_rx) = mpsc::channel::>(1); diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index d793bba6658..e87a23ef4d8 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -7,21 +7,17 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; pub use malachitebft_app_channel::app::net::Multiaddr; -/// One entry of the validator set. +/// secp256k1 public key of one validator. The on-chain address is derived from +/// it (`keccak256(uncompressed_pubkey[1..])[12..]`). The set is unweighted — +/// every validator carries the same voting power, so the BFT quorum threshold +/// is a plain `> 2/3` of the validator count. // -// TODO: #5480 add `libp2p_peer_id: PeerId` so receivers can gate -// `ReceivedProposalPart` against a validator-peer-id allowlist -// (libp2p peer-id is not derivable from `public_key` alone — operators -// must compute it offline via `libp2p_peer_id(&secret)` and embed it). -#[derive(Clone, Debug)] -pub struct ValidatorEntry { - /// secp256k1 public key for this validator. The on-chain address - /// is derived from it (`keccak256(uncompressed_pubkey[1..])[12..]`). - pub public_key: gsigner::schemes::secp256k1::PublicKey, - /// Voting power. Must be > 0; the BFT quorum threshold is - /// `> 2/3` of the total voting power across the set. - pub voting_power: u64, -} +// TODO: #5480 if receivers ever need to gate `ReceivedProposalPart` against a +// validator-peer-id allowlist, this must grow into a struct carrying a +// `libp2p_peer_id: PeerId` alongside the key (the peer-id is not derivable +// from `public_key` alone — operators must compute it offline via +// `libp2p_peer_id(&secret)` and embed it). +pub type ValidatorPublicKey = gsigner::schemes::secp256k1::PublicKey; /// Role this node plays in the BFT swarm. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -55,7 +51,7 @@ pub struct MalachiteCoreConfig { /// Validator set the engine uses to drive consensus /// (see [`NodeRole`] for local-key membership rules). - pub validators: Vec, + pub validators: Vec, /// Whether this node casts votes (`Validator`) or just observes /// (`FullNode`). @@ -65,15 +61,3 @@ pub struct MalachiteCoreConfig { /// before the round rolls over. pub propose_timeout: Duration, } - -impl MalachiteCoreConfig { - /// Default propose timeout. - pub const DEFAULT_PROPOSE_TIMEOUT: Duration = Duration::from_secs(13); - - /// Default libp2p listen address — TCP next to the typical - /// 20333/udp application QUIC port. - pub const DEFAULT_LISTEN_ADDR: SocketAddr = SocketAddr::new( - std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), - 20334, - ); -} diff --git a/ethexe/malachite/core/src/context.rs b/ethexe/malachite/core/src/context.rs index 85b11a35b0f..a275a22bd7a 100644 --- a/ethexe/malachite/core/src/context.rs +++ b/ethexe/malachite/core/src/context.rs @@ -111,6 +111,10 @@ impl fmt::Debug for ValueId { } } +/// Voting power every validator is given. The set is unweighted, so quorum +/// reduces to a plain `> 2/3` of the validator count. +pub const EQUAL_VOTING_POWER: VotingPower = 1; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Validator { pub address: Address, diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index ad52163c86c..35de4dfdc14 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -3,7 +3,10 @@ //! Application callbacks the service makes to the outside world. -use crate::types::{Block, BlockPayload, CommitCertificate, H256}; +use crate::{ + config::ValidatorPublicKey, + types::{Block, BlockPayload, CommitCertificate, H256}, +}; use anyhow::Result; use async_trait::async_trait; use ethexe_common::Acceptance; @@ -51,4 +54,16 @@ pub trait Externalities: Send + Sync + 'static { parent_mb_hash: H256, payload: &BlockPayload, ) -> Result>; + + /// Resolve the on-chain validator set that governs the **child** MB of + /// `parent_mb_hash` (zero = the genesis MB's parent). The governing era is + /// the one the parent's last `AdvanceTillEthereumBlock` landed in, so once + /// a producer advances into the next era every descendant MB is built and + /// validated against that era's set. Used by the consensus layer to verify + /// each height's commit certificate against the correct historical set + /// during sync, not just the live shared set. + async fn validators_for_child_of( + &self, + parent_mb_hash: H256, + ) -> Result>; } diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 4b0c9c15dbd..722a5e13593 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -48,11 +48,11 @@ //! - [`Externalities`] — Async application callbacks: `process_mb_proposal`, `process_mb_finalized`, `build_block_above`, //! `validate_block_above`. //! - [`MalachiteCore`] — Running service; owns the swarm and store. Implements `Stream` and -//! [`MService`]. `update_validators` rotates the active validator set, taking effect at the next height boundary. +//! [`MService`]. Per-height validators are resolved from the era via [`Externalities::validators_for_child_of`]. //! - [`MService`] — Supertrait bound implemented by [`MalachiteCore`]. //! - [`Block`] — Service-level block envelope: `{ parent_hash: H256, height: u64, payload: BlockPayload, reserved: [u8; 64] }`. -//! - [`ValidatorEntry`] — Validator set member: `public_key` + `voting_power`, used in [`MalachiteCoreConfig`] and -//! `update_validators`. +//! - [`ValidatorPublicKey`] — Validator set member (secp256k1 public key); the set is unweighted, used in +//! [`MalachiteCoreConfig`]. //! - [`MalachiteCoreConfig`] — Node configuration: validator secret, validator set, `persistent_peers`, propose timeout, //! [`NodeRole`], `listen_addr`, and `base` project directory. //! - [`CommitCertificate`] — Finalization certificate delivered with `process_mb_finalized`. @@ -61,8 +61,7 @@ //! //! ## Caller invariants //! -//! - `listen_addr` must be set explicitly; [`MalachiteCoreConfig::DEFAULT_LISTEN_ADDR`] is -//! available but is not applied by any struct-level default. +//! - `listen_addr` must be set explicitly; there is no struct-level default. //! - `base` must be a persistent path: the service writes `/malachite/` (store and //! WAL) on first run and resumes from it on restart; a transient path loses BFT state. //! - Every persistent-peer multiaddr must include a `/p2p/` suffix (see @@ -89,7 +88,7 @@ mod store; mod streaming; pub use crate::{ - config::{MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry}, + config::{MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorPublicKey}, externalities::Externalities, service::{MService, MalachiteCore}, signing::{ diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 7be2c95a1cc..1d10ea16295 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -7,12 +7,12 @@ use crate::{ app, codec::ScaleCodec, config::{MalachiteCoreConfig, NodeRole}, - context::{MalachiteCtx, Validator, ValidatorSet}, + context::{EQUAL_VOTING_POWER, MalachiteCtx, Validator, ValidatorSet}, externalities::Externalities, signing::{ MalachiteSigner, libp2p_keypair_from, private_key_from_gsigner, public_key_from_gsigner, }, - state::{SharedValidatorSet, State}, + state::State, store::Store, types::Address, }; @@ -58,8 +58,6 @@ pub struct MalachiteCore { /// WAL file path; [`Self::shutdown`] probes its advisory lock before /// returning so a restart on the same base dir doesn't race the writer. wal_path: PathBuf, - /// Shared with the app loop; [`Self::update_validators`] writes here. - validator_set: SharedValidatorSet, /// Keeps the externalities alive for the app task. _externalities: Arc, } @@ -162,14 +160,13 @@ impl MalachiteCore { return Err(anyhow::anyhow!("MalachiteCoreConfig::validators is empty")); } let mut validators = Vec::with_capacity(config.validators.len()); - for entry in &config.validators { - let pk = public_key_from_gsigner(&entry.public_key) - .context("converting validator public key")?; - validators.push(Validator::new(pk, entry.voting_power)); + for public_key in &config.validators { + let pk = + public_key_from_gsigner(public_key).context("converting validator public key")?; + validators.push(Validator::new(pk, EQUAL_VOTING_POWER)); } let initial_validator_set = ValidatorSet::new(validators); let in_set = initial_validator_set.get_by_address(&address).is_some(); - let validator_set = SharedValidatorSet::new(initial_validator_set); // ---- network identity, role-dependent ---- let identity = match config.role { @@ -229,13 +226,7 @@ impl MalachiteCore { // ---- store + state ---- let store = Store::open(&store_path).context("opening Store")?; - let state = State::new( - signer, - validator_set.clone(), - address, - store, - config.propose_timeout, - )?; + let state = State::new(signer, address, store, config.propose_timeout)?; // ---- spawn app task ---- let (errors_tx, errors_rx) = mpsc::unbounded_channel(); @@ -252,31 +243,9 @@ impl MalachiteCore { engine, app_handle, wal_path, - validator_set, _externalities: externalities, }) } - - /// Swap the active validator set, taking effect at the next height start - /// (the current height runs to completion with the old set). - /// The caller must keep the local key in the set while in - /// [`NodeRole::Validator`]. Empty input is rejected. - pub fn update_validators(&self, validators: Vec) -> Result<()> { - if validators.is_empty() { - return Err(anyhow::anyhow!( - "MalachiteCore::update_validators: empty validators list" - )); - } - let mut converted = Vec::with_capacity(validators.len()); - for entry in &validators { - let pk = public_key_from_gsigner(&entry.public_key) - .context("converting validator public key")?; - converted.push(Validator::new(pk, entry.voting_power)); - } - let new_set = ValidatorSet::new(converted); - self.validator_set.update(new_set); - Ok(()) - } } impl Stream for MalachiteCore { diff --git a/ethexe/malachite/core/src/state.rs b/ethexe/malachite/core/src/state.rs index efa0fb465bc..e255fac1ee5 100644 --- a/ethexe/malachite/core/src/state.rs +++ b/ethexe/malachite/core/src/state.rs @@ -9,10 +9,7 @@ //! cascade-save / cascade-finalize flows live in [`crate::app`] //! which calls into this struct. -use std::{ - sync::{Arc, RwLock}, - time::Duration, -}; +use std::time::Duration; use anyhow::{Result, anyhow}; use malachitebft_app_channel::app::{ @@ -27,8 +24,8 @@ use malachitebft_core_types::CommitCertificate; use crate::{ context::{ - Height, MalachiteCtx, ProposalData, ProposalFin, ProposalInit, ProposalPart, ValidatorSet, - Value, sign_proposal_fin, + Height, MalachiteCtx, ProposalData, ProposalFin, ProposalInit, ProposalPart, Value, + sign_proposal_fin, }, signing::MalachiteSigner, store::Store, @@ -49,32 +46,10 @@ pub struct DecidedValue { pub certificate: CommitCertificate, } -/// Shared validator set handle — an external writer swaps the set -/// in [`Self::update`], and the next `ConsensusReady` / `Finalized` -/// reply via [`State::get_validator_set`] picks it up. -#[derive(Clone)] -pub(crate) struct SharedValidatorSet(Arc>); - -impl SharedValidatorSet { - pub fn new(set: ValidatorSet) -> Self { - Self(Arc::new(RwLock::new(set))) - } - - pub fn get(&self) -> ValidatorSet { - self.0.read().expect("validator set lock poisoned").clone() - } - - pub fn update(&self, set: ValidatorSet) { - *self.0.write().expect("validator set lock poisoned") = set; - } -} - /// Volatile bookkeeping of the app event loop. pub(crate) struct State { /// Consensus signer of the local node. pub signer: MalachiteSigner, - /// Active validator set handle. - pub validator_set: SharedValidatorSet, /// Local node's address. pub address: Address, /// Persistent block store. @@ -94,7 +69,6 @@ pub(crate) struct State { impl State { pub fn new( signer: MalachiteSigner, - validator_set: SharedValidatorSet, address: Address, store: Store, propose_timeout: Duration, @@ -105,7 +79,6 @@ impl State { .unwrap_or_else(|| Height::INITIAL); Ok(Self { signer, - validator_set, address, store, streams_map: PartStreamsMap::new(), @@ -116,10 +89,6 @@ impl State { }) } - pub fn get_validator_set(&self, _height: Height) -> ValidatorSet { - self.validator_set.get() - } - /// Round timeouts. Propose phase is bounded by the configured /// [`crate::MalachiteCoreConfig::propose_timeout`] plus a small margin /// for non-proposers; everything else (including the per-round diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index b7d7569059b..d9b072475ea 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -38,7 +38,7 @@ use async_trait::async_trait; use ethexe_common::Acceptance; use ethexe_malachite_core::{ Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteCore, - MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry, libp2p_peer_id, + MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorPublicKey, libp2p_peer_id, }; use proptest::prelude::*; use tempfile::TempDir; @@ -87,12 +87,21 @@ struct TestState { violations: Vec, } -#[derive(Default)] struct TestExt { state: Mutex, + /// Active validator set, returned by [`Externalities::validators_for_child_of`] + /// (the resolver no longer falls back to the engine's shared set). + validators: Vec, } impl TestExt { + fn new(validators: Vec) -> Self { + Self { + state: Mutex::new(TestState::default()), + validators, + } + } + fn finalized_count(&self) -> usize { self.state.lock().unwrap().finalized.len() } @@ -190,6 +199,12 @@ impl Externalities for TestExt { } Ok(Acceptance::Accepted(())) } + + async fn validators_for_child_of(&self, _parent_hash: H256) -> Result> { + // These tests run a single era; every height resolves to the same + // configured validator set. + Ok(self.validators.clone()) + } } // -------------------------------------------------------------------- @@ -247,14 +262,8 @@ fn make_validators(n: usize) -> Vec { .collect() } -fn validator_entries(setups: &[ValidatorSetup]) -> Vec { - setups - .iter() - .map(|s| ValidatorEntry { - public_key: s.private_key.public_key(), - voting_power: 1, - }) - .collect() +fn validator_entries(setups: &[ValidatorSetup]) -> Vec { + setups.iter().map(|s| s.private_key.public_key()).collect() } fn build_multiaddrs_excluding(setups: &[ValidatorSetup], exclude: usize) -> Vec { @@ -284,7 +293,7 @@ fn build_config( fn build_config_with_role( setup: &ValidatorSetup, peers: Vec, - validators: Vec, + validators: Vec, role: NodeRole, ) -> MalachiteCoreConfig { MalachiteCoreConfig { @@ -351,7 +360,9 @@ fn assert_no_violations(name: &str, ext: &TestExt) { async fn three_validators_make_progress() { init_tracing(); let setups = make_validators(3); - let exts: Vec> = (0..3).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..3) + .map(|_| Arc::new(TestExt::new(validator_entries(&setups)))) + .collect(); let mut services = Vec::with_capacity(3); for (i, setup) in setups.iter().enumerate() { let svc = start_service(setup, &setups, i, Arc::clone(&exts[i])).await; @@ -381,7 +392,9 @@ async fn seven_validators_full_network_restart() { let setups = make_validators(7); // One Arc per validator slot — reused across the // restart so the contract checks accumulate. - let exts: Vec> = (0..7).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..7) + .map(|_| Arc::new(TestExt::new(validator_entries(&setups)))) + .collect(); // ---- first run ------------------------------------------------ let mut services = Vec::with_capacity(7); @@ -431,7 +444,9 @@ async fn seven_validators_full_network_restart() { async fn restart_one_validator_mid_run() { let setups = make_validators(3); - let exts: Vec> = (0..3).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..3) + .map(|_| Arc::new(TestExt::new(validator_entries(&setups)))) + .collect(); let mut services: Vec>> = Vec::with_capacity(3); for (i, setup) in setups.iter().enumerate() { let svc = start_service(setup, &setups, i, Arc::clone(&exts[i])).await; @@ -477,15 +492,14 @@ async fn restart_one_validator_mid_run() { #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn full_node_syncs_from_validators() { let setups = make_validators(4); - let validator_set: Vec = setups[..3] + let validator_set: Vec = setups[..3] .iter() - .map(|s| ValidatorEntry { - public_key: s.private_key.public_key(), - voting_power: 1, - }) + .map(|s| s.private_key.public_key()) .collect(); - let exts: Vec> = (0..4).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..4) + .map(|_| Arc::new(TestExt::new(validator_set.clone()))) + .collect(); let mut services = Vec::with_capacity(4); for (i, setup) in setups.iter().enumerate() { let role = if i < 3 { @@ -564,7 +578,9 @@ fn run_churn_scenario(events: Vec) { let quorum = 2 * n / 3 + 1; let setups = make_validators(n); - let exts: Vec> = (0..n).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..n) + .map(|_| Arc::new(TestExt::new(validator_entries(&setups)))) + .collect(); let mut services: Vec>> = (0..n).map(|_| None).collect(); // Bootstrap all validators with a stagger. @@ -648,7 +664,9 @@ async fn shutdown_releases_wal_advisory_lock() { init_tracing(); let setups = make_validators(3); - let exts: Vec> = (0..3).map(|_| Arc::new(TestExt::default())).collect(); + let exts: Vec> = (0..3) + .map(|_| Arc::new(TestExt::new(validator_entries(&setups)))) + .collect(); let wal_paths: Vec = setups .iter() .map(|s| s.home.path().join("malachite").join("consensus.wal")) diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index e177b259294..cb35b6feb45 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -10,7 +10,7 @@ use crate::Mempool; use ethexe_common::ecdsa::{PublicKey, Signer}; -pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; +pub use ethexe_malachite_core::{Multiaddr, ValidatorPublicKey}; use std::{net::SocketAddr, path::PathBuf, time::Duration}; #[derive(Clone, Debug)] @@ -40,9 +40,10 @@ pub struct MalachiteServiceConfig { /// must include a `/p2p/` suffix (discovery is off). pub persistent_peers: Vec, - /// The complete validator set. Quorum is `> 2/3` of total voting power. - /// A validator node's own public key must appear in this list. - pub validators: Vec, + /// The complete validator set. The set is unweighted, so quorum is + /// `> 2/3` of the validator count. A validator node's own public key + /// must appear in this list. + pub validators: Vec, /// How long the proposer may wait for proposable content before the /// round times out and rotates. @@ -96,7 +97,7 @@ impl MalachiteServiceConfig { /// Replace the validator set. #[must_use] - pub fn with_validators(mut self, validators: Vec) -> Self { + pub fn with_validators(mut self, validators: Vec) -> Self { self.validators = validators; self } diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index a1e0894b96c..68505115d35 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -46,18 +46,25 @@ use crate::{ use anyhow::{Context, Result, anyhow, ensure}; use async_trait::async_trait; use ethexe_common::{ - Acceptance, MAX_TOUCHED_PROGRAMS_PER_MB, + Acceptance, Address, MAX_TOUCHED_PROGRAMS_PER_MB, db::{ - CompactMb, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW, OnChainStorageRO, + CompactMb, ConfigStorageRO, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW, + OnChainStorageRO, }, injected::{MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB, SignedInjectedTransaction}, malachite::{Operation, Operations}, }; use ethexe_db::Database; -use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; +use ethexe_malachite_core::{ + Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES, ValidatorPublicKey, +}; use gprimitives::H256; +use gsigner::schemes::secp256k1::PublicKey; use parity_scale_codec::{DecodeAll, Encode}; -use std::{collections::VecDeque, sync::Arc}; +use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, +}; use tokio::sync::{RwLock, mpsc::UnboundedSender}; use tracing::{debug, error, trace, warn}; @@ -86,6 +93,10 @@ pub(crate) struct EthexeExternalities { pub pending_events: RwLock>, /// Channel to poll events in MalachiteService. pub event_tx: UnboundedSender>, + /// On-chain address → pub key for every validator across all eras this node + /// knows. Lets [`Externalities::validators_for_child_of`] turn a stored era + /// validator set (addresses) back into engine [`ValidatorPublicKey`]s. + pub validators: HashMap, } /// One outbound [`MalachiteEvent`] that can't be released until its @@ -106,10 +117,17 @@ impl Externalities for EthexeExternalities { let parent = mb.parent_hash; - let parent_advanced = parent - .is_zero() - .then(H256::zero) - .unwrap_or_else(|| self.db.mb_meta(parent).last_advanced_eb); + // Propagate `last_advanced_eb` forward — the latest + // `AdvanceTillEthereumBlock` in this MB wins; otherwise we + // inherit the parent's value (zero if pre-genesis). + let parent_advanced = if parent.is_zero() { + H256::zero() + } else { + self.db + .mb_meta(parent) + .last_advanced_eb + .ok_or_else(|| anyhow!("proposed parent MB must have last_advanced_eb set"))? + }; let last_advanced = payload .iter() .rev() @@ -129,7 +147,7 @@ impl Externalities for EthexeExternalities { }, ); self.db.mutate_mb_meta(mb_hash, |meta| { - meta.last_advanced_eb = last_advanced; + meta.last_advanced_eb = Some(last_advanced); }); self.try_emit_or_queue( @@ -174,6 +192,12 @@ impl Externalities for EthexeExternalities { } } + // Mark this MB finalized so the batch-commitment path can gate on it + // directly, and advance the canonical finalized pointer downstream + // consumers (compute, batch commitment) walk. + self.db.mutate_mb_meta(mb_hash, |meta| { + meta.finalized = true; + }); self.db .globals_mutate(|g| g.latest_finalized_mb_hash = mb_hash); @@ -182,7 +206,13 @@ impl Externalities for EthexeExternalities { mb_hash, signatures: cert.signatures, }; - let last_advanced = self.db.mb_meta(mb_hash).last_advanced_eb; + // Same prerequisite as the matching BlockProposal — by the + // time `process_mb_finalized` runs, `process_mb_proposal` has + // already populated `mb_meta(block_hash).last_advanced_eb`. + let last_advanced = + self.db.mb_meta(mb_hash).last_advanced_eb.ok_or_else(|| { + anyhow!("finalized MB must have last_advanced_eb set by proposal") + })?; self.try_emit_or_queue( MalachiteEvent::BlockFinalized { cert: app_cert, @@ -199,14 +229,22 @@ impl Externalities for EthexeExternalities { async fn build_block_above(&self, parent_mb_hash: H256) -> Result { ensure!( self.mempool.is_some(), - "build_block_above must not be called when node is not validator" + "build_block_above must not be called when node is not a validator" ); - let parent_advanced = parent_mb_hash - .is_zero() - .then(H256::zero) - .unwrap_or_else(|| self.db.mb_meta(parent_mb_hash).last_advanced_eb); - let (advance, injected) = self.wait_for_proposable_content(parent_advanced).await?; + let parent_advanced = if parent_mb_hash.is_zero() { + // The parent of the genesis MB is the zero hash, advanced is zero too. + H256::zero() + } else { + self.db + .mb_meta(parent_mb_hash) + .last_advanced_eb + .with_context(|| { + format!("parent MB {parent_mb_hash:?} must have last_advanced_eb in mb meta") + })? + }; + + let (mut advance, injected) = self.wait_for_proposable_content(parent_advanced).await?; debug!( %parent_mb_hash, @@ -216,6 +254,72 @@ impl Externalities for EthexeExternalities { "build_block_above: proposable content resolved", ); + // In case advance changes era, only advancing can be only till first block of the `era + 1`. + if let Some(advance) = advance.as_mut() { + let timelines = self.db.config().timelines; + + let parent_advanced_era = if parent_advanced.is_zero() { + // The parent of the genesis MB is the zero hash, which is in era 0. + 0 + } else { + let parent_advanced_timestamp = self + .db + .block_header(parent_advanced) + .with_context(|| { + format!("parent advanced EB {parent_advanced} header not found in DB") + })? + .timestamp; + timelines + .era_from_ts(parent_advanced_timestamp) + .with_context(|| { + format!("parent advanced EB {parent_advanced} is beyond genesis") + })? + }; + + let advance_eb = self + .db + .block_simple_data(*advance) + .with_context(|| format!("advance EB {advance} header not found in DB"))?; + + let new_advanced_era = timelines + .era_from_ts(advance_eb.header.timestamp) + .with_context(|| format!("advance EB {advance} is beyond genesis"))?; + + if new_advanced_era > parent_advanced_era { + // Bypass blocks till the first block of `era + 1` + // wait_for_proposable_content - already checked + // that `advance_eb` is strict descendant of `parent_advanced`, so we can safely walk back. + let mut cursor = advance_eb; + loop { + let parent_timestamp = self + .db + .block_header(cursor.header.parent_hash) + .with_context(|| format!("{cursor} parent header not found in DB"))? + .timestamp; + + let parent_era = timelines + .era_from_ts(parent_timestamp) + .with_context(|| format!("{cursor} parent is beyond genesis"))?; + + if parent_era == parent_advanced_era { + break; + } + + ensure!( + parent_era > parent_advanced_era, + "reached previous era while searching for first block of next era" + ); + + cursor = self + .db + .block_simple_data(cursor.header.parent_hash) + .with_context(|| format!("{cursor} parent header not found in DB"))?; + } + + *advance = cursor.hash; + } + } + // Filter the fetched injected txs down to the valid ones before we start MB assembly let valid_injected_txs = { let chain_head = *self.chain_head.latest_synced.read().await; @@ -317,8 +421,7 @@ impl Externalities for EthexeExternalities { } }; - // Reject operations not allowed at this protocol version (e.g. the - // deprecated `ProcessQueues` v1 with the old mailbox validity). + // Reject operations not allowed at this protocol version for op in payload.iter() { match op { Operation::AdvanceTillEthereumBlock { .. } @@ -406,10 +509,15 @@ impl Externalities for EthexeExternalities { ))); } - let parent_advanced = parent_hash - .is_zero() - .then(H256::zero) - .unwrap_or_else(|| self.db.mb_meta(parent_hash).last_advanced_eb); + let parent_advanced = if parent_hash.is_zero() { + H256::zero() + } else { + self.db + .mb_meta(parent_hash) + .last_advanced_eb + .ok_or_else(|| anyhow!("proposed parent MB must have last_advanced_eb set"))? + }; + let start_block_hash = self.db.globals().start_block_hash; match quarantine::is_strict_descendant_of( &self.db, @@ -425,6 +533,57 @@ impl Externalities for EthexeExternalities { } Err(e) => return Err(e), } + + let timelines = self.db.config().timelines; + + let previous_era = if parent_advanced.is_zero() { + // The parent of the genesis MB is the zero hash, which resolves to era 0. + 0 + } else { + let timestamp = self + .db + .block_header(parent_advanced) + .with_context(|| { + format!("missing eth header for last_advanced_eb {parent_advanced}") + })? + .timestamp; + timelines + .era_from_ts(timestamp) + .with_context(|| format!("eb {parent_advanced} timestamp before genesis"))? + }; + let advanced_to_era = timelines + .era_from_ts(advance.header.timestamp) + .with_context(|| format!("eb {advance} timestamp before genesis"))?; + + let diff = advanced_to_era + .checked_sub(previous_era) + .context("advanced timestamp is earlier than parent advanced timestamp")?; + + if diff > 1 { + return Ok(Acceptance::Rejected(format!( + "advance EB {advance} jumps eras too far: parent era {previous_era}, advanced to era {advanced_to_era}" + ))); + } + + // If era advanced, ensure that the advance is the first block of the new era. + if diff == 1 { + let advance_parent_eb_timestamp = self + .db + .block_header(advance.header.parent_hash) + .with_context(|| format!("missing eth header for advance EB {advance} parent"))? + .timestamp; + let era_of_advanced_block_parent = timelines + .era_from_ts(advance_parent_eb_timestamp) + .with_context(|| { + format!("advance EB {advance} parent timestamp is before genesis") + })?; + + if era_of_advanced_block_parent != previous_era { + return Ok(Acceptance::Rejected(format!( + "advance {advance} is advancing to the next era, but advance is not the first block of that era" + ))); + } + } } // Validate injected txs @@ -444,10 +603,19 @@ impl Externalities for EthexeExternalities { } } - let parent_advanced = parent_hash - .is_zero() - .then(H256::zero) - .unwrap_or_else(|| self.db.mb_meta(parent_hash).last_advanced_eb); + // (4) Touched-programs cap. Only enforced on the validator side — the + // proposer in `build_block_above` already shapes the MB to stay within + // the cap; this is the participant's guard against a malicious proposer. + // `limit = max(initial_touched.len(), MAX_*)`: the proposer can't avoid + // programs already touched by EB events, so those set the floor. + let parent_advanced = if parent_hash.is_zero() { + H256::zero() + } else { + self.db + .mb_meta(parent_hash) + .last_advanced_eb + .ok_or_else(|| anyhow!("proposed parent MB must have last_advanced_eb set"))? + }; let mut touched = match advance { Some(advanced_eb) => eb_touched_programs(&self.db, parent_advanced, advanced_eb)?, None => Default::default(), @@ -467,6 +635,73 @@ impl Externalities for EthexeExternalities { Ok(Acceptance::Accepted(())) } + + async fn validators_for_child_of( + &self, + parent_mb_hash: H256, + ) -> Result> { + let parent_era = if parent_mb_hash.is_zero() { + // The parent of the genesis MB is the zero hash, which resolves to era 0. + 0 + } else { + let advanced_eb_hash = self + .db + .mb_meta(parent_mb_hash) + .last_advanced_eb + .with_context(|| format!("parent MB {parent_mb_hash} has no last_advanced_eb"))?; + + if advanced_eb_hash.is_zero() { + // The advanced EB is parent of genesis EB - resolve to era 0. + 0 + } else { + // Wait for the advanced EB to be synced in local DB + + let mut counter = 0; + loop { + let notified = self.chain_head.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.db.block_synced(advanced_eb_hash) { + break; + } + notified.await; + + counter += 1; + if counter % 100 == 0 { + warn!( + advanced_eb_hash = %advanced_eb_hash, + counter, + "waiting for advanced EB to be synced in local DB, counter={counter} synced EB notifications received...", + ); + } + } + + let timelines = self.db.config().timelines; + self.db + .block_header(advanced_eb_hash) + .with_context(|| format!("missing header for advanced eb {advanced_eb_hash}")) + .map(|header| timelines.era_from_ts(header.timestamp))? + .with_context(|| { + format!("advanced eb {advanced_eb_hash} timestamp before genesis") + })? + } + }; + + let validator_addresses = self + .db + .validators(parent_era) + .with_context(|| format!("no validators stored for era {parent_era}"))?; + + validator_addresses + .iter() + .map(|addr| { + self.validators + .get(addr) + .copied() + .with_context(|| format!("validator pool missing pub key for {addr}")) + }) + .collect() + } } impl EthexeExternalities { @@ -640,6 +875,7 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, + validators: Default::default(), pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, @@ -839,15 +1075,15 @@ mod tests { } while rx.try_recv().is_ok() {} - assert!(db.mb_meta(chain[0]).last_advanced_eb.is_zero()); + assert_eq!(db.mb_meta(chain[0]).last_advanced_eb, Some(H256::zero())); assert_eq!( db.mb_meta(chain[1]).last_advanced_eb, - H256::repeat_byte(0xAB), + Some(H256::repeat_byte(0xAB)), "h2 should anchor to its own AdvanceTillEthereumBlock" ); assert_eq!( db.mb_meta(chain[2]).last_advanced_eb, - H256::repeat_byte(0xAB), + Some(H256::repeat_byte(0xAB)), "h3 inherits h2's anchor" ); } @@ -1055,6 +1291,7 @@ mod tests { mempool: Some(Arc::clone(&tracker) as Arc), chain_head: make_chain_head(), event_tx, + validators: Default::default(), pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, @@ -1122,6 +1359,7 @@ mod tests { mempool: Some(mempool as Arc), chain_head: make_chain_head(), event_tx, + validators: Default::default(), pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, @@ -1195,7 +1433,10 @@ mod tests { ); } db.set_mb_program_states(mb_hash, program_states); - db.mutate_mb_meta(mb_hash, |meta| meta.computed = true); + db.mutate_mb_meta(mb_hash, |meta| { + meta.computed = true; + meta.last_advanced_eb = Some(H256::zero()); + }); mb_hash } @@ -1740,7 +1981,7 @@ mod tests { db.set_mb_program_states(parent_mb, ethexe_common::ProgramStates::default()); db.mutate_mb_meta(parent_mb, |meta| { meta.computed = true; - meta.last_advanced_eb = chain[3].0; + meta.last_advanced_eb = Some(chain[3].0); }); let (ext, _rx) = make_externalities(db.clone()); @@ -1876,6 +2117,7 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, + validators: Default::default(), pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 763968da150..2aa16d5e526 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -21,7 +21,7 @@ //! - [`MalachiteEvent`] (enum) — Output event: proposal, finalization, purged txs //! - [`CommitCertificate`] (struct) — BFT commit proof attached to `BlockFinalized` //! - [`MalachiteServiceConfig`] (struct) — Service configuration -//! - [`ValidatorEntry`] (struct) — Single entry in the validator set +//! - [`ValidatorPublicKey`] (type) — Single entry in the unweighted validator set //! - [`Mempool`] (trait) — Producer-side injected-tx source //! - [`InjectedTxMempool`] (struct) — Real mempool implementation //! - [`TxValidityChecker`] (struct) — Per-tx validity against the MB world @@ -60,7 +60,7 @@ mod tx_validity; mod types; pub use crate::{ - config::{MalachiteServiceConfig, ValidatorConfig, ValidatorEntry}, + config::{MalachiteServiceConfig, ValidatorConfig, ValidatorPublicKey}, mempool::{InjectedTxMempool, Mempool, TxInsertionStatus}, service::MalachiteService, starter::MalachiteServiceStarter, diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 1a3f41d379d..2c10a980004 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -7,23 +7,17 @@ //! engine and exposes its outputs as a `Stream` of [`MalachiteEvent`]s. use crate::{ - Mempool, ValidatorEntry, + Mempool, externalities::EthexeExternalities, mempool::TxInsertionStatus, types::{ChainHead, MalachiteEvent}, }; use anyhow::Result; -use ethexe_common::{ - Address, SimpleBlockData, - db::{ConfigStorageRO, OnChainStorageRO}, - injected::SignedInjectedTransaction, -}; +use ethexe_common::{SimpleBlockData, db::OnChainStorageRO, injected::SignedInjectedTransaction}; use ethexe_malachite_core::MalachiteCore; use futures::{Stream, stream::FusedStream}; use gprimitives::H256; -use gsigner::schemes::secp256k1::PublicKey; use std::{ - collections::HashMap, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -40,10 +34,6 @@ pub struct MalachiteService { pub(crate) mempool: Option>, /// Externalities shared with the inner consensus core. pub(crate) externalities: Arc, - /// Known validator public keys by on-chain address, for era rotation. - pub(crate) validators: HashMap, - /// Era whose validator set is currently active in the engine. - pub(crate) active_era: u64, /// Inner consensus core; `None` after shutdown. pub(crate) inner: Option>, } @@ -79,7 +69,7 @@ impl MalachiteService { } /// Handle a fully synced Ethereum block: publish it for the producer's - /// quarantine checks, rotate the validator set on era change and GC the mempool. + /// quarantine checks, wake the producer and GC the mempool. pub async fn receive_eb_synced(&mut self, eb_hash: H256) { let Some(synced) = self.externalities.db.block_simple_data(eb_hash) else { tracing::error!(synced = %eb_hash, "synced EB header not found in local DB, ignoring"); @@ -102,17 +92,14 @@ impl MalachiteService { drop(latest_synced); // Still wake the producer: a lower-height sync may have just // landed parent headers a failed descendant walk needs. - self.chain_head.notify.notify_one(); + self.chain_head.notify.notify_waiters(); return; } *latest_synced = synced; } // Notify inner proposer if it waits (see EthexeExternalities::wait_for_proposable_content) - self.chain_head.notify.notify_one(); - - // Rotate before waking the producer so the next round sees the new set. - self.maybe_rotate_validators_for_era(synced); + self.chain_head.notify.notify_waiters(); if let Some(pool) = self.mempool.as_ref() { let purged_txs = pool.set_chain_head(synced).await; @@ -143,70 +130,6 @@ impl MalachiteService { inner.shutdown().await; } } - - /// Push the on-chain validators for `head`'s era into the engine, - /// if the era moved. Skips on missing DB data or unknown pub keys - /// (wait-and-retry: the next `BlockSynced` re-evaluates). - fn maybe_rotate_validators_for_era(&mut self, head: SimpleBlockData) { - let db = &self.externalities.db; - let timelines = db.config().timelines; - let Some(era) = timelines.era_from_ts(head.header.timestamp) else { - return; - }; - if self.active_era == era { - return; - } - let Some(addrs) = db.validators(era) else { - // trace like error because `head` must be synced - tracing::error!(era, "validators for era not yet in DB; deferring rotation"); - return; - }; - - let mut new_set = Vec::with_capacity(self.validators.len()); - let mut missing: Vec
= Vec::new(); - for addr in addrs.iter() { - match self.validators.get(addr) { - Some(pk) => new_set.push(ValidatorEntry { - public_key: *pk, - voting_power: 1, - }), - None => missing.push(*addr), - } - } - - if !missing.is_empty() { - tracing::warn!( - era, - missing = ?missing, - "validator pool missing pub keys for some on-chain era validators; \ - keeping the previous active set", - ); - return; - } - - // Bug-class failure — advance active_era so we don't loop on the same broken input. - let inner = match self.inner.as_ref() { - Some(inner) => inner, - None => { - tracing::error!(era, "rotate after shutdown"); - self.active_era = era; - return; - } - }; - - if let Err(e) = inner.update_validators(new_set) { - tracing::error!(era, error = %e, "rotating malachite validator set failed"); - self.active_era = era; - return; - } - - self.active_era = era; - - tracing::info!( - era, - "rotated malachite validator set to era's on-chain quorum" - ); - } } impl Stream for MalachiteService { diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 017fe154cb4..1a5e473b7d1 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -8,10 +8,7 @@ use crate::{ types::{ChainHead, MalachiteEvent}, }; use anyhow::{Context as _, Result, anyhow}; -use ethexe_common::{ - Address, SimpleBlockData, - db::{ConfigStorageRO, GlobalsStorageRO}, -}; +use ethexe_common::{Address, SimpleBlockData, db::GlobalsStorageRO}; use ethexe_db::Database; use ethexe_malachite_core::{MalachiteCore, MalachiteCoreConfig, NodeRole}; use gsigner::schemes::secp256k1::{PrivateKey, PublicKey}; @@ -28,8 +25,6 @@ pub struct MalachiteServiceStarter { chain_head: Arc, mempool: Option>, externalities: Arc, - validators: HashMap, - active_era: u64, core_config: MalachiteCoreConfig, } @@ -50,12 +45,6 @@ impl MalachiteServiceStarter { return Err(anyhow!("MalachiteServiceConfig::validators is empty")); } - let active_era = db - .config() - .timelines - .era_from_ts(initial_chain_head.header.timestamp) - .context("initial chain head must be after genesis")?; - // Validators sign votes/proposals using their on-chain key; // full nodes get an ephemeral secret used only as the libp2p // peer identity. @@ -96,6 +85,15 @@ impl MalachiteServiceStarter { let (event_tx, events_rx) = mpsc::unbounded_channel(); + // On-chain address → pub key, so the era resolver + // ([`EthexeExternalities::validators_for_child_of`]) maps a stored era's + // addresses back to engine keys. + let validators: HashMap = config + .validators + .iter() + .map(|pk| (pk.to_address(), *pk)) + .collect(); + let externalities = Arc::new(EthexeExternalities { db, cfg: ExternalitiesConfig { @@ -107,22 +105,14 @@ impl MalachiteServiceStarter { chain_head: chain_head.clone(), pending_events: Default::default(), event_tx, + validators, }); - // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. - let validators = config - .validators - .iter() - .map(|v| (v.public_key.to_address(), v.public_key)) - .collect(); - Ok(Self { events_rx, chain_head, mempool, externalities, - validators, - active_era, core_config, }) } @@ -134,8 +124,6 @@ impl MalachiteServiceStarter { chain_head, mempool, externalities, - validators, - active_era, core_config, } = self; @@ -148,8 +136,6 @@ impl MalachiteServiceStarter { chain_head, mempool, externalities, - validators, - active_era, inner: Some(inner), }) } diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 98c72f0d7b2..8e2f1d8973d 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -20,14 +20,14 @@ use std::{path::Path, time::Duration}; use async_trait::async_trait; use ethexe_common::{ - BlockHeader, SimpleBlockData, + BlockHeader, SimpleBlockData, ValidatorsVec, db::{BlockMetaStorageRW, CompactMb, GlobalsStorageRO, MbStorageRO, OnChainStorageRW}, injected::{PurgedTransaction, SignedInjectedTransaction}, }; use ethexe_db::Database; use ethexe_malachite::{ MalachiteEvent, MalachiteService, MalachiteServiceConfig, MalachiteServiceStarter, Mempool, - TxInsertionStatus, ValidatorConfig, ValidatorEntry, + TxInsertionStatus, ValidatorConfig, }; use futures::StreamExt as _; use gprimitives::H256; @@ -100,6 +100,11 @@ fn seed_chain(db: &Database, len: usize, seed: u32) -> Vec { db.set_block_header(hash, header); db.set_block_events(hash, &[]); db.mutate_block_meta(hash, |m| m.prepared = true); + // Mark synced too: `validators_for_child_of` now waits for the + // `last_advanced_eb` block to be synced before resolving its era. + // In production the observer sets this flag; this test has no observer, + // and it seeds the whole chain as already-synced. + db.set_block_synced(hash); chain.push(SimpleBlockData { hash, header }); parent = hash; } @@ -134,10 +139,7 @@ fn build_config( ), home_dir: home.to_path_buf(), persistent_peers: Vec::new(), - validators: vec![ValidatorEntry { - public_key: pub_key, - voting_power: 1, - }], + validators: vec![pub_key], propose_timeout: Duration::from_secs(5), } } @@ -222,6 +224,19 @@ async fn single_validator_finalizes_and_recovers_after_restart() { let (signer, pub_key) = build_signer(home.path()); + // Seed on-chain validators for every era the chain spans: the era-aware + // resolver (`EthexeExternalities::validators_for_child_of`) looks these up to + // map the governing era's addresses back to engine keys. The memory DB uses + // a 1-second era with genesis at 0, so block timestamp `i` lands in era `i`; + // the 64-block chain therefore touches eras 0..64. Production stores this via + // the observer; tests must seed it. + let validators: ValidatorsVec = vec![pub_key.to_address()] + .try_into() + .expect("non-empty validator set"); + for era in 0..chain.len() as u64 { + db.set_validators(era, validators.clone()); + } + // ---- first run ------------------------------------------------- let mut svc = MalachiteServiceStarter::new( build_config(home.path(), 30_001, pub_key), diff --git a/ethexe/observer/src/sync.rs b/ethexe/observer/src/sync.rs index 67e136d415b..cf74b5975be 100644 --- a/ethexe/observer/src/sync.rs +++ b/ethexe/observer/src/sync.rs @@ -24,7 +24,7 @@ use ethexe_ethereum::{ router::RouterQuery, }; use gprimitives::H256; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque, hash_map::Entry}; /// Outcome of one chain-sync attempt. `RpcError` is recoverable (caller /// retries on the next chain head); `Fatal` propagates. @@ -97,8 +97,8 @@ impl ChainSync { let blocks_data = self.pre_load_data(&block.header).await?; let chain = self.load_chain(&block, blocks_data).await?; - self.ensure_validators(block).await?; - self.mark_chain_as_synced(chain.into_iter().rev()); + self.ensure_validators(&chain).await?; + self.mark_chain_as_synced(chain); Ok(block.hash) } @@ -106,13 +106,13 @@ impl ChainSync { async fn load_chain( &self, block: &SimpleBlockData, - mut blocks_data: HashMap, - ) -> Result> { - let mut chain = Vec::new(); + mut pre_loaded_blocks_data: HashMap, + ) -> Result> { + let mut chain = VecDeque::new(); let mut current_block_hash = block.hash; while !self.db.block_synced(current_block_hash) { - let block_data = match blocks_data.remove(¤t_block_hash) { + let block_data = match pre_loaded_blocks_data.remove(¤t_block_hash) { Some(data) => data, None => { self.block_loader @@ -150,7 +150,8 @@ impl ChainSync { self.db .set_block_events(current_block_hash, &block_data.events); - chain.push(SimpleBlockData { + // Push front so the chain is in order from oldest to newest + chain.push_front(SimpleBlockData { hash: current_block_hash, header: block_data.header, }); @@ -195,46 +196,79 @@ impl ChainSync { } /// This function guarantees the next things: - /// 1. if there is no validators for current era in database - it fetches them. + /// 1. if there is no validators for `block_chain` eras - it fetches them. /// 2. if the election result is `finalized` it requests for next era validators and sets them in database. /// /// See [`Self::election_timestamp_finalized`] for the our timestamp `finalization` rules. - async fn ensure_validators(&self, block_data: SimpleBlockData) -> Result<()> { - let chain_head_era = self - .config - .timelines - .era_from_ts(block_data.header.timestamp) - .context("failed to calculate era from timestamp")?; + async fn ensure_validators(&self, chain: &VecDeque) -> Result<()> { + if cfg!(debug_assertions) { + // Check timestamps are in ascending order + let mut timestamp = 0; + for block in chain { + let block_timestamp = block.header.timestamp; + if block_timestamp < timestamp { + return Err(anyhow!( + "Block timestamps are not in ascending order: {block_timestamp} < {timestamp}" + )); + } + timestamp = block_timestamp; + } + } + + let Some(chain_head) = chain.back().copied() else { + return Ok(()); + }; - // If we don't have validators for current era - set them. - if self.db.validators(chain_head_era).is_none() { - let validators = self.router_query.validators_at(block_data.hash).await?; - self.db.set_validators(chain_head_era, validators); + let timelines = &self.config.timelines; + let mut era_validators_map = HashMap::new(); + for block in chain.iter().rev() { + let era = timelines + .era_from_ts(block.header.timestamp) + .context("block timestamp is before genesis")?; + if self.db.validators(era).is_some() { + // We already have validators for this era, and that means we have validators + // for all previous eras too, so we can stop here. + break; + } + if let Entry::Vacant(entry) = era_validators_map.entry(era) { + entry.insert(self.router_query.validators_at(block.hash).await?); + } + } + + for (era, validators) in era_validators_map { + self.db.set_validators(era, validators); } + let next_era = self + .config + .timelines + .era_from_ts(chain_head.header.timestamp) + .context("failed to calculate era from timestamp")? + .checked_add(1) + .context("u64 era index overflow")?; + // Fetch next era validators if timestamp `finalized` and we don't set them in database already. - if let Some(election_ts) = self.election_timestamp_finalized(block_data.header.timestamp) - && self.db.validators(chain_head_era + 1).is_none() + if let Some(election_ts) = self.election_timestamp_finalized(chain_head.header.timestamp) + && self.db.validators(next_era).is_none() { let next_era_validators = self .middleware_query .make_election_at(election_ts, 10) .await?; - self.db - .set_validators(chain_head_era + 1, next_era_validators); + self.db.set_validators(next_era, next_era_validators); } Ok(()) } - fn mark_chain_as_synced(&self, chain: impl Iterator) { + fn mark_chain_as_synced(&self, chain: impl IntoIterator) { for data in chain { let SimpleBlockData { hash, header } = data; self.db.set_block_synced(hash); log::trace!( - "✅ block {hash} synced, events: {:?}", + "⛓️ block {hash} synced, events: {:?}", self.db.block_events(hash) ); diff --git a/ethexe/prometheus/src/lib.rs b/ethexe/prometheus/src/lib.rs index 5dc8892a8db..6deb20a883f 100644 --- a/ethexe/prometheus/src/lib.rs +++ b/ethexe/prometheus/src/lib.rs @@ -267,7 +267,7 @@ fn update_liveness_metrics(db: Database, metrics: LivenessMetrics) { let Some(latest_committed_block_header) = db .block_meta(db.globals().latest_prepared_eb_hash) .last_committed_mb - .map(|mb_hash| db.mb_meta(mb_hash).last_advanced_eb) + .and_then(|mb_hash| db.mb_meta(mb_hash).last_advanced_eb) .and_then(|eth_block| db.block_header(eth_block)) else { return; diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 2913a6ac636..3b7ad278399 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -58,7 +58,7 @@ use ethexe_db::{ use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; use ethexe_malachite::{ InjectedTxMempool, MalachiteEvent, MalachiteServiceConfig, MalachiteServiceStarter, - ValidatorEntry, + ValidatorPublicKey, }; use ethexe_network::{ NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType, @@ -134,26 +134,22 @@ impl ExternalDataProvider for RouterDataProvider { /// `address -> public key` table loaded from the /// `--validators-malachite-pub-keys` JSON file. /// -/// Voting power is fixed at 1 — Malachite quorum is `> 2/3` of the -/// total, which under uniform weights matches the Router's -/// signature threshold. If/when the Router exposes per-validator -/// stake, the lookup here is the natural place to plumb it through. +/// The set is unweighted — Malachite quorum is `> 2/3` of the +/// validator count, which matches the Router's signature threshold. +/// If/when the Router exposes per-validator stake, the lookup here is +/// the natural place to plumb it through. fn build_malachite_validator_set( on_chain_validators: impl IntoIterator, pub_keys: &BTreeMap, -) -> Result> { +) -> Result> { on_chain_validators .into_iter() .map(|addr| { - let pub_key = pub_keys.get(&addr).copied().with_context(|| { + pub_keys.get(&addr).copied().with_context(|| { format!( "validator address {addr} has no entry in --validators-malachite-pub-keys; \ every on-chain validator must be present in the table" ) - })?; - Ok(ValidatorEntry { - public_key: pub_key, - voting_power: 1, }) }) .collect() diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 83234c94b11..486fb276e0c 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -2008,6 +2008,9 @@ async fn validators_election() { let env_config = TestEnvConfig { validators: ValidatorsConfig::ProvidedValidators(current_validators), + // Reserve malachite endpoints for the next-era set too, so all nodes + // boot at once and the handover needs no restart. + future_validators: next_validators.clone(), deploy_params, network: EnvNetworkConfig::Enabled, signer: signer.clone(), @@ -2031,9 +2034,20 @@ async fn validators_election() { .header .timestamp; - // Start initial validators + // Start the full validator universe up front: the initial on-chain set + // (current era) plus the validators elected for the next era. All run + // continuously; the per-era resolver rotates who votes, so the handover + // happens in place with no restart. + let current_validators_configs = env.validators.clone(); + let next_validators_configs = TestEnv::define_session_keys(next_validators.clone()); + let mut validators = vec![]; - for (i, v) in env.validators.clone().into_iter().enumerate() { + for (i, v) in current_validators_configs + .iter() + .chain(next_validators_configs.iter()) + .cloned() + .enumerate() + { test_info!("📗 Starting validator-{i}"); let mut validator = env .new_node(NodeConfig::named(format!("validator-{i}")).validator(v)) @@ -2042,13 +2056,8 @@ async fn validators_election() { validators.push(validator); } - // Setup next validators to be elected for previous era - let next_validators_configs = TestEnv::define_session_keys(next_validators); - - let next_validator_addrs: Vec<_> = next_validators_configs - .iter() - .map(|cfg| cfg.public_key.to_address()) - .collect(); + // Elect the next validator set during the current era's election window. + let next_validator_addrs: Vec<_> = next_validators.iter().map(|pk| pk.to_address()).collect(); env.election_provider .set_predefined_election_at( @@ -2095,20 +2104,7 @@ async fn validators_election() { .unwrap(); assert_eq!(ping_actor.code_id, uploaded_code.code_id); - stop_nodes(validators).await; - - env.extend_malachite_endpoints(&next_validators_configs); - env.validators = next_validators_configs; - let mut new_validators = vec![]; - for (i, v) in env.validators.clone().into_iter().enumerate() { - test_info!("📗 Starting next validator-{i}"); - let mut validator = env - .new_node(NodeConfig::named(format!("validator-{i}")).validator(v)) - .await; - validator.start_service().await; - new_validators.push(validator); - } - + // Cross into the next era; the elected set takes over consensus in place. env.provider .anvil_set_next_block_timestamp(era_duration + genesis_ts) .await @@ -2126,7 +2122,7 @@ async fn validators_election() { assert_eq!(reply.payload, b"PONG"); assert_eq!(reply.program_id, ping_actor.program_id); - stop_nodes(new_validators).await; + stop_nodes(validators).await; } /// Validators must NOT fold an Ethereum event into MB execution before the diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 1d48a40f213..2077999e1ef 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -39,7 +39,7 @@ use ethexe_ethereum::{ }; use ethexe_malachite::{ InjectedTxMempool, MalachiteServiceConfig, MalachiteServiceStarter, - Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, + Multiaddr as MalachiteMultiaddr, PeerId, ValidatorPublicKey, derive_libp2p_secret, malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; @@ -115,7 +115,8 @@ pub struct TestEnv { pub kicking_per_blocks: Option, #[allow(unused)] pub db: Database, - /// Endpoints aligned 1:1 with `validators`. + /// Endpoints for the full malachite validator universe: the initial + /// on-chain `validators` plus any `future_validators` reserved up front. pub malachite_endpoints: Vec, /// Pre-bound TCP listeners holding each validator's port until handed off in `new_node`. malachite_listeners: HashMap, @@ -171,6 +172,7 @@ impl TestEnv { pub async fn new(config: TestEnvConfig) -> anyhow::Result { let TestEnvConfig { validators, + future_validators, block_time, rpc, wallets, @@ -418,9 +420,17 @@ impl TestEnv { (handle, bootstrap_address, nonce) }); - // Hold listeners alive until `start_service` to keep concurrent test processes off our ports. + // Reserve endpoints for the full malachite validator universe (initial + // on-chain set + any `future_validators` elected in a later era), so all + // nodes can boot at once and rotate without a restart. Hold the listeners + // alive until `start_service` to keep concurrent test processes off our ports. + let all_validator_configs: Vec = validator_configs + .iter() + .cloned() + .chain(Self::define_session_keys(future_validators)) + .collect(); let (malachite_endpoints, malachite_listeners) = - build_malachite_endpoints(&signer, &validator_configs); + build_malachite_endpoints(&signer, &all_validator_configs); Ok(TestEnv { eth_cfg, @@ -500,10 +510,6 @@ impl TestEnv { .as_ref() .and_then(|c| self.malachite_listeners.remove(&c.public_key)); - // Snapshot env.validators now so a node spawned post-rotation boots with the new set. - let active_validator_pub_keys: Vec = - self.validators.iter().map(|v| v.public_key).collect(); - Node { name, db, @@ -525,7 +531,6 @@ impl TestEnv { commitment_delay_limit: self.commitment_delay_limit, batch_commitment_period: self.batch_commitment_period, malachite_endpoints: self.malachite_endpoints.clone(), - active_validator_pub_keys, malachite_home, malachite_listener, running_service_handle: None, @@ -538,26 +543,6 @@ impl TestEnv { } } - /// Pre-allocate malachite endpoints for an *additional* validator set - /// (e.g. the "next" set in an era handover test) and merge them into - /// `self.malachite_endpoints` / `self.malachite_listeners`. Without this, - /// `start_service` panics when asked to boot a validator whose pubkey - /// wasn't part of `TestEnv::new` time. - pub fn extend_malachite_endpoints(&mut self, validators: &[ValidatorConfig]) { - let (extra_endpoints, extra_listeners) = - build_malachite_endpoints(&self.signer, validators); - for ep in extra_endpoints { - if !self - .malachite_endpoints - .iter() - .any(|e| e.pub_key == ep.pub_key) - { - self.malachite_endpoints.push(ep); - } - } - self.malachite_listeners.extend(extra_listeners); - } - pub async fn new_initialized_db(&self) -> Database { ethexe_db::create_initialized_empty_memory_db(InitConfig { ethereum_rpc: self.eth_cfg.rpc.clone(), @@ -850,6 +835,12 @@ pub struct TestEnvConfig { /// How many validators will be in deployed router. /// By default uses 1 auto generated validator. pub validators: ValidatorsConfig, + /// Validators that are NOT in the initial on-chain set but whose malachite + /// endpoints (TCP port + peer-id) must be reserved up front, so they can be + /// booted immediately and join consensus on a later era handover without a + /// restart. Their pub keys are added to every node's malachite validator + /// pool; the per-era resolver still governs who actually votes. Empty by default. + pub future_validators: Vec, /// By default uses 1 second block time. pub block_time: Duration, /// By default creates new anvil instance if rpc is not provided. @@ -886,6 +877,7 @@ impl Default for TestEnvConfig { fn default() -> Self { Self { validators: ValidatorsConfig::PreDefined(1), + future_validators: Vec::new(), block_time: Duration::from_secs(1), rpc: EnvRpcConfig::CustomAnvil { // speeds up block finalization, so we don't have to calculate @@ -1040,11 +1032,10 @@ pub struct Node { /// Malachite WAL + store.db tempdir; lives with the node. malachite_home: Option, - /// Endpoints of every validator (this node + peers). + /// Endpoints of the full malachite validator universe (this node + peers, + /// including any future-era validators). The per-era resolver decides who + /// actually votes, so the whole set is passed as the validator pool. malachite_endpoints: Vec, - /// Snapshot of `env.validators` at `new_node` time — drives the - /// boot-time filter on `malachite_endpoints` in `start_service`. - active_validator_pub_keys: Vec, /// Port reservation; dropped just before the first malachite service start. malachite_listener: Option, @@ -1138,31 +1129,21 @@ impl Node { // receive `BlockFinalized` and trigger local compute so promise // bodies reach the RPC subscription manager. let malachite = { - // Filter `malachite_endpoints` to era-current pubkeys — - // leftover entries from `extend_malachite_endpoints` would - // skew the >2/3 threshold. - let active: Vec<&MalachiteEndpoint> = self - .malachite_endpoints - .iter() - .filter(|e| self.active_validator_pub_keys.contains(&e.pub_key)) - .collect(); + // The whole reserved universe is the validator pool; the per-era + // resolver inside the engine governs who actually votes at each + // height, so era handovers happen in-place with no restart. + let endpoints = &self.malachite_endpoints; let (listen_addr, persistent_peers) = match self.validator_config.as_ref() { Some(config) => { - let me = self - .malachite_endpoints + let me = endpoints .iter() .find(|e| e.pub_key == config.public_key) .cloned() .expect( "validator's malachite endpoint missing — env not aware of this key", ); - assert!( - active.iter().any(|e| e.pub_key == config.public_key), - "test setup bug: local validator {} not in env.validators when start_service was called", - config.public_key, - ); - let peers: Vec = active + let peers: Vec = endpoints .iter() .filter(|e| e.pub_key != config.public_key) .map(|e| e.multiaddr()) @@ -1170,26 +1151,20 @@ impl Node { (me.listen_addr, peers) } None => { - // Full node: bind a fresh port, dial every active - // validator. No reserved listener since `new_node` - // never allocated one for this key. + // Full node: bind a fresh port, dial every validator. + // No reserved listener since `new_node` never allocated + // one for this key. let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) .expect("bind 127.0.0.1:0 for full-node malachite endpoint"); let addr = listener.local_addr().expect("local_addr"); let peers: Vec = - active.iter().map(|e| e.multiaddr()).collect(); + endpoints.iter().map(|e| e.multiaddr()).collect(); drop(listener); (addr, peers) } }; - let validators: Vec = active - .iter() - .map(|e| ValidatorEntry { - public_key: e.pub_key, - voting_power: 1, - }) - .collect(); + let validators: Vec = endpoints.iter().map(|e| e.pub_key).collect(); // Reuse the home dir from `new_node` so stop+start resumes from WAL. let home_path = self diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index 4c8530ecd5e..0bcf37757ea 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -354,13 +354,16 @@ impl TestingEventReceiver { else { return None; }; - let last_advanced = db.mb_meta(mb_hash).last_advanced_eb; + let last_advanced = db.mb_meta(mb_hash).last_advanced_eb?; if last_advanced.is_zero() { return None; } // Anchor: previous MB's `last_advanced_eb` (genesis if none). let prev_advanced = match db.mb_compact_block(mb_hash) { - Some(c) if !c.parent.is_zero() => db.mb_meta(c.parent).last_advanced_eb, + Some(c) if !c.parent.is_zero() => db + .mb_meta(c.parent) + .last_advanced_eb + .unwrap_or(H256::zero()), _ => H256::zero(), }; // Walk the eth chain from this MB's `last_advanced_eb` back to