From 7914128ab5a9bdba8e3e9b2807d1b211338a6056 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 28 Oct 2025 17:05:18 +0100 Subject: [PATCH 001/146] Implement Cell Dissemination --- Cargo.lock | 151 ++++---- Cargo.toml | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 75 ++-- .../src/data_availability_checker.rs | 99 ++++- .../overflow_lru_cache.rs | 116 ++++-- .../src/data_column_verification.rs | 361 ++++++++++++----- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 17 + .../beacon_chain/src/fetch_blobs/mod.rs | 65 ++-- beacon_node/beacon_chain/src/kzg_utils.rs | 155 +++++++- beacon_node/beacon_chain/src/metrics.rs | 21 + beacon_node/beacon_chain/src/test_utils.rs | 2 +- beacon_node/beacon_chain/tests/events.rs | 2 +- beacon_node/beacon_processor/src/lib.rs | 16 +- beacon_node/execution_layer/src/engine_api.rs | 1 + .../execution_layer/src/engine_api/http.rs | 16 + .../src/engine_api/json_structures.rs | 3 + beacon_node/execution_layer/src/lib.rs | 19 +- .../execution_layer/src/test_utils/mod.rs | 1 + beacon_node/http_api/src/publish_blocks.rs | 14 +- beacon_node/lighthouse_network/Cargo.toml | 7 +- .../lighthouse_network/src/service/mod.rs | 60 ++- .../lighthouse_network/src/types/mod.rs | 1 + .../lighthouse_network/src/types/partial.rs | 123 ++++++ .../lighthouse_network/src/types/pubsub.rs | 64 ++- .../lighthouse_network/src/types/topics.rs | 6 + beacon_node/lighthouse_tracing/src/lib.rs | 1 + beacon_node/network/src/lib.rs | 1 + beacon_node/network/src/metrics.rs | 40 ++ .../gossip_methods.rs | 303 ++++++++++++++- .../src/network_beacon_processor/mod.rs | 59 ++- .../network/src/partial_data_column_cache.rs | 76 ++++ beacon_node/network/src/router.rs | 15 + beacon_node/network/src/service.rs | 2 +- consensus/types/src/das_column.rs | 105 +++++ consensus/types/src/data_column_sidecar.rs | 92 ++++- consensus/types/src/lib.rs | 2 + .../types/src/partial_data_column_sidecar.rs | 364 ++++++++++++++++++ testing/ef_tests/src/cases/fork_choice.rs | 11 +- 38 files changed, 2137 insertions(+), 331 deletions(-) create mode 100644 beacon_node/lighthouse_network/src/types/partial.rs create mode 100644 beacon_node/network/src/partial_data_column_cache.rs create mode 100644 consensus/types/src/das_column.rs create mode 100644 consensus/types/src/partial_data_column_sidecar.rs diff --git a/Cargo.lock b/Cargo.lock index 1efb1fbc706..4827786a167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2420,7 +2420,7 @@ dependencies = [ "parking_lot 0.12.3", "rand 0.8.5", "smallvec", - "socket2", + "socket2 0.5.8", "tokio", "tracing", "uint 0.10.0", @@ -4156,7 +4156,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "ring", - "socket2", + "socket2 0.5.8", "thiserror 2.0.12", "tinyvec", "tokio", @@ -4396,7 +4396,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -4477,7 +4477,7 @@ dependencies = [ "http-body 1.0.1", "hyper 1.6.0", "pin-project-lite", - "socket2", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -4865,7 +4865,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.8", "widestring 1.1.0", "windows-sys 0.48.0", "winreg", @@ -5161,9 +5161,8 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce71348bf5838e46449ae240631117b487073d5f347c06d434caddcb91dceb5a" +version = "0.56.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "bytes", "either", @@ -5174,6 +5173,7 @@ dependencies = [ "libp2p-connection-limits", "libp2p-core", "libp2p-dns", + "libp2p-gossipsub", "libp2p-identify", "libp2p-identity", "libp2p-mdns", @@ -5194,8 +5194,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16ccf824ee859ca83df301e1c0205270206223fd4b1f2e512a693e1912a8f4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5205,8 +5204,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18b8b607cf3bfa2f8c57db9c7d8569a315d5cc0a282e6bfd5ebfc0a9840b2a0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5216,8 +5214,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d28e2d2def7c344170f5c6450c0dbe3dfef655610dbfde2f6ac28a527abbe36" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "fnv", @@ -5234,15 +5231,14 @@ dependencies = [ "rw-stream-sink", "thiserror 2.0.12", "tracing", - "unsigned-varint 0.8.0", + "unsigned-varint", "web-time", ] [[package]] name = "libp2p-dns" version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b770c1c8476736ca98c578cba4b505104ff8e842c2876b528925f9766379f9a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "async-trait", "futures", @@ -5257,7 +5253,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/sigp/rust-libp2p.git?rev=5acdf89a65d64098f9346efa5769e57bcd19dea9#5acdf89a65d64098f9346efa5769e57bcd19dea9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5287,8 +5283,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab792a8b68fdef443a62155b01970c81c3aadab5e659621b063ef252a8e65e8" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "either", @@ -5330,8 +5325,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66872d0f1ffcded2788683f76931be1c52e27f343edb93bc6d0bcd8887be443" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "hickory-proto", @@ -5341,16 +5335,15 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "smallvec", - "socket2", + "socket2 0.6.1", "tokio", "tracing", ] [[package]] name = "libp2p-metrics" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805a555148522cb3414493a5153451910cb1a146c53ffbf4385708349baf62b7" +version = "0.17.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "libp2p-core", @@ -5364,9 +5357,8 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aaa6fee3722e355443058472fc4705d78681bc2d8e447a0bdeb3fecf40cd197" +version = "0.43.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5378,14 +5370,13 @@ dependencies = [ "rand 0.8.5", "smallvec", "tracing", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] name = "libp2p-noise" version = "0.46.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc73eacbe6462a0eb92a6527cac6e63f02026e5407f8831bde8293f19217bfbf" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5407,8 +5398,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e659439578fc6d305da8303834beb9d62f155f40e7f5b9d81c9f2b2c69d1926" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5423,8 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc448b2de9f4745784e3751fe8bc6c473d01b8317edd5ababcb0dec803d843f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", @@ -5436,7 +5425,7 @@ dependencies = [ "rand 0.8.5", "ring", "rustls 0.23.23", - "socket2", + "socket2 0.6.1", "thiserror 2.0.12", "tokio", "tracing", @@ -5445,17 +5434,16 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa762e5215919a34e31c35d4b18bf2e18566ecab7f8a3d39535f4a3068f8b62" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "fnv", "futures", "futures-timer", + "hashlink 0.10.0", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", - "lru", "multistream-select", "rand 0.8.5", "smallvec", @@ -5467,8 +5455,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "heck 0.5.0", "quote", @@ -5478,15 +5465,14 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b4e030c52c46c8d01559b2b8ca9b7c4185f10576016853129ca1fe5cd1a644" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "socket2", + "socket2 0.6.1", "tokio", "tracing", ] @@ -5494,8 +5480,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ff65a82e35375cbc31ebb99cacbbf28cb6c4fefe26bf13756ddcf708d40080" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-rustls", @@ -5512,9 +5497,8 @@ dependencies = [ [[package]] name = "libp2p-upnp" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4757e65fe69399c1a243bbb90ec1ae5a2114b907467bf09f3575e899815bb8d3" +version = "0.6.0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", @@ -5528,8 +5512,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f15df094914eb4af272acf9adaa9e287baa269943f32ea348ba29cfb9bfc60d8" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "futures", @@ -5673,7 +5656,7 @@ dependencies = [ "tracing", "tracing-subscriber", "types", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] @@ -6174,7 +6157,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint 0.8.0", + "unsigned-varint", "url", ] @@ -6196,21 +6179,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] name = "multistream-select" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "bytes", "futures", - "log", "pin-project", "smallvec", - "unsigned-varint 0.7.2", + "tracing", + "unsigned-varint", ] [[package]] @@ -7319,9 +7301,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf41c1a7c32ed72abe5082fb19505b969095c12da9f5732a4bc9878757fd087c" +checksum = "e4500adecd7af8e0e9f4dbce15cfee07ce913fbf6ad605cc468b83f2d531ee94" dependencies = [ "dtoa", "itoa", @@ -7331,9 +7313,9 @@ dependencies = [ [[package]] name = "prometheus-client-derive-encode" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" dependencies = [ "proc-macro2", "quote", @@ -7452,14 +7434,13 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", "quick-protobuf", - "thiserror 1.0.69", - "unsigned-varint 0.8.0", + "thiserror 2.0.12", + "unsigned-varint", ] [[package]] @@ -7497,7 +7478,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.23", - "socket2", + "socket2 0.5.8", "thiserror 2.0.12", "tokio", "tracing", @@ -7532,7 +7513,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.5.8", "tracing", "windows-sys 0.59.0", ] @@ -8266,8 +8247,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "pin-project", @@ -8911,6 +8891,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "spin" version = "0.9.8" @@ -9499,7 +9489,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.8", "tokio-macros", "tracing", "windows-sys 0.52.0", @@ -9633,7 +9623,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "socket2", + "socket2 0.5.8", "tokio", "tokio-stream", "tower 0.4.13", @@ -10037,12 +10027,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "unsigned-varint" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" - [[package]] name = "unsigned-varint" version = "0.8.0" @@ -10817,6 +10801,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.42.2" diff --git a/Cargo.toml b/Cargo.toml index ae84d645bb9..96f9e2664f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,7 +163,7 @@ fs2 = "0.4" futures = "0.3" genesis = { path = "beacon_node/genesis" } # This is tracking the sigp-gossipsub branch on sigp/rust-libp2p commit: Aug 20 2025 -gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/sigp/rust-libp2p.git", rev = "5acdf89a65d64098f9346efa5769e57bcd19dea9", "features" = ["metrics"] } +gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages", features = ["partial_messages", "metrics"] } graffiti_file = { path = "validator_client/graffiti_file" } hashlink = "0.9.0" health_metrics = { path = "common/health_metrics" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3e02baf9017..6a8d5af5293 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -24,7 +24,7 @@ use crate::chain_config::ChainConfig; use crate::custody_context::CustodyContextSsz; use crate::data_availability_checker::{ Availability, AvailabilityCheckError, AvailableBlock, AvailableBlockData, - DataAvailabilityChecker, DataColumnReconstructionResult, + DataAvailabilityChecker, DataColumnReconstructionResult, MergedData, }; use crate::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use crate::early_attester_cache::EarlyAttesterCache; @@ -130,7 +130,9 @@ use tokio_stream::Stream; use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace, warn}; use tree_hash::TreeHash; use types::blob_sidecar::FixedBlobSidecarList; +use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::payload::BlockProductionVersion; use types::*; @@ -2194,7 +2196,8 @@ impl BeaconChain { self: &Arc, data_column_sidecar: Arc>, subnet_id: DataColumnSubnetId, - ) -> Result, GossipDataColumnError> { + ) -> Result>, GossipDataColumnError> + { metrics::inc_counter(&metrics::DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); GossipVerifiedDataColumn::new(data_column_sidecar, subnet_id, self).inspect(|_| { @@ -2202,6 +2205,22 @@ impl BeaconChain { }) } + #[instrument(skip_all, level = "trace")] + pub fn verify_partial_data_column_sidecar_for_gossip( + self: &Arc, + data_column_sidecar: Arc>, + ) -> Result< + GossipVerifiedDataColumn>, + GossipDataColumnError, + > { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); + let _timer = + metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + GossipVerifiedDataColumn::new_partial(data_column_sidecar, self).inspect(|_| { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); + }) + } + #[instrument(skip_all, level = "trace")] pub fn verify_blob_sidecar_for_gossip( self: &Arc, @@ -3042,10 +3061,11 @@ impl BeaconChain { /// Cache the data columns in the processing cache, process it, then evict it from the cache if it was /// imported or errors. #[instrument(skip_all, level = "debug")] - pub async fn process_gossip_data_columns( + pub async fn process_gossip_data_columns>( self: &Arc, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let Ok((slot, block_root)) = data_columns .iter() @@ -3068,16 +3088,18 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - self.emit_sse_data_column_sidecar_events( - &block_root, - data_columns.iter().map(|column| column.as_data_column()), - ); + // TODO(dknopik): impl SSE + //self.emit_sse_data_column_sidecar_events( + // &block_root, + // data_columns.iter().map(|column| column.as_data_column()), + //); self.check_gossip_data_columns_availability_and_import( slot, block_root, data_columns, publish_fn, + data_publish_fn, ) .await } @@ -3143,11 +3165,12 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CustodyColumns(columns) => { - self.emit_sse_data_column_sidecar_events( - &block_root, - columns.iter().map(|column| column.as_data_column()), - ); + EngineGetBlobsOutput::CustodyColumns(_columns) => { + // TODO(dknopik): impl SSE + //self.emit_sse_data_column_sidecar_events( + // &block_root, + // columns.iter().map(|column| column.as_data_column()), + //); } } @@ -3543,22 +3566,23 @@ impl BeaconChain { /// Checks if the provided data column can make any cached blocks available, and imports immediately /// if so, otherwise caches the data column in the data availability checker. - async fn check_gossip_data_columns_availability_and_import( + async fn check_gossip_data_columns_availability_and_import>( self: &Arc, slot: Slot, block_root: Hash256, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { if let Some(slasher) = self.slasher.as_ref() { - for data_colum in &data_columns { - slasher.accept_block_header(data_colum.signed_block_header()); + for header in data_columns.iter().filter_map(|c| c.signed_block_header()) { + slasher.accept_block_header(header.clone()); } } let availability = self .data_availability_checker - .put_gossip_verified_data_columns(block_root, slot, data_columns)?; + .put_gossip_verified_data_columns(block_root, slot, data_columns, data_publish_fn)?; self.process_availability(slot, availability, publish_fn) .await @@ -3635,7 +3659,7 @@ impl BeaconChain { data_columns.iter().map(|c| c.as_data_column()), )?; self.data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, data_columns)? + .put_kzg_verified_custody_data_columns(block_root, data_columns, |_| ())? } }; @@ -3668,10 +3692,13 @@ impl BeaconChain { .await } - fn check_data_column_sidecar_header_signature_and_slashability<'a>( + fn check_data_column_sidecar_header_signature_and_slashability< + 'a, + C: DasColumn + 'a, + >( self: &Arc, block_root: Hash256, - custody_columns: impl IntoIterator>, + custody_columns: impl IntoIterator, ) -> Result<(), BlockError> { let mut slashable_cache = self.observed_slashable.write(); // Process all unique block headers - previous logic assumed all headers were identical and @@ -3679,13 +3706,13 @@ impl BeaconChain { // from RPC. for header in custody_columns .into_iter() - .map(|c| c.signed_block_header.clone()) + .filter_map(|c| c.signed_block_header()) .unique() { // Return an error if *any* header signature is invalid, we do not want to import this // list of blobs into the DA checker. However, we will process any valid headers prior // to the first invalid header in the slashable cache & slasher. - verify_header_signature::(self, &header)?; + verify_header_signature::(self, header)?; slashable_cache .observe_slashable( @@ -3695,7 +3722,7 @@ impl BeaconChain { ) .map_err(|e| BlockError::BeaconChainError(Box::new(e.into())))?; if let Some(slasher) = self.slasher.as_ref() { - slasher.accept_block_header(header); + slasher.accept_block_header(header.clone()); } } Ok(()) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4dbc634b241..8807d8ad980 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -39,7 +39,9 @@ use crate::metrics::{ }; use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; +use types::das_column::{ColumnComparison, DasColumn}; use types::non_zero_usize::new_non_zero_usize; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -179,18 +181,40 @@ impl DataAvailabilityChecker { }) } - /// Check if the exact data column is in the availability cache. - pub fn is_data_column_cached( + /// Check if the (potentially partial) data column is in the availability cache. + /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells + /// otherwise. + pub fn determine_missing_cells>( &self, block_root: &Hash256, - data_column: &DataColumnSidecar, - ) -> bool { + data_column: &C, + ) -> Option> { + fn do_compare, C2: DasColumn>( + cached: &C1, + data_column: &C2, + ) -> Option> { + match cached.compare(data_column) { + ColumnComparison::Equal => Some(vec![]), + ColumnComparison::MissingCells { missing_in_lhs, .. } => Some(missing_in_lhs), + _ => None, + } + } + self.availability_cache .peek_pending_components(block_root, |components| { - components.is_some_and(|components| { - let cached_column_opt = components.get_cached_data_column(data_column.index); - cached_column_opt.is_some_and(|cached| *cached == *data_column) - }) + if let Some(components) = components { + if let Some(cached_column) = + components.get_cached_data_column(data_column.index()) + { + return do_compare(cached_column.as_ref(), data_column); + } + if let Some(cached_column) = + components.get_cached_partial_data_column(data_column.index()) + { + return do_compare(cached_column.as_ref(), data_column); + } + } + Some(data_column.cells_present().collect()) }) } @@ -266,8 +290,11 @@ impl DataAvailabilityChecker { .map(KzgVerifiedCustodyDataColumn::from_asserted_custody) .collect::>(); - self.availability_cache - .put_kzg_verified_data_columns(block_root, verified_custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + verified_custody_columns, + |_| (), + ) } /// Check if we've cached other blobs for this block. If it completes a set and we also @@ -306,12 +333,14 @@ impl DataAvailabilityChecker { #[instrument(skip_all, level = "trace")] pub fn put_gossip_verified_data_columns< O: ObservationStrategy, - I: IntoIterator>, + C: DasColumn, + I: IntoIterator>, >( &self, block_root: Hash256, slot: Slot, data_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = slot.epoch(T::EthSpec::slots_per_epoch()); let sampling_columns = self @@ -323,20 +352,28 @@ impl DataAvailabilityChecker { .map(|c| KzgVerifiedCustodyDataColumn::from_asserted_custody(c.into_inner())) .collect::>(); - self.availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + custody_columns, + publish_fn, + ) } #[instrument(skip_all, level = "trace")] pub fn put_kzg_verified_custody_data_columns< - I: IntoIterator>, + I: IntoIterator>, + C: DasColumn, >( &self, block_root: Hash256, custody_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + custody_columns, + publish_fn, + ) } /// Check if we have all the blobs for a block. Returns `Availability` which has information @@ -651,7 +688,11 @@ impl DataAvailabilityChecker { ); self.availability_cache - .put_kzg_verified_data_columns(*block_root, data_columns_to_import_and_publish.clone()) + .put_kzg_verified_data_columns( + *block_root, + data_columns_to_import_and_publish.clone(), + |_| (), + ) .map(|availability| { DataColumnReconstructionResult::Success(( availability, @@ -861,6 +902,20 @@ impl MaybeAvailableBlock { } } +pub struct MergedData { + pub completed_columns: Vec>>, + pub updated_partials: Vec>>, +} + +impl MergedData { + pub fn empty() -> Self { + MergedData { + completed_columns: vec![], + updated_partials: vec![], + } + } +} + #[cfg(test)] mod test { use super::*; @@ -878,7 +933,9 @@ mod test { use std::time::Duration; use store::HotColdDB; use types::data_column_sidecar::DataColumn; - use types::{ChainSpec, ColumnIndex, EthSpec, ForkName, MainnetEthSpec, Slot}; + use types::{ + ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, ForkName, MainnetEthSpec, Slot, + }; type E = MainnetEthSpec; type T = EphemeralHarnessType; @@ -1011,10 +1068,10 @@ mod test { let gossip_columns = data_columns .into_iter() .filter(|d| requested_columns.contains(&d.index)) - .map(GossipVerifiedDataColumn::::__new_for_testing) + .map(GossipVerifiedDataColumn::>::__new_for_testing) .collect::>(); da_checker - .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns) + .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns, |_| ()) .expect("should put gossip custody columns"); // THEN the sampling size for the end slot of the same epoch remains unchanged @@ -1141,7 +1198,7 @@ mod test { da_checker .availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + .put_kzg_verified_data_columns(block_root, custody_columns, |_| ()) .expect("should put custody columns"); // Try reconstrucing diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index b842a1a3f99..c5826afb5ee 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1,5 +1,5 @@ -use super::AvailableBlockData; use super::state_lru_cache::{DietAvailabilityPendingExecutedBlock, StateLRUCache}; +use super::{AvailableBlockData, MergedData}; use crate::CustodyContext; use crate::beacon_chain::BeaconStore; use crate::blob_verification::KzgVerifiedBlob; @@ -18,6 +18,8 @@ use std::sync::Arc; use tracing::{Span, debug, debug_span}; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobIdentifier; +use types::das_column::DasColumn; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ BlobSidecar, BlockImportSource, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeFixedVector, RuntimeVariableList, @@ -73,7 +75,10 @@ impl CachedBlock { pub struct PendingComponents { pub block_root: Hash256, pub verified_blobs: RuntimeFixedVector>>, - pub verified_data_columns: Vec>, + // TODO(dknopik): four options: two fields, one field containing an enum, converting everything to partial, or refactor for cell level storage + pub verified_partial_columns: + Vec>>, + pub verified_data_columns: Vec>>, pub block: Option>, pub reconstruction_started: bool, span: Span, @@ -93,7 +98,7 @@ impl PendingComponents { }) } - /// Returns an immutable reference to the cached data column. + /// Returns an immutable reference to the full cached data column. pub fn get_cached_data_column( &self, data_column_index: u64, @@ -104,6 +109,17 @@ impl PendingComponents { .map(|d| d.clone_arc()) } + /// Returns an immutable reference to the partial cached data column. + pub fn get_cached_partial_data_column( + &self, + data_column_index: u64, + ) -> Option>> { + self.verified_partial_columns + .iter() + .find(|d| d.index() == data_column_index) + .map(|d| d.clone_arc()) + } + /// Returns a mutable reference to the fixed vector of cached blobs. pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector>> { &mut self.verified_blobs @@ -186,17 +202,56 @@ impl PendingComponents { } /// Merges a given set of data columns into the cache. - fn merge_data_columns>>( + fn merge_data_columns< + I: IntoIterator>, + C: DasColumn, + >( &mut self, kzg_verified_data_columns: I, - ) -> Result<(), AvailabilityCheckError> { + ) -> Result, AvailabilityCheckError> { + let mut merged_data = MergedData::empty(); + for data_column in kzg_verified_data_columns { - if self.get_cached_data_column(data_column.index()).is_none() { - self.verified_data_columns.push(data_column); + if self.get_cached_data_column(data_column.index()).is_some() { + // already have the full column + continue; + } + + if let Some(full) = data_column.try_as_full() { + self.verified_partial_columns + .retain(|col| col.index() != full.index()); + self.verified_data_columns.push(full); + continue; + } + + let partial = data_column.into_partial(); + if let Some((idx, cached_partial)) = self + .verified_partial_columns + .iter_mut() + .enumerate() + .find(|d| d.1.index() == partial.index()) + { + let did_merge = cached_partial.merge(&partial); + if did_merge { + if let Some(block) = &self.block + && let Some(full) = cached_partial.try_upgrade_full(block.as_block()) + { + merged_data.completed_columns.push(full.clone_arc()); + self.verified_data_columns.push(full); + self.verified_partial_columns.remove(idx); + } else { + merged_data + .updated_partials + .push(cached_partial.clone_arc()); + } + } + } else { + merged_data.updated_partials.push(partial.clone_arc()); + self.verified_partial_columns.push(partial); } } - Ok(()) + Ok(merged_data) } /// Inserts a new block and revalidates the existing blobs against it. @@ -247,7 +302,12 @@ impl PendingComponents { let data_columns = self .verified_data_columns .iter() - .map(|d| d.clone().into_inner()) + .filter_map(|d| { + d.clone() + .into_inner() + .as_full(Some(block.as_block())) + .map(|c| Arc::new(c.into_owned())) + }) .collect::>(); Some(AvailableBlockData::DataColumns(data_columns)) } @@ -340,6 +400,7 @@ impl PendingComponents { block_root, verified_blobs: RuntimeFixedVector::new(vec![None; max_len]), verified_data_columns: vec![], + verified_partial_columns: vec![], block: None, reconstruction_started: false, span, @@ -412,7 +473,7 @@ pub struct DataAvailabilityCheckerInner { // the current usage, as it's deconstructed immediately. #[allow(clippy::large_enum_variant)] pub(crate) enum ReconstructColumnsDecision { - Yes(Vec>), + Yes(Vec>>), No(&'static str), } @@ -514,7 +575,7 @@ impl DataAvailabilityCheckerInner { *blob_opt = Some(blob); } } - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_blobs(fixed_blobs); Ok(()) @@ -533,17 +594,21 @@ impl DataAvailabilityCheckerInner { #[allow(clippy::type_complexity)] pub fn put_kzg_verified_data_columns< - I: IntoIterator>, + I: IntoIterator>, + C: DasColumn, >( &self, block_root: Hash256, kzg_verified_data_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); - let Some(epoch) = kzg_verified_data_columns - .peek() - .map(|verified_blob| verified_blob.as_data_column().epoch()) - else { + let Some(epoch) = kzg_verified_data_columns.peek().map(|verified_blob| { + verified_blob + .as_data_column() + .slot() + .epoch(T::EthSpec::slots_per_epoch()) + }) else { // No columns are processed. This can occur if all received columns were filtered out // before this point, e.g. due to a CGC change that caused extra columns to be downloaded // // before the new CGC took effect. @@ -551,11 +616,13 @@ impl DataAvailabilityCheckerInner { return Ok(Availability::MissingComponents(block_root)); }; - let pending_components = + let (pending_components, merged_data) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_data_columns(kzg_verified_data_columns) })?; + publish_fn(merged_data); + let num_expected_columns = self .custody_context .num_of_data_columns_to_sample(epoch, &self.spec); @@ -610,23 +677,23 @@ impl DataAvailabilityCheckerInner { /// /// Once the update is complete, the write lock is downgraded and a read guard with a /// reference of the updated `PendingComponents` is returned. - fn update_or_insert_pending_components( + fn update_or_insert_pending_components( &self, block_root: Hash256, epoch: Epoch, update_fn: F, - ) -> Result>, AvailabilityCheckError> + ) -> Result<(MappedRwLockReadGuard<'_, PendingComponents>, R), AvailabilityCheckError> where - F: FnOnce(&mut PendingComponents) -> Result<(), AvailabilityCheckError>, + F: FnOnce(&mut PendingComponents) -> Result, { let mut write_lock = self.critical.write(); - { + let ret = { let pending_components = write_lock.get_or_insert_mut(block_root, || { PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) }); update_fn(pending_components)? - } + }; RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { cache.peek(&block_root) @@ -634,6 +701,7 @@ impl DataAvailabilityCheckerInner { .map_err(|_| { AvailabilityCheckError::Unexpected("pending components should exist".to_string()) }) + .map(|guard| (guard, ret)) } /// Check whether data column reconstruction should be attempted. @@ -703,7 +771,7 @@ impl DataAvailabilityCheckerInner { source: BlockImportSource, ) -> Result<(), AvailabilityCheckError> { let epoch = block.epoch(); - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.insert_pre_execution_block(block, source); Ok(()) @@ -745,7 +813,7 @@ impl DataAvailabilityCheckerInner { .state_cache .register_pending_executed_block(executed_block); - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_block(diet_executed_block); Ok(()) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 07f85b045ab..7c4f558a168 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -15,10 +15,12 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; +use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, - SignedBeaconBlockHeader, Slot, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip data column. @@ -187,13 +189,19 @@ impl From for GossipDataColumnError { /// A wrapper around a `DataColumnSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. #[derive(Debug)] -pub struct GossipVerifiedDataColumn { +pub struct GossipVerifiedDataColumn< + T: BeaconChainTypes, + C: DasColumn, + O: ObservationStrategy = Observe, +> { block_root: Hash256, - data_column: KzgVerifiedDataColumn, + data_column: KzgVerifiedDataColumn, _phantom: PhantomData, } -impl Clone for GossipVerifiedDataColumn { +impl, O: ObservationStrategy> Clone + for GossipVerifiedDataColumn +{ fn clone(&self) -> Self { Self { block_root: self.block_root, @@ -203,23 +211,69 @@ impl Clone for GossipVerifiedDataCo } } -impl GossipVerifiedDataColumn { +impl, O: ObservationStrategy> + GossipVerifiedDataColumn +{ + /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. + pub fn __new_for_testing(column_sidecar: Arc) -> Self { + Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), + _phantom: Default::default(), + } + } + + pub fn as_data_column(&self) -> &C { + self.data_column.as_data_column() + } + + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc { + self.data_column.clone_data_column() + } + + pub fn block_root(&self) -> Hash256 { + self.block_root + } + + pub fn slot(&self) -> Slot { + self.data_column.data.slot() + } + + pub fn index(&self) -> ColumnIndex { + self.data_column.index() + } + + pub fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + self.data_column.data.signed_block_header() + } + + pub fn into_inner(self) -> KzgVerifiedDataColumn { + self.data_column + } +} + +impl + GossipVerifiedDataColumn, O> +{ pub fn new( column_sidecar: Arc>, subnet_id: DataColumnSubnetId, chain: &BeaconChain, ) -> Result { - let header = column_sidecar.signed_block_header.clone(); + let header = column_sidecar.signed_block_header().cloned(); // We only process slashing info if the gossip verification failed // since we do not process the data column any further in that case. - validate_data_column_sidecar_for_gossip::(column_sidecar, subnet_id, chain).map_err( - |e| { + validate_data_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { + if let Some(header) = header { process_block_slash_info::<_, GossipDataColumnError>( chain, BlockSlashInfo::from_early_error_data_column(header, e), ) - }, - ) + } else { + e + } + }) } /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY. @@ -238,9 +292,12 @@ impl GossipVerifiedDataColumn // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; + // TODO(dknopik): is proper handling of none? if chain .data_availability_checker - .is_data_column_cached(&column_sidecar.block_root(), &column_sidecar) + .determine_missing_cells(&column_sidecar.block_root(), column_sidecar.as_ref()) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)? + .is_empty() { // Observe this data column so we don't process it again. if O::observe() { @@ -255,97 +312,76 @@ impl GossipVerifiedDataColumn _phantom: Default::default(), }) } +} - /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { - Self { - block_root: column_sidecar.block_root(), - data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), - _phantom: Default::default(), - } - } - - pub fn as_data_column(&self) -> &DataColumnSidecar { - self.data_column.as_data_column() - } - - /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { - self.data_column.clone_data_column() - } - - pub fn block_root(&self) -> Hash256 { - self.block_root - } - - pub fn slot(&self) -> Slot { - self.data_column.data.slot() - } - - pub fn index(&self) -> ColumnIndex { - self.data_column.data.index - } - - pub fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.data_column.data.signed_block_header.clone() - } - - pub fn into_inner(self) -> KzgVerifiedDataColumn { - self.data_column +impl + GossipVerifiedDataColumn, O> +{ + pub fn new_partial( + column_sidecar: Arc>, + chain: &BeaconChain, + ) -> Result { + validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) } } /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. -#[derive(Debug, Derivative, Clone, Encode, Decode)] +#[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] -#[ssz(struct_behaviour = "transparent")] -pub struct KzgVerifiedDataColumn { - data: Arc>, +pub struct KzgVerifiedDataColumn> { + data: Arc, + _phantom: PhantomData, } -impl KzgVerifiedDataColumn { - pub fn new( - data_column: Arc>, - kzg: &Kzg, - ) -> Result, KzgError)> { +impl> KzgVerifiedDataColumn { + pub fn new(data_column: Arc, kzg: &Kzg) -> Result, KzgError)> { verify_kzg_for_data_column(data_column, kzg) } /// Mark a data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc>) -> Self { - Self { data: data_column } + pub fn from_execution_verified(data_column: Arc) -> Self { + Self { + data: data_column, + _phantom: PhantomData, + } } /// Create a `KzgVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { - Self { data: data_column } + pub(crate) fn __new_for_testing(data_column: Arc) -> Self { + Self { + data: data_column, + _phantom: PhantomData, + } } pub fn from_batch_with_scoring( - data_columns: Vec>>, + data_columns: Vec>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { verify_kzg_for_data_column_list(data_columns.iter(), kzg)?; Ok(data_columns .into_iter() - .map(|column| Self { data: column }) + .map(|column| Self { + data: column, + _phantom: PhantomData, + }) .collect()) } - pub fn to_data_column(self) -> Arc> { + pub fn to_data_column(self) -> Arc { self.data } - pub fn as_data_column(&self) -> &DataColumnSidecar { + pub fn as_data_column(&self) -> &C { &self.data } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc { self.data.clone() } pub fn index(&self) -> ColumnIndex { - self.data.index + self.data.index() } } @@ -383,22 +419,70 @@ impl CustodyDataColumn { } /// Data column that we must custody and has completed kzg verification -#[derive(Debug, Derivative, Clone, Encode, Decode)] +#[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] -#[ssz(struct_behaviour = "transparent")] -pub struct KzgVerifiedCustodyDataColumn { - data: Arc>, +pub struct KzgVerifiedCustodyDataColumn> { + data: Arc, + _phantom: PhantomData, } -impl KzgVerifiedCustodyDataColumn { +impl> KzgVerifiedCustodyDataColumn { /// Mark a column as custody column. Caller must ensure that our current custody requirements /// include this column - pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { + pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { Self { data: kzg_verified.to_data_column(), + _phantom: PhantomData, } } + pub fn into_inner(self) -> Arc { + self.data + } + + pub fn as_data_column(&self) -> &C { + &self.data + } + pub fn clone_arc(&self) -> Arc { + self.data.clone() + } + pub fn index(&self) -> ColumnIndex { + self.data.index() + } + + // TODO(dknopik): below three functions are baaaad. They clone in far too many cases + + pub fn try_as_full(&self) -> Option>> { + self.data + .as_full(None) + .map(|full| KzgVerifiedCustodyDataColumn { + data: Arc::new(full.into_owned()), + _phantom: PhantomData, + }) + } + + pub fn try_upgrade_full( + &self, + block: &SignedBeaconBlock, + ) -> Option>> { + self.data + .as_full(Some(block)) + .map(|full| KzgVerifiedCustodyDataColumn { + data: Arc::new(full.into_owned()), + _phantom: PhantomData, + }) + } + + pub fn into_partial(self) -> KzgVerifiedCustodyDataColumn> { + let column = Arc::try_unwrap(self.data).unwrap_or_else(|column| (*column).clone()); + KzgVerifiedCustodyDataColumn { + data: Arc::new(column.into_partial()), + _phantom: PhantomData, + } + } +} + +impl KzgVerifiedCustodyDataColumn> { /// Verify a column already marked as custody column pub fn new( data_column: CustodyDataColumn, @@ -407,6 +491,7 @@ impl KzgVerifiedCustodyDataColumn { verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; Ok(Self { data: data_column.data, + _phantom: PhantomData, }) } @@ -414,7 +499,7 @@ impl KzgVerifiedCustodyDataColumn { kzg: &Kzg, partial_set_of_columns: &[Self], spec: &ChainSpec, - ) -> Result>, KzgError> { + ) -> Result, KzgError> { let all_data_columns = reconstruct_data_columns( kzg, partial_set_of_columns @@ -427,23 +512,45 @@ impl KzgVerifiedCustodyDataColumn { Ok(all_data_columns .into_iter() .map(|data| { - KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { data }) + KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { + data, + _phantom: PhantomData, + }) }) .collect::>()) } +} - pub fn into_inner(self) -> Arc> { - self.data - } +impl KzgVerifiedCustodyDataColumn> { + pub fn merge(&mut self, other: &Self) -> bool { + let to_be_added = other + .data + .column + .sidecar + .cells_present_bitmap + .difference(&self.data.column.sidecar.cells_present_bitmap); + if to_be_added.is_zero() { + return false; + }; - pub fn as_data_column(&self) -> &DataColumnSidecar { - &self.data - } - pub fn clone_arc(&self) -> Arc> { - self.data.clone() - } - pub fn index(&self) -> ColumnIndex { - self.data.index + let Some(merged) = self.data.column.sidecar.merge(&other.data.column.sidecar) else { + return false; + }; + + *self = Self { + data: Arc::new(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + block_root: self.data.column.block_root, + index: self.data.column.index, + sidecar: merged, + }), + kzg_commitments: self.data.kzg_commitments.clone(), + slot: self.data.slot, + }), + _phantom: PhantomData, + }; + + true } } @@ -451,13 +558,16 @@ impl KzgVerifiedCustodyDataColumn { /// /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] -pub fn verify_kzg_for_data_column( - data_column: Arc>, +pub fn verify_kzg_for_data_column>( + data_column: Arc, kzg: &Kzg, -) -> Result, (Option, KzgError)> { +) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); validate_data_columns(kzg, iter::once(&data_column))?; - Ok(KzgVerifiedDataColumn { data: data_column }) + Ok(KzgVerifiedDataColumn { + data: data_column, + _phantom: PhantomData, + }) } /// Complete kzg verification for a list of `DataColumnSidecar`s. @@ -465,12 +575,14 @@ pub fn verify_kzg_for_data_column( /// /// Note: This function should be preferred over calling `verify_kzg_for_data_column` /// in a loop since this function kzg verifies a list of data columns more efficiently. -pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I>( +pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I, A, C>( data_column_iter: I, kzg: &'a Kzg, ) -> Result<(), (Option, KzgError)> where - I: Iterator>> + Clone, + I: Iterator + Clone, + A: AsRef + 'a, + C: DasColumn + 'a, { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES); validate_data_columns(kzg, data_column_iter)?; @@ -482,7 +594,7 @@ pub fn validate_data_column_sidecar_for_gossip>, subnet: DataColumnSubnetId, chain: &BeaconChain, -) -> Result, GossipDataColumnError> { +) -> Result, O>, GossipDataColumnError> { let column_slot = data_column.slot(); verify_data_column_sidecar(&data_column, &chain.spec)?; verify_index_matches_subnet(&data_column, subnet, &chain.spec)?; @@ -491,13 +603,16 @@ pub fn validate_data_column_sidecar_for_gossip( + data_column: Arc>, + chain: &BeaconChain, +) -> Result< + GossipVerifiedDataColumn, O>, + GossipDataColumnError, +> { + // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply + let column_slot = data_column.slot(); + verify_sidecar_not_from_future_slot(chain, column_slot)?; + verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; + + let missing_cells = chain + .data_availability_checker + .determine_missing_cells(&data_column.block_root(), data_column.as_ref()) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?; + if missing_cells.is_empty() { + return Err(GossipDataColumnError::PriorKnownUnpublished); + } + + let filtered_column = Arc::new( + data_column + .clone_filter(|idx| missing_cells.contains(&idx)) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?, + ); + + // We do not have to check block related data here, as we create the verifiable column from + // gossip accepted block + + let kzg = &chain.kzg; + let kzg_verified_data_column = verify_kzg_for_data_column(filtered_column, kzg) + .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; + + // TODO(dknopik): observe?! I do not think so, as publishing works differently anyway... + // Basically we publish if we update our internal view, but this is merged elsewhere + //if O::observe() { + // observe_gossip_data_column(&data_column, chain)?; + //} + + Ok(GossipVerifiedDataColumn { + block_root: data_column.block_root(), + data_column: kzg_verified_data_column, + _phantom: PhantomData, + }) +} + /// Verify if the data column sidecar is valid. fn verify_data_column_sidecar( data_column: &DataColumnSidecar, @@ -851,7 +1016,7 @@ mod test { harness.advance_slot(); let verify_fn = |column_sidecar: DataColumnSidecar| { - GossipVerifiedDataColumn::<_>::new_for_block_publishing( + GossipVerifiedDataColumn::<_, _>::new_for_block_publishing( column_sidecar.into(), &harness.chain, ) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 9526921da73..4e98e81df24 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -51,6 +51,7 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } + // TODO(dknopik): fall back to this pub(crate) async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -67,6 +68,22 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } + pub(crate) async fn get_blobs_v3( + &self, + versioned_hashes: Vec, + ) -> Result>>>, FetchEngineBlobError> { + let execution_layer = self + .chain + .execution_layer + .as_ref() + .ok_or(FetchEngineBlobError::ExecutionLayerMissing)?; + + execution_layer + .get_blobs_v3(versioned_hashes) + .await + .map_err(FetchEngineBlobError::RequestFailed) + } + pub(crate) fn blobs_known_for_proposal( &self, proposer: u64, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 4c6b2d10a95..55f43289900 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -17,7 +17,7 @@ use crate::block_verification_types::AsBlock; use crate::data_column_verification::{KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn}; #[cfg_attr(test, double)] use crate::fetch_blobs::fetch_blobs_beacon_adapter::FetchBlobsBeaconAdapter; -use crate::kzg_utils::blobs_to_data_column_sidecars; +use crate::kzg_utils::blobs_to_partial_data_columns; use crate::observed_block_producers::ProposalKey; use crate::validator_monitor::timestamp_now; use crate::{ @@ -32,9 +32,11 @@ use mockall_double::double; use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use tracing::{Span, debug, instrument, warn}; +use tracing::{Span, debug, instrument}; use types::blob_sidecar::BlobSidecarError; +use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ BeaconStateError, Blob, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, KzgProofs, SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, @@ -47,7 +49,9 @@ use types::{ pub enum EngineGetBlobsOutput { Blobs(Vec>), /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. - CustodyColumns(Vec>), + CustodyColumns( + Vec>>, + ), } #[derive(Debug)] @@ -246,9 +250,10 @@ async fn fetch_and_process_blobs_v2( let num_expected_blobs = versioned_hashes.len(); metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); + // TODO(dknopik): implement fallback to get_blobs_v2 debug!(num_expected_blobs, "Fetching blobs from the EL"); let response = chain_adapter - .get_blobs_v2(versioned_hashes) + .get_blobs_v3(versioned_hashes) .await .inspect_err(|_| { inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); @@ -260,30 +265,15 @@ async fn fetch_and_process_blobs_v2( return Ok(None); }; - let (blobs, proofs): (Vec<_>, Vec<_>) = blobs_and_proofs + let blobs_and_proofs: Vec<_> = blobs_and_proofs .into_iter() - .map(|blob_and_proof| { - let BlobAndProofV2 { blob, proofs } = blob_and_proof; - (blob, proofs) - }) - .unzip(); + .map(|blob_and_proof| blob_and_proof.map(|BlobAndProofV2 { blob, proofs }| (blob, proofs))) + .collect(); - let num_fetched_blobs = blobs.len(); + let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); - if num_fetched_blobs != num_expected_blobs { - // This scenario is not supposed to happen if the EL is spec compliant. - // It should either return all requested blobs or none, but NOT partial responses. - // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. - warn!( - num_fetched_blobs, - num_expected_blobs, "The EL did not return all requested blobs" - ); - inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); - return Ok(None); - } - - debug!(num_fetched_blobs, "All expected blobs received from the EL"); + debug!(num_fetched_blobs, "Blobs received from the EL"); inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); if chain_adapter.fork_choice_contains_block(&block_root) { @@ -300,8 +290,7 @@ async fn fetch_and_process_blobs_v2( &chain_adapter, block_root, block.clone(), - blobs, - proofs, + blobs_and_proofs, custody_columns_indices, ) .await?; @@ -337,10 +326,12 @@ async fn compute_custody_columns_to_import( chain_adapter: &Arc>, block_root: Hash256, block: Arc>>, - blobs: Vec>, - proofs: Vec>, + blobs_and_proofs: Vec, KzgProofs)>>, custody_columns_indices: &[ColumnIndex], -) -> Result>, FetchEngineBlobError> { +) -> Result< + Vec>>, + FetchEngineBlobError, +> { let kzg = chain_adapter.kzg().clone(); let spec = chain_adapter.spec().clone(); let chain_adapter_cloned = chain_adapter.clone(); @@ -353,13 +344,19 @@ async fn compute_custody_columns_to_import( let _guard = current_span.enter(); let mut timer = metrics::start_timer_vec( &metrics::DATA_COLUMN_SIDECAR_COMPUTATION, - &[&blobs.len().to_string()], + &[&blobs_and_proofs.len().to_string()], ); - let blob_refs = blobs.iter().collect::>(); - let cell_proofs = proofs.into_iter().flatten().collect(); + let blob_and_cell_refs = blobs_and_proofs + .iter() + .map(|option| { + option + .as_ref() + .map(|(blob, proofs)| (blob, proofs.as_ref())) + }) + .collect::>(); let data_columns_result = - blobs_to_data_column_sidecars(&blob_refs, cell_proofs, &block, &kzg, &spec) + blobs_to_partial_data_columns(blob_and_cell_refs, &block, &kzg, &spec) .discard_timer_on_break(&mut timer); drop(timer); @@ -370,7 +367,7 @@ async fn compute_custody_columns_to_import( .map(|data_columns| { data_columns .into_iter() - .filter(|col| custody_columns_indices.contains(&col.index)) + .filter(|col| custody_columns_indices.contains(&col.index())) .map(|col| { KzgVerifiedCustodyDataColumn::from_asserted_custody( KzgVerifiedDataColumn::from_execution_verified(col), diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 382775ab50f..1661c7fe605 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,7 +7,11 @@ use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; use types::beacon_block_body::KzgCommitments; +use types::das_column::DasColumn; use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError}; +use types::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, +}; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, @@ -46,12 +50,14 @@ pub fn validate_blob( } /// Validate a batch of `DataColumnSidecar`. -pub fn validate_data_columns<'a, E: EthSpec, I>( +pub fn validate_data_columns<'a, E: EthSpec, I, A, C>( kzg: &Kzg, data_column_iter: I, ) -> Result<(), (Option, KzgError)> where - I: Iterator>> + Clone, + I: Iterator + Clone, + A: AsRef + 'a, + C: DasColumn + 'a, { let mut cells = Vec::new(); let mut proofs = Vec::new(); @@ -59,22 +65,23 @@ where let mut commitments = Vec::new(); for data_column in data_column_iter { - let col_index = data_column.index; + let data_column = data_column.as_ref(); + let col_index = data_column.index(); - if data_column.column.is_empty() { + if data_column.column().is_empty() { return Err((Some(col_index), KzgError::KzgVerificationFailed)); } - for cell in &data_column.column { + for cell in data_column.column() { cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); column_indices.push(col_index); } - for &proof in &data_column.kzg_proofs { + for &proof in data_column.kzg_proofs() { proofs.push(Bytes48::from(proof)); } - for &commitment in &data_column.kzg_commitments { + for &commitment in data_column.kzg_commitments() { commitments.push(Bytes48::from(commitment)); } @@ -217,6 +224,58 @@ pub fn blobs_to_data_column_sidecars( .map_err(DataColumnSidecarError::BuildSidecarFailed) } +/// Build data column sidecars from a signed beacon block and its blobs. +#[instrument(skip_all, level = "debug", fields(blob_count = blobs_and_proofs.len()))] +pub fn blobs_to_partial_data_columns( + blobs_and_proofs: Vec, &[KzgProof])>>, + block: &SignedBeaconBlock, + kzg: &Kzg, + spec: &ChainSpec, +) -> Result>>, DataColumnSidecarError> { + if blobs_and_proofs.is_empty() { + return Ok(vec![]); + } + + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_err| DataColumnSidecarError::PreDeneb)?; + let signed_block_header = block.signed_block_header(); + + let blob_cells_and_proofs_vec = blobs_and_proofs + .into_par_iter() + .map(|maybe_blob_and_proofs| { + let Some((blob, proofs)) = maybe_blob_and_proofs else { + return Ok(None); + }; + + let blob = blob.as_ref().try_into().map_err(|e| { + KzgError::InconsistentArrayLength(format!( + "blob should have a guaranteed size due to FixedVector: {e:?}" + )) + })?; + + kzg.compute_cells(blob).and_then(|cells| { + let proofs = proofs.try_into().map_err(|e| { + KzgError::InconsistentArrayLength(format!( + "proof chunks should have exactly `number_of_columns` proofs: {e:?}" + )) + })?; + Ok(Some((cells, proofs))) + }) + }) + .collect::, KzgError>>()?; + + build_partial_data_columns( + kzg_commitments.clone(), + signed_block_header, + blob_cells_and_proofs_vec, + spec, + ) + .map_err(DataColumnSidecarError::BuildSidecarFailed) +} + pub fn compute_cells(blobs: &[&Blob], kzg: &Kzg) -> Result, KzgError> { let cells_vec = blobs .into_par_iter() @@ -295,6 +354,86 @@ pub(crate) fn build_data_column_sidecars( Ok(sidecars) } +pub(crate) fn build_partial_data_columns( + kzg_commitments: KzgCommitments, + signed_block_header: SignedBeaconBlockHeader, + blob_cells_and_proofs_vec: Vec>, + spec: &ChainSpec, +) -> Result>>, String> { + let number_of_columns = E::number_of_columns(); + let max_blobs_per_block = spec + .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) + as usize; + let mut bitmap = + CellBitmap::::with_capacity(blob_cells_and_proofs_vec.len()).map_err(|_| { + format!( + "Exceeded max committment count: {} (got {})", + E::max_blob_commitments_per_block(), + blob_cells_and_proofs_vec.len() + ) + })?; + let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + + for (idx, maybe_cells_and_proofs) in blob_cells_and_proofs_vec.into_iter().enumerate() { + let Some((blob_cells, blob_cell_proofs)) = maybe_cells_and_proofs else { + continue; + }; + + bitmap + .set(idx, true) + .expect("bitmap constructed from iterator length above"); + + // we iterate over each column, and we construct the column from "top to bottom", + // pushing on the cell and the corresponding proof at each column index. we do this for + // each blob (i.e. the outer loop). + for col in 0..number_of_columns { + let cell = blob_cells + .get(col) + .ok_or(format!("Missing blob cell at index {col}"))?; + let cell: Vec = cell.to_vec(); + let cell = Cell::::from(cell); + + let proof = blob_cell_proofs + .get(col) + .ok_or(format!("Missing blob cell KZG proof at index {col}"))?; + + let column = columns + .get_mut(col) + .ok_or(format!("Missing data column at index {col}"))?; + let column_proofs = column_kzg_proofs + .get_mut(col) + .ok_or(format!("Missing data column proofs at index {col}"))?; + + column.push(cell); + column_proofs.push(*proof); + } + } + + let sidecars: Vec>> = columns + .into_iter() + .zip(column_kzg_proofs) + .enumerate() + .map(|(index, (col, proofs))| { + Arc::new(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + block_root: signed_block_header.message.canonical_root(), + index: index as u64, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap.clone(), + column: DataColumn::::from(col), + kzg_proofs: VariableList::from(proofs), + }, + }), + kzg_commitments: kzg_commitments.clone(), + slot: signed_block_header.message.slot, + }) + }) + .collect(); + + Ok(sidecars) +} + /// Reconstruct blobs from a subset of data column sidecars (requires at least 50%). /// /// If `blob_indices_opt` is `None`, this function attempts to reconstruct all blobs associated @@ -471,7 +610,7 @@ mod test { blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) .unwrap(); - let result = validate_data_columns::(kzg, column_sidecars.iter()); + let result = validate_data_columns(kzg, column_sidecars.iter()); assert!(result.is_ok()); } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 8f1da7b67b7..6ca2af01589 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1662,6 +1662,27 @@ pub static DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_processing_requests_total", + "Count of all partial data column sidecars submitted for processing", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_processing_successes_total", + "Number of partial data column sidecars verified for gossip", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = + LazyLock::new(|| { + try_create_histogram( + "beacon_partial_data_column_sidecar_gossip_verification_seconds", + "Full runtime of partial data column sidecars gossip verification", + ) + }); pub static BLOBS_FROM_EL_HIT_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 38797d0264d..639b933795f 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -3206,7 +3206,7 @@ where if !verified_columns.is_empty() { self.chain - .process_gossip_data_columns(verified_columns, || Ok(())) + .process_gossip_data_columns(verified_columns, || Ok(()), |_| ()) .await .unwrap(); } diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index 466058eea38..d3e8af7a01e 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -79,7 +79,7 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let _ = harness .chain - .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(())) + .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(()), |_| ()) .await .unwrap(); diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 28ed0cca913..c3dfd0a9126 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -121,6 +121,7 @@ pub struct BeaconProcessorQueueLengths { gossip_block_queue: usize, gossip_blob_queue: usize, gossip_data_column_queue: usize, + gossip_partial_data_column_queue: usize, delayed_block_queue: usize, status_queue: usize, bbrange_queue: usize, @@ -187,6 +188,7 @@ impl BeaconProcessorQueueLengths { gossip_block_queue: 1024, gossip_blob_queue: 1024, gossip_data_column_queue: 1024, + gossip_partial_data_column_queue: 1024, delayed_block_queue: 1024, status_queue: 1024, bbrange_queue: 1024, @@ -579,6 +581,7 @@ pub enum Work { GossipBlock(AsyncFn), GossipBlobSidecar(AsyncFn), GossipDataColumnSidecar(AsyncFn), + GossipPartialDataColumnSidecar(AsyncFn), DelayedImportBlock { beacon_block_slot: Slot, beacon_block_root: Hash256, @@ -641,6 +644,7 @@ pub enum WorkType { GossipBlock, GossipBlobSidecar, GossipDataColumnSidecar, + GossipPartialDataColumnSidecar, DelayedImportBlock, GossipVoluntaryExit, GossipProposerSlashing, @@ -688,6 +692,7 @@ impl Work { Work::GossipBlock(_) => WorkType::GossipBlock, Work::GossipBlobSidecar(_) => WorkType::GossipBlobSidecar, Work::GossipDataColumnSidecar(_) => WorkType::GossipDataColumnSidecar, + Work::GossipPartialDataColumnSidecar(_) => WorkType::GossipPartialDataColumnSidecar, Work::DelayedImportBlock { .. } => WorkType::DelayedImportBlock, Work::GossipVoluntaryExit(_) => WorkType::GossipVoluntaryExit, Work::GossipProposerSlashing(_) => WorkType::GossipProposerSlashing, @@ -873,6 +878,8 @@ impl BeaconProcessor { let mut gossip_block_queue = FifoQueue::new(queue_lengths.gossip_block_queue); let mut gossip_blob_queue = FifoQueue::new(queue_lengths.gossip_blob_queue); let mut gossip_data_column_queue = FifoQueue::new(queue_lengths.gossip_data_column_queue); + let mut gossip_partial_data_column_queue = + FifoQueue::new(queue_lengths.gossip_partial_data_column_queue); let mut delayed_block_queue = FifoQueue::new(queue_lengths.delayed_block_queue); let mut status_queue = FifoQueue::new(queue_lengths.status_queue); @@ -1325,6 +1332,9 @@ impl BeaconProcessor { Work::GossipDataColumnSidecar { .. } => { gossip_data_column_queue.push(work, work_id) } + Work::GossipPartialDataColumnSidecar { .. } => { + gossip_partial_data_column_queue.push(work, work_id) + } Work::DelayedImportBlock { .. } => { delayed_block_queue.push(work, work_id) } @@ -1416,6 +1426,9 @@ impl BeaconProcessor { WorkType::GossipBlock => gossip_block_queue.len(), WorkType::GossipBlobSidecar => gossip_blob_queue.len(), WorkType::GossipDataColumnSidecar => gossip_data_column_queue.len(), + WorkType::GossipPartialDataColumnSidecar => { + gossip_partial_data_column_queue.len() + } WorkType::DelayedImportBlock => delayed_block_queue.len(), WorkType::GossipVoluntaryExit => gossip_voluntary_exit_queue.len(), WorkType::GossipProposerSlashing => gossip_proposer_slashing_queue.len(), @@ -1591,7 +1604,8 @@ impl BeaconProcessor { Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), Work::GossipBlock(work) | Work::GossipBlobSidecar(work) - | Work::GossipDataColumnSidecar(work) => task_spawner.spawn_async(async move { + | Work::GossipDataColumnSidecar(work) + | Work::GossipPartialDataColumnSidecar(work) => task_spawner.spawn_async(async move { work.await; }), Work::BlobsByRangeRequest(process_fn) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 98da7dbf2c7..9e3b984b969 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -592,6 +592,7 @@ pub struct EngineCapabilities { pub get_client_version_v1: bool, pub get_blobs_v1: bool, pub get_blobs_v2: bool, + pub get_blobs_v3: bool, } impl EngineCapabilities { diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index bc927e19b41..d85f5965c77 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -61,6 +61,7 @@ pub const ENGINE_GET_CLIENT_VERSION_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_GET_BLOBS_V1: &str = "engine_getBlobsV1"; pub const ENGINE_GET_BLOBS_V2: &str = "engine_getBlobsV2"; +pub const ENGINE_GET_BLOBS_V3: &str = "engine_getBlobsV3"; pub const ENGINE_GET_BLOBS_TIMEOUT: Duration = Duration::from_secs(1); /// This error is returned during a `chainId` call by Geth. @@ -736,6 +737,20 @@ impl HttpJsonRpc { .await } + pub async fn get_blobs_v3( + &self, + versioned_hashes: Vec, + ) -> Result>>, Error> { + let params = json!([versioned_hashes]); + + self.rpc_request( + ENGINE_GET_BLOBS_V3, + params, + ENGINE_GET_BLOBS_TIMEOUT * self.execution_timeout_multiplier, + ) + .await + } + pub async fn get_block_by_number( &self, query: BlockByNumberQuery<'_>, @@ -1186,6 +1201,7 @@ impl HttpJsonRpc { get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), get_blobs_v1: capabilities.contains(ENGINE_GET_BLOBS_V1), get_blobs_v2: capabilities.contains(ENGINE_GET_BLOBS_V2), + get_blobs_v3: capabilities.contains(ENGINE_GET_BLOBS_V3), }) } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 33decd4ec86..8b8fe228956 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -812,6 +812,9 @@ pub struct BlobAndProof { pub proofs: KzgProofs, } +/// A BlobAndProofV3 is just a BlobAndProofV2 that may also be `null` if unknown by the EL. +pub type BlobAndProofV3 = Option>; + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonForkchoiceStateV1 { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index a5fa0f34158..09a130dfdb3 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -4,7 +4,7 @@ //! This crate only provides useful functionality for "The Merge", it does not provide any of the //! deposit-contract functionality that the `beacon_node/eth1` crate already provides. -use crate::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use crate::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use crate::payload_cache::PayloadCache; use arc_swap::ArcSwapOption; use auth::{Auth, JwtKey, strip_prefix}; @@ -1895,6 +1895,23 @@ impl ExecutionLayer { } } + pub async fn get_blobs_v3( + &self, + query: Vec, + ) -> Result>>, Error> { + let capabilities = self.get_engine_capabilities(None).await?; + + if capabilities.get_blobs_v3 { + self.engine() + .request(|engine| async move { engine.api.get_blobs_v3(query).await }) + .await + .map_err(Box::new) + .map_err(Error::EngineError) + } else { + Err(Error::GetBlobsNotSupported) + } + } + pub async fn get_block_by_number( &self, query: BlockByNumberQuery<'_>, diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 712c773dda0..622132a0a8d 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -58,6 +58,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_client_version_v1: true, get_blobs_v1: true, get_blobs_v2: true, + get_blobs_v3: true, }; pub static DEFAULT_CLIENT_VERSION: LazyLock = diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 05a4a4b7a4a..a1412debc28 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -29,8 +29,9 @@ use tracing::{Span, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, - DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, - FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, SignedBlindedBeaconBlock, + DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, + FullPayload, FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, + SignedBlindedBeaconBlock, }; use warp::http::StatusCode; use warp::{Rejection, Reply, reply::Response}; @@ -247,7 +248,8 @@ pub async fn publish_block>( // Importing the columns could trigger block import and network publication in the case // where the block was already seen on gossip. if let Err(e) = - Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn)).await + Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn, |_| ())) + .await { let msg = format!("Invalid data column: {e}"); return if let BroadcastValidation::Gossip = validation_level { @@ -347,7 +349,7 @@ pub async fn publish_block>( type BuildDataSidecarTaskResult = Result< ( Vec>>, - Vec>, + Vec::EthSpec>>>, ), Rejection, >; @@ -405,7 +407,7 @@ fn build_data_columns( block: &SignedBeaconBlock>, blobs: BlobsList, kzg_cell_proofs: KzgProofs, -) -> Result>, Rejection> { +) -> Result>>, Rejection> { let slot = block.slot(); let data_column_sidecars = build_blob_data_column_sidecars(chain, block, blobs, kzg_cell_proofs).map_err(|e| { @@ -499,7 +501,7 @@ fn publish_blob_sidecars( fn publish_column_sidecars( sender_clone: &UnboundedSender>, - data_column_sidecars: &[GossipVerifiedDataColumn], + data_column_sidecars: &[GossipVerifiedDataColumn>], chain: &BeaconChain, ) -> Result<(), BlockError> { let malicious_withhold_count = chain.config.malicious_withhold_count; diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 7e69f6770bf..f7853e2412f 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -24,7 +24,7 @@ futures = { workspace = true } gossipsub = { workspace = true } hex = { workspace = true } itertools = { workspace = true } -libp2p-mplex = "0.43" +libp2p-mplex = { git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages" } lighthouse_version = { workspace = true } local-ip-address = "0.6" logging = { workspace = true } @@ -33,7 +33,7 @@ lru_cache = { workspace = true } metrics = { workspace = true } network_utils = { workspace = true } parking_lot = { workspace = true } -prometheus-client = "0.23.0" +prometheus-client = "0.24.0" rand = { workspace = true } regex = { workspace = true } serde = { workspace = true } @@ -52,7 +52,8 @@ types = { workspace = true } unsigned-varint = { version = "0.8", features = ["codec"] } [dependencies.libp2p] -version = "0.56" +git = "https://github.com/jxs/rust-libp2p.git" +branch = "gossipsub-partial-messages" default-features = false features = [ "identify", diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index ea2c53a07fe..64120b38f2a 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -22,7 +22,7 @@ use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; @@ -805,9 +805,10 @@ impl Network { .write() .insert(topic.clone()); + let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic) { + match self.gossipsub_mut().subscribe(&topic, partial) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false @@ -838,11 +839,8 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let message_data = message.encode(GossipEncoding::default()); - if let Err(e) = self - .gossipsub_mut() - .publish(Topic::from(topic.clone()), message_data.clone()) - { + let result = message.publish(self.gossipsub_mut(), topic.clone().into()); + if let Err(e) = result { match e { PublishError::Duplicate => { debug!( @@ -880,7 +878,8 @@ impl Network { } if let PublishError::NoPeersSubscribedToTopic = e { - self.gossip_cache.insert(topic, message_data); + // TODO(dknopik): fix gossip cache + //self.gossip_cache.insert(gossip_topic, message); } } } @@ -1291,6 +1290,44 @@ impl Network { } } } + Event::Partial { + topic_id, + propagation_source, + group_id, + message, + metadata: _, + } => { + // TODO(dknopik): take a look at the metadata and publish if we have something for them + // maybe by reusing the gossip cache?! + + if let Some(message) = message { + match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { + Err(error) => { + debug!( + topic = ?topic_id, + error, + "Could not decode gossipsub partial message" + ); + //reject the message + // TODO(dknopik): implement when ready in libp2p + //self.gossipsub_mut().report_message_validation_result( + // &todo!(), + // &propagation_source, + // MessageAcceptance::Reject, + //); + } + Ok(message) => { + // Notify the network + return Some(NetworkEvent::PubsubMessage { + id: MessageId::new(&[]), // TODO(dknopik): waht to send + source: propagation_source, + topic: topic_id, + message, + }); + } + } + } + } gossipsub::Event::Subscribed { peer_id, topic } => { if let Ok(topic) = GossipTopic::decode(topic.as_str()) { if let Some(subnet_id) = topic.subnet_id() { @@ -1764,7 +1801,10 @@ impl Network { fn inject_upnp_event(&mut self, event: libp2p::upnp::Event) { match event { - libp2p::upnp::Event::NewExternalAddr(addr) => { + libp2p::upnp::Event::NewExternalAddr { + external_addr: addr, + .. + } => { info!(%addr, "UPnP route established"); let mut iter = addr.iter(); let is_ip6 = { @@ -1794,7 +1834,7 @@ impl Network { } } } - libp2p::upnp::Event::ExpiredExternalAddr(_) => {} + libp2p::upnp::Event::ExpiredExternalAddr { .. } => {} libp2p::upnp::Event::GatewayNotFound => { info!("UPnP not available"); } diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 3f57406fc78..748df3cb15c 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -1,4 +1,5 @@ mod globals; +mod partial; mod pubsub; mod subnet; mod topics; diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs new file mode 100644 index 00000000000..3a165127e1e --- /dev/null +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -0,0 +1,123 @@ +use gossipsub::partial::{Metadata, PublishAction}; +use gossipsub::{Partial, PartialMessageError}; +use ssz::{Decode, Encode}; +use std::fmt::Debug; +use std::sync::Arc; +use types::EthSpec; +use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; + +#[derive(Debug, Clone, PartialEq)] +pub struct PartialDataColumnSidecarMessage { + pub partial_column: Arc>, + pub send_eager: Option>, +} + +impl From> for PartialDataColumnSidecarMessage { + fn from(value: DanglingPartialDataColumn) -> Self { + Self { + partial_column: Arc::new(value), + send_eager: None, + } + } +} + +impl From>> for PartialDataColumnSidecarMessage { + fn from(value: Arc>) -> Self { + Self { + partial_column: value, + send_eager: None, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SendEager { + /// The encoded message to send eagerly, i.e. when we have no metadata for that peer. + data: Vec, + /// The metadata to associate with a peer after sending it the eager message. + metadata: CellBitmapMetadata, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CellBitmapMetadata { + bitmap: CellBitmap, + encoded: Vec, +} + +impl Metadata for CellBitmapMetadata { + fn as_slice(&self) -> &[u8] { + &self.encoded + } + + fn update(&mut self, data: &[u8]) -> Result { + let data = CellBitmap::::from_ssz_bytes(data) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if data.len() != self.bitmap.len() { + return Err(PartialMessageError::OutOfRange); + } + let new_bitmap = self.bitmap.union(&data); + if self.bitmap == new_bitmap { + return Ok(false); + } + self.bitmap = new_bitmap; + self.encoded = self.bitmap.as_ssz_bytes(); + Ok(true) + } +} + +impl From> for CellBitmapMetadata { + fn from(value: CellBitmap) -> Self { + Self { + encoded: value.as_ssz_bytes(), + bitmap: value, + } + } +} + +impl Partial for PartialDataColumnSidecarMessage { + fn group_id(&self) -> impl AsRef<[u8]> { + &self.partial_column.block_root + } + + fn parts_metadata(&self) -> impl AsRef<[u8]> { + self.partial_column + .sidecar + .cells_present_bitmap + .as_ssz_bytes() + } + + fn partial_message_bytes_from_metadata( + &self, + metadata: Option>, + ) -> Result { + match metadata { + None => { + // Send the eager message if any + match &self.send_eager { + None => Ok(PublishAction::NothingToSend), + Some(send) => Ok(PublishAction::Send { + message: send.data.clone(), + metadata: Box::new(send.metadata.clone()), + }), + } + } + Some(metadata) => { + let peer_has = CellBitmap::::from_ssz_bytes(metadata.as_ref()) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if peer_has == self.partial_column.sidecar.cells_present_bitmap { + return Ok(PublishAction::SameMetadata); + } + + let Some(send) = self.partial_column.sidecar.with_missing_cells(&peer_has) else { + return Ok(PublishAction::NothingToSend); + }; + + let new_metadata = peer_has.union(&send.cells_present_bitmap); + Ok(PublishAction::Send { + message: send.as_ssz_bytes().to_vec(), + metadata: Box::new(CellBitmapMetadata::::from(new_metadata)), + }) + } + } + } +} diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 72f2873def9..a784eae2e9a 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,14 +1,17 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::TopicHash; +use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use crate::{Gossipsub, TopicHash}; +use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, - DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, + DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, @@ -26,6 +29,8 @@ pub enum PubsubMessage { BlobSidecar(Box<(u64, Arc>)>), /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), + /// Gossipsub message providing notification of a [`PartialDataColumnSidecar`] along with the subnet id where it was received. + PartialDataColumnSidecar(Box<(DataColumnSubnetId, PartialDataColumnSidecarMessage)>), // TODO(dknopik): review `Arc` situation /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a `SingleAttestation` with its subnet id. @@ -135,6 +140,9 @@ impl PubsubMessage { PubsubMessage::DataColumnSidecar(column_sidecar_data) => { GossipKind::DataColumnSidecar(column_sidecar_data.0) } + PubsubMessage::PartialDataColumnSidecar(partial_column_sidecar_data) => { + GossipKind::DataColumnSidecar(partial_column_sidecar_data.0) + } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) @@ -392,17 +400,49 @@ impl PubsubMessage { } } + pub fn decode_partial(topic: &TopicHash, group: &[u8], data: &[u8]) -> Result { + match GossipTopic::decode(topic.as_str()) { + Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)), + Ok(gossip_topic) => match gossip_topic.kind() { + GossipKind::DataColumnSidecar(id) => { + let block_root = Hash256::from_ssz_bytes(group) + .map_err(|e| format!("Error decoding group: {:?}", e))?; + let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) + .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; + let data_column = DanglingPartialDataColumn { + block_root, + // Partial messages are spec'd under the assumption that there is one column per subnet. + index: **id, + sidecar, + }; + Ok(Self::PartialDataColumnSidecar(Box::new(( + *id, + data_column.into(), + )))) + } + other => Err(format!("Partial message unsupported for topic: {other}")), + }, + } + } + /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If /// no encoding is known, and error is returned. - pub fn encode(&self, _encoding: GossipEncoding) -> Vec { + pub fn publish( + &self, + gossipsub: &mut Gossipsub, + topic: IdentTopic, + ) -> Result<(), PublishError> { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the // messages for us. - match &self { + let bytes = match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), + PubsubMessage::PartialDataColumnSidecar(data) => { + return gossipsub.publish_partial(topic, &data.1); + } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -413,7 +453,8 @@ impl PubsubMessage { PubsubMessage::BlsToExecutionChange(data) => data.as_ssz_bytes(), PubsubMessage::LightClientFinalityUpdate(data) => data.as_ssz_bytes(), PubsubMessage::LightClientOptimisticUpdate(data) => data.as_ssz_bytes(), - } + }; + gossipsub.publish(topic, bytes).map(|_| ()) } } @@ -438,6 +479,19 @@ impl std::fmt::Display for PubsubMessage { data.1.slot(), data.1.index, ), + PubsubMessage::PartialDataColumnSidecar(data) => write!( + f, + "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", + data.1.partial_column.block_root, + data.1.partial_column.block_root, + hex::encode( + data.1 + .partial_column + .sidecar + .cells_present_bitmap + .as_slice() + ), + ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {:?}, aggregator_index: {}", diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index b22adfbc487..fcc6bb9051d 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -178,6 +178,12 @@ pub enum GossipKind { LightClientOptimisticUpdate, } +impl GossipKind { + pub fn supports_partial_messages(&self) -> bool { + matches!(self, GossipKind::DataColumnSidecar(_)) + } +} + impl std::fmt::Display for GossipKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs index 56dccadaa94..154c5dbf6fe 100644 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ b/beacon_node/lighthouse_tracing/src/lib.rs @@ -12,6 +12,7 @@ pub const SPAN_PUBLISH_BLOCK: &str = "publish_block"; pub const SPAN_PENDING_COMPONENTS: &str = "pending_components"; /// Gossip methods root spans +pub const SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN: &str = "process_gossip_partial_data_column"; pub const SPAN_PROCESS_GOSSIP_DATA_COLUMN: &str = "process_gossip_data_column"; pub const SPAN_PROCESS_GOSSIP_BLOB: &str = "process_gossip_blob"; pub const SPAN_PROCESS_GOSSIP_BLOCK: &str = "process_gossip_block"; diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index 2a7fedb53e9..db2fd876287 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,6 +4,7 @@ pub mod service; mod metrics; mod nat; mod network_beacon_processor; +mod partial_data_column_cache; mod persisted_dht; mod router; mod status; diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index cea06a28c86..efaae60267f 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -143,6 +143,22 @@ pub static BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< "Total number of gossip data column sidecar verified for propagation.", ) }); +pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_int_counter( + "beacon_processor_gossip_partial_data_column_verified_total", + "Total number of gossip partial data column sidecar verified for propagation.", + ) +}); +pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_int_counter( + "beacon_processor_gossip_partial_data_column_cached_total", + "Total number of gossip partial data column sidecars received before their block.", + ) +}); // Gossip Exits. pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { @@ -560,6 +576,16 @@ pub static BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLo decimal_buckets(-3, -1), ) }); +pub static BEACON_PARTIAL_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_histogram_with_buckets( + "beacon_partial_data_column_gossip_propagation_verification_delay_time", + "Duration between when the partial data column sidecar is received over gossip and when it is verified for propagation.", + // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + decimal_buckets(-3, -1), + ) +}); pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( @@ -574,6 +600,20 @@ pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = + LazyLock::new(|| { + try_create_histogram_with_buckets( + "beacon_partial_data_column_gossip_slot_start_delay_time", + "Duration between when the partial data column sidecar is received over gossip and the start of the slot it belongs to.", + // Create a custom bucket list for greater granularity in block delay + Ok(vec![ + 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0, + 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, + ]), // NOTE: Previous values, which we may want to switch back to. + // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] + //decimal_buckets(-1,2) + ) + }); pub static BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: LazyLock> = LazyLock::new( || { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index fa6b5fd2434..f3554a45954 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -6,11 +6,12 @@ use crate::{ }; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::AsBlock; +use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, - GossipVerifiedBlock, NotifyExecutionLayer, + GossipVerifiedBlock, IntoExecutionPendingBlock, NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -20,9 +21,12 @@ use beacon_chain::{ validator_monitor::{get_block_delay_ms, get_slot_delay_ms}, }; use beacon_processor::{Work, WorkEvent}; -use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerId, ReportSource}; +use lighthouse_network::{ + Client, MessageAcceptance, MessageId, PeerAction, PeerId, PubsubMessage, ReportSource, +}; use lighthouse_tracing::{ SPAN_PROCESS_GOSSIP_BLOB, SPAN_PROCESS_GOSSIP_BLOCK, SPAN_PROCESS_GOSSIP_DATA_COLUMN, + SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, }; use logging::crit; use operation_pool::ReceivedPreCapella; @@ -51,6 +55,9 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; +use store::DatabaseBlock; +use types::das_column::DasColumn; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -768,6 +775,227 @@ impl NetworkBeaconProcessor { } } + #[instrument( + name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, + parent = None, + level = "debug", + skip_all, + fields(block_root = ?column_sidecar.block_root, index = column_sidecar.index), + )] + pub async fn process_gossip_dangling_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + _subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, + seen_duration: Duration, + ) { + let partial_column = match self + .chain + .store + .try_get_full_block(&column_sidecar.block_root) + { + Ok(Some(DatabaseBlock::Full(block))) => { + Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )) + } + Ok(Some(DatabaseBlock::Blinded(block))) => { + Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )) + } + Ok(None) => None, + Err(err) => { + warn!(?err, "Error getting block for partial data column"); + None + } + }; + + match partial_column { + Some(Ok(partial_column)) => { + debug!("Received partial while having block"); + self.process_gossip_verifiable_partial_data_column_sidecar( + peer_id, + Arc::new(partial_column), + seen_duration, + ) + .await; + } + Some(Err(err)) => { + warn!(?err, "Error creating verifiable partial data column"); + } + None => { + debug!("Received partial while not having block"); + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, + ); + self.partial_data_column_cache.lock().insert( + column_sidecar, + peer_id, + seen_duration, + ); + } + } + } + + #[instrument( + name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, + parent = None, + level = "debug", + skip_all, + fields(slot = %column_sidecar.slot(), block_root = ?column_sidecar.block_root(), index = column_sidecar.index()), + )] + pub async fn process_gossip_verifiable_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + column_sidecar: Arc>, + seen_duration: Duration, + ) { + let slot = column_sidecar.slot(); + let block_root = column_sidecar.block_root(); + let index = column_sidecar.index(); + let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); + // Log metrics to track delay from other nodes on the network. + metrics::observe_duration( + &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, + delay, + ); + match self + .chain + .verify_partial_data_column_sidecar_for_gossip(column_sidecar.clone()) + { + Ok(gossip_verified_data_column) => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL, + ); + + debug!( + %slot, + %block_root, + %index, + "Successfully verified gossip partial data column sidecar" + ); + + // TODO(dknopik): wait for joao's validation result impl + //self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); + + // Log metrics to keep track of propagation delay times. + if let Some(duration) = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|now| now.checked_sub(seen_duration)) + { + metrics::observe_duration( + &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, + duration, + ); + } + + self.process_gossip_verified_data_column( + peer_id, + gossip_verified_data_column, + seen_duration, + ) + .await + } + Err(err) => { + match err { + GossipDataColumnError::PriorKnownUnpublished => { + debug!( + %slot, + %block_root, + %index, + "Gossip data column already processed via the EL. Accepting the column sidecar without re-processing." + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Accept, + //); + } + // ParentUnknown can't really happen, so we treat it like an internal error + GossipDataColumnError::ParentUnknown { .. } + | GossipDataColumnError::PubkeyCacheTimeout + | GossipDataColumnError::BeaconChainError(_) => { + crit!( + error = ?err, + "Internal error when verifying column sidecar" + ) + } + GossipDataColumnError::ProposalSignatureInvalid + | GossipDataColumnError::UnknownValidator(_) + | GossipDataColumnError::ProposerIndexMismatch { .. } + | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::InvalidSubnetId { .. } + | GossipDataColumnError::InvalidInclusionProof + | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::UnexpectedDataColumn + | GossipDataColumnError::InvalidColumnIndex(_) + | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } + | GossipDataColumnError::InconsistentCommitmentsLength { .. } + | GossipDataColumnError::InconsistentProofsLength { .. } + | GossipDataColumnError::NotFinalizedDescendant { .. } => { + debug!( + error = ?err, + %slot, + %block_root, + %index, + "Could not verify column sidecar for gossip. Rejecting the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_data_column_low", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Reject, + //); + } + GossipDataColumnError::PriorKnown { .. } => { + // Data column is available via either the EL or reconstruction. + // Do not penalise the peer. + // Gossip filter should filter any duplicates received after this. + debug!( + %slot, + %block_root, + %index, + "Received already available column sidecar. Ignoring the column sidecar" + ) + } + GossipDataColumnError::FutureSlot { .. } + | GossipDataColumnError::PastFinalizedSlot { .. } => { + debug!( + error = ?err, + %slot, + %block_root, + %index, + "Could not verify column sidecar for gossip. Ignoring the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::HighToleranceError, + "gossip_data_column_high", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Ignore, + //); + } + } + } + } + } + #[allow(clippy::too_many_arguments)] #[instrument( name = SPAN_PROCESS_GOSSIP_BLOB, @@ -1014,10 +1242,10 @@ impl NetworkBeaconProcessor { } } - async fn process_gossip_verified_data_column( + async fn process_gossip_verified_data_column>( self: &Arc, peer_id: PeerId, - verified_data_column: GossipVerifiedDataColumn, + verified_data_column: GossipVerifiedDataColumn, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { @@ -1026,9 +1254,45 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); + let cloned_self = self.clone(); + let data_publish_fn = move |merged_data: MergedData| { + debug!( + partial = merged_data.updated_partials.len(), + full = merged_data.completed_columns.len(), + "Sending merged data" + ); + let messages: Vec<_> = merged_data + .updated_partials + .into_iter() + .map(|partial| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index( + partial.index(), + &cloned_self.chain.spec, + ), + partial.column.clone().into(), + ))) + }) + .chain(merged_data.completed_columns.into_iter().flat_map(|full| { + let subnet = + DataColumnSubnetId::from_column_index(full.index, &self.chain.spec); + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*full).clone().into_partial().column.clone().into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), + ] + })) + .collect(); + if !messages.is_empty() { + cloned_self.send_network_message(NetworkMessage::Publish { messages }) + } + }; + let result = self .chain - .process_gossip_data_columns(vec![verified_data_column], || Ok(())) + .process_gossip_data_columns(vec![verified_data_column], || Ok(()), data_publish_fn) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); @@ -1166,6 +1430,35 @@ impl NetworkBeaconProcessor { Span::current().record("block_root", block_root.to_string()); if let Some(handle) = duplicate_cache.check_and_insert(block_root) { + // First, get the partial columns that have been waiting. Do this first in order to gossip them quickly (i guess?) TODO(dknopik): check if good + let partials = { + let mut partial_cache = self.partial_data_column_cache.lock(); + let partials = partial_cache.get_for_block(block_root); + // Opportunistically clean the cache as we are holding the lock anyway. TODO(dknopik): do somewhere else + partial_cache.clean(); + partials + }; + + for cached_partial in partials { + // TODO(dknopik): Do this in parallel (queueing it maybe) + match VerifiablePartialDataColumn::from_dangling_and_block( + cached_partial.sidecar, + gossip_verified_block.block(), + ) { + Ok(partial_data_column) => { + self.process_gossip_verifiable_partial_data_column_sidecar( + cached_partial.peer_id, + Arc::new(partial_data_column), + cached_partial.seen_duration, + ) + .await; + } + Err(err) => { + warn!(?err, "Unable to process cached partial column") + } + } + } + self.process_gossip_verified_block( peer_id, gossip_verified_block, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7441e928712..3310b9c72ec 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1,3 +1,4 @@ +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::sync::manager::BlockProcessType; use crate::{service::NetworkMessage, sync::manager::SyncMessage}; use beacon_chain::blob_verification::{GossipBlobError, observe_gossip_blob}; @@ -11,6 +12,7 @@ use beacon_processor::{ BeaconProcessorSend, DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, }; +use futures::FutureExt; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, @@ -21,6 +23,7 @@ use lighthouse_network::{ Client, MessageId, NetworkGlobals, PeerId, PubsubMessage, rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, }; +use parking_lot::Mutex; use rand::prelude::SliceRandom; use std::path::PathBuf; use std::sync::Arc; @@ -28,6 +31,7 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; use tracing::{debug, error, instrument, trace, warn}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -55,6 +59,7 @@ pub enum InvalidBlockStorage { pub struct NetworkBeaconProcessor { pub beacon_processor_send: BeaconProcessorSend, pub duplicate_cache: DuplicateCache, + pub partial_data_column_cache: Mutex>, pub chain: Arc>, pub network_tx: mpsc::UnboundedSender>, pub sync_tx: mpsc::UnboundedSender>, @@ -249,6 +254,32 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for some partial data column sidecar. + pub fn send_gossip_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .process_gossip_dangling_partial_data_column_sidecar( + peer_id, + subnet_id, + column_sidecar, + seen_timestamp, + ) + .await + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipPartialDataColumnSidecar(Box::pin(process_fn)), + }) + } + /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( self: &Arc, @@ -762,6 +793,7 @@ impl NetworkBeaconProcessor { let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let custody_columns = self.chain.sampling_columns_for_epoch(epoch); let self_cloned = self.clone(); + let block_cloned = block.clone(); let publish_fn = move |blobs_or_data_column| { if publish_blobs { match blobs_or_data_column { @@ -772,10 +804,33 @@ impl NetworkBeaconProcessor { ); } EngineGetBlobsOutput::CustodyColumns(columns) => { + // Gradually publish any full columns self_cloned.publish_data_columns_gradually( - columns.into_iter().map(|c| c.clone_arc()).collect(), + columns + .iter() + .flat_map(|c| { + c.clone_arc() + .as_full(Some(&block_cloned)) + .map(|block| Arc::new(block.into_owned())) + }) + .collect(), block_root, ); + // "Publish" all columns as partial without eager send + self_cloned.send_network_message(NetworkMessage::Publish { + messages: columns + .into_iter() + .map(|c| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index( + c.index(), + &self_cloned.chain.spec, + ), + c.into_partial().into_inner().column.clone().into(), + ))) + }) + .collect(), + }) } }; } @@ -1036,6 +1091,7 @@ impl NetworkBeaconProcessor { } } +use types::das_column::DasColumn; #[cfg(test)] use { beacon_chain::builder::Witness, beacon_processor::BeaconProcessorChannels, @@ -1068,6 +1124,7 @@ impl NetworkBeaconProcessor> { let network_beacon_processor = Self { beacon_processor_send: beacon_processor_tx, duplicate_cache: DuplicateCache::default(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain, network_tx, sync_tx, diff --git a/beacon_node/network/src/partial_data_column_cache.rs b/beacon_node/network/src/partial_data_column_cache.rs new file mode 100644 index 00000000000..07f7cccabcc --- /dev/null +++ b/beacon_node/network/src/partial_data_column_cache.rs @@ -0,0 +1,76 @@ +use lighthouse_network::PeerId; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::{EthSpec, Hash256}; + +const BLOCK_LIMIT: usize = 16; +const SIDECAR_PER_BLOCK_LIMIT: usize = 16; +const EXPIRATION_TIME: Duration = Duration::from_secs(24); + +/// Really dumb hacky implementation of a cache for partial data column sidecars. +/// +/// The issue is that (with the current spec draft) we have to match up the dangling partial data +/// columns sidecars with the corresponding block. Of course, usually we have to take great care to +/// not be exploitable, but this cache design (for now) assumes no malicious behaviour. +/// +/// Do not take anything in this file as an actual implementation suggestion, it is just a hack +/// while we discuss the spec! +pub struct PartialDataColumnCache { + per_block: HashMap>>, +} + +pub struct CachedPartial { + pub sidecar: Arc>, + pub peer_id: PeerId, + pub seen_duration: Duration, +} + +impl PartialDataColumnCache { + pub fn new() -> Self { + Self { + per_block: HashMap::new(), + } + } + + pub fn insert( + &mut self, + sidecar: Arc>, + peer_id: PeerId, + seen_duration: Duration, + ) { + let len = self.per_block.len(); + let entry = self.per_block.entry(sidecar.block_root); + if matches!(entry, Entry::Vacant(_)) && len >= BLOCK_LIMIT { + return; + } + + let sidecars = entry.or_default(); + if sidecars.len() < SIDECAR_PER_BLOCK_LIMIT { + sidecars.push(CachedPartial { + sidecar, + peer_id, + seen_duration, + }); + } + } + + pub fn get_for_block(&mut self, block_root: Hash256) -> Vec> { + self.per_block.remove(&block_root).unwrap_or_default() + } + + pub fn clean(&mut self) { + if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { + self.per_block.retain(|_, sidecars| { + sidecars + .iter() + .map(|cached| cached.seen_duration) + .max() + .map(|last_seen| now.saturating_sub(last_seen) < EXPIRATION_TIME) + .unwrap_or(false) + }); + } + } +} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 60fe094bb7c..30ccb4319ae 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -6,6 +6,7 @@ #![allow(clippy::unit_arg)] use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::service::NetworkMessage; use crate::status::status_message; use crate::sync::SyncMessage; @@ -19,6 +20,7 @@ use lighthouse_network::{ }; use logging::TimeLatch; use logging::crit; +use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; @@ -97,6 +99,7 @@ impl Router { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send, duplicate_cache: DuplicateCache::default(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: beacon_chain.clone(), network_tx: network_send.clone(), sync_tx: sync_send.clone(), @@ -384,6 +387,18 @@ impl Router { ), ) } + PubsubMessage::PartialDataColumnSidecar(data) => { + let (subnet_id, column_sidecar) = *data; + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_partial_data_column_sidecar( + peer_id, + subnet_id, + column_sidecar.partial_column, + timestamp_now(), + ), + ) + } PubsubMessage::VoluntaryExit(exit) => { debug!(%peer_id, "Received a voluntary exit"); self.handle_beacon_processor_send_result( diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 4bd649ba824..dd9448983c6 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -534,7 +534,7 @@ impl NetworkService { let subnet_id = subnet_and_attestation.0; let attestation = &subnet_and_attestation.1; // checks if we have an aggregator for the slot. If so, we should process - // the attestation, else we just just propagate the Attestation. + // the attestation, else we just propagate the Attestation. let should_process = self.subnet_service.should_process_attestation( Subnet::Attestation(subnet_id), &attestation.data, diff --git a/consensus/types/src/das_column.rs b/consensus/types/src/das_column.rs new file mode 100644 index 00000000000..3e25576fdac --- /dev/null +++ b/consensus/types/src/das_column.rs @@ -0,0 +1,105 @@ +//! A PeerDAS data column. May or may not contain all cells. This is not necessarily implementable +//! for a beacon spec container, more as more data might be required, such as the partial message +//! group id alongside a partial message. + +use crate::beacon_block_body::KzgCommitments; +use crate::data_column_sidecar::{Cell, DataColumn}; +use crate::partial_data_column_sidecar::VerifiablePartialDataColumn; +use crate::{ + ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, + Slot, +}; +use kzg::{KzgCommitment, KzgProof}; +use ssz_types::VariableList; +use std::borrow::Cow; + +// TODO(dknopik): Name good? +// TODO(dknopik): Maybe move to unified cell storage? +// TODO(dknopik): Move generic parameter to associated type? +pub trait DasColumn: Clone { + fn slot(&self) -> Slot; + fn index(&self) -> ColumnIndex; + fn cell_count_total(&self) -> usize; + fn cells_present(&self) -> impl Iterator; + fn column(&self) -> &DataColumn; + fn kzg_proofs(&self) -> &VariableList; + fn kzg_commitments(&self) -> &KzgCommitments; + fn block_root(&self) -> Hash256; + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader>; + fn into_partial(self) -> VerifiablePartialDataColumn; + + /// Convert this column into a full data column (e.g. for gossip). Note that this is potentially + /// expensive. + fn as_full( + &self, + header: Option<&SignedBeaconBlock>, + ) -> Option>>; + + fn iter(&self) -> impl Iterator>>; + + fn compare>(&self, rhs: &C) -> ColumnComparison { + if self.slot() == rhs.slot() + && self.index() == rhs.index() + && self.block_root() == rhs.block_root() + { + return ColumnComparison::DifferentColumns; + } + + if self.cell_count_total() != rhs.cell_count_total() { + return ColumnComparison::DataConflict; + } + + let mut missing_in_rhs = vec![]; + let mut missing_in_lhs = vec![]; + for (index, (lhs, rhs)) in self.iter().zip(rhs.iter()).enumerate() { + match (lhs, rhs) { + (None, None) => {} + (Some(_), None) => missing_in_rhs.push(index), + (None, Some(_)) => missing_in_lhs.push(index), + (Some(lhs), Some(rhs)) => { + if lhs != rhs { + return ColumnComparison::DataConflict; + } + } + } + } + if missing_in_rhs.is_empty() && missing_in_lhs.is_empty() { + return ColumnComparison::Equal; + } + + ColumnComparison::MissingCells { + missing_in_lhs, + missing_in_rhs, + } + } +} + +pub enum ColumnComparison { + DifferentColumns, + DataConflict, + MissingCells { + missing_in_lhs: Vec, + missing_in_rhs: Vec, + }, + Equal, +} + +impl PartialEq> for VerifiablePartialDataColumn { + fn eq(&self, other: &DataColumnSidecar) -> bool { + // Slight optimisation: Can only be the same if `self` is fully present + self.column.sidecar.is_complete() + && self.slot() == other.slot() + && self.index() == other.index() + && self.block_root() == other.block_root() + && self.kzg_commitments() == other.kzg_commitments() + && self.column() == other.column() + && self.kzg_proofs() == other.kzg_proofs() + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct CellWithMetadata<'a, E: EthSpec> { + pub cell: &'a Cell, + pub proof: &'a KzgProof, + pub commitment: &'a KzgCommitment, +} diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index 2272b1695c9..725425c44e8 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,10 +1,14 @@ use crate::beacon_block_body::{BLOB_KZG_COMMITMENTS_INDEX, KzgCommitments}; -use crate::context_deserialize; +use crate::das_column::{CellWithMetadata, DasColumn}; +use crate::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, +}; use crate::test_utils::TestRandom; use crate::{ BeaconBlockHeader, BeaconStateError, Epoch, EthSpec, ForkName, Hash256, SignedBeaconBlockHeader, Slot, }; +use crate::{SignedBeaconBlock, context_deserialize}; use bls::Signature; use derivative::Derivative; use kzg::Error as KzgError; @@ -16,13 +20,14 @@ use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; +use std::borrow::Cow; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; pub type ColumnIndex = u64; -pub type Cell = FixedVector::BytesPerCell>; +pub type Cell = FixedVector::BytesPerCell>; // TODO(dknopik): Arc<[u8; E::BytesPerCell]> ??? cell level arcing acors the codebase seems reasonable pub type DataColumn = VariableList, ::MaxBlobCommitmentsPerBlock>; /// Identifies a set of data columns associated with a specific beacon block. @@ -169,3 +174,86 @@ impl From for DataColumnSidecarError { Self::SszError(e) } } + +impl DasColumn for DataColumnSidecar { + fn slot(&self) -> Slot { + self.slot() + } + + fn index(&self) -> ColumnIndex { + self.index + } + + fn cell_count_total(&self) -> usize { + self.column.len() + } + + fn cells_present(&self) -> impl Iterator { + 0..self.cell_count_total() + } + + fn column(&self) -> &DataColumn { + &self.column + } + + fn kzg_proofs(&self) -> &VariableList { + &self.kzg_proofs + } + + fn kzg_commitments(&self) -> &KzgCommitments { + &self.kzg_commitments + } + + fn block_root(&self) -> Hash256 { + self.block_root() + } + + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + Some(&self.signed_block_header) + } + + fn into_partial(self) -> VerifiablePartialDataColumn { + let mut bitmap = CellBitmap::::with_capacity(self.cell_count_total()) + .expect("our column has the same bound"); + for idx in 0..self.cell_count_total() { + bitmap + .set(idx, true) + .expect("The correct size is initialized right above"); + } + + VerifiablePartialDataColumn { + slot: self.slot(), + column: Arc::new(DanglingPartialDataColumn { + block_root: self.block_root(), + index: self.index(), + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: self.column, + kzg_proofs: self.kzg_proofs, + }, + }), + kzg_commitments: self.kzg_commitments, + } + } + + fn as_full( + &self, + _block: Option<&SignedBeaconBlock>, + ) -> Option>> { + Some(Cow::Borrowed(self)) + } + + fn iter(&self) -> impl Iterator>> { + self.column + .iter() + .zip(self.kzg_commitments.iter()) + .zip(self.kzg_proofs.iter()) + .map(|((cell, commitment), proof)| { + Some(CellWithMetadata { + cell, + commitment, + proof, + }) + }) + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 8e83fed1d9a..eb8726a6963 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -104,11 +104,13 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod das_column; pub mod data_column_custody_group; pub mod data_column_sidecar; pub mod data_column_subnet_id; pub mod light_client_header; pub mod non_zero_usize; +pub mod partial_data_column_sidecar; pub mod runtime_fixed_vector; pub mod runtime_var_list; diff --git a/consensus/types/src/partial_data_column_sidecar.rs b/consensus/types/src/partial_data_column_sidecar.rs new file mode 100644 index 00000000000..e0a87d6cf02 --- /dev/null +++ b/consensus/types/src/partial_data_column_sidecar.rs @@ -0,0 +1,364 @@ +use crate::beacon_block_body::KzgCommitments; +use crate::das_column::{CellWithMetadata, DasColumn}; +use crate::data_column_sidecar::{Cell, DataColumn}; +use crate::test_utils::TestRandom; +use crate::{AbstractExecPayload, ColumnIndex, DataColumnSidecar, context_deserialize}; +use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot}; +use derivative::Derivative; +use kzg::KzgProof; +use serde::{Deserialize, Serialize}; +use ssz::{BitList, Encode}; +use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; +use std::borrow::Cow; +use std::sync::Arc; +use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; + +pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; + +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive( + Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Derivative, +)] +#[serde(bound = "E: EthSpec")] +#[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +#[context_deserialize(ForkName)] +pub struct PartialDataColumnSidecar { + pub cells_present_bitmap: CellBitmap, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub column: DataColumn, + pub kzg_proofs: VariableList, +} + +impl PartialDataColumnSidecar { + pub fn min_size() -> usize { + // min size is one cell + Self { + cells_present_bitmap: BitList::with_capacity(1).unwrap(), + column: VariableList::new(vec![Cell::::default()]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn size(present_blobs: usize, block_blobs: usize) -> usize { + // min size is one cell + Self { + cells_present_bitmap: BitList::with_capacity(block_blobs).unwrap(), + column: VariableList::new(vec![Cell::::default(); present_blobs]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); present_blobs]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn max_size(max_blobs_per_block: usize) -> usize { + Self { + cells_present_bitmap: BitList::with_capacity(max_blobs_per_block).unwrap(), + column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn is_complete(&self) -> bool { + self.cells_present_bitmap.iter().all(|bit| bit) + } + + pub fn with_missing_cells(&self, bitmap: &CellBitmap) -> Option { + if self.cells_present_bitmap.len() != bitmap.len() { + return None; + } + self.clone_filter(|idx| !bitmap.get(idx).expect("Bounds checked above")) + } + + /// Creates a new partial data column sidecar containing only the blob indices for which the + /// passed closure returns `true` and were present in `self`. Will return `None` if there is no + /// overlap. + pub fn clone_filter(&self, filter: F) -> Option + where + F: Fn(usize) -> bool, + { + let mut new_bitmap = self.cells_present_bitmap.clone(); + let mut new_column = VariableList::default(); + let mut new_proofs = VariableList::default(); + let mut column_idx = 0; + + for (blob_idx, present) in self.cells_present_bitmap.iter().enumerate() { + if present { + if filter(blob_idx) { + // Keep this cell + let cell = self.column.get(column_idx)?; + new_column + .push(cell.clone()) + .expect("Has same capacity as existing column"); + let proof = self.kzg_proofs.get(column_idx)?; + new_proofs + .push(*proof) + .expect("Has same capacity as existing column"); + } else { + // Mark as not present + new_bitmap + .set(blob_idx, false) + .expect("Within bounds due to clone above"); + } + column_idx = column_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + + if new_column.is_empty() { + return None; + } + + Some(Self { + cells_present_bitmap: new_bitmap, + column: new_column, + kzg_proofs: new_proofs, + }) + } + + pub fn merge(&self, other: &Self) -> Option { + let new_bitmap = self.cells_present_bitmap.union(&other.cells_present_bitmap); + let mut new_column = VariableList::default(); + let mut new_proofs = VariableList::default(); + let mut self_cell_idx = 0usize; + let mut other_cell_idx = 0usize; + + for presence_bits in self + .cells_present_bitmap + .iter() + .zip(other.cells_present_bitmap.iter()) + { + match presence_bits { + (false, false) => {} + (true, other) => { + new_column + .push(self.column.get(self_cell_idx)?.clone()) + .expect("Has same capacity"); + new_proofs + .push(*self.kzg_proofs.get(self_cell_idx)?) + .expect("Has same capacity"); + self_cell_idx = self_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + if other { + other_cell_idx = other_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + (false, true) => { + new_column + .push(other.column.get(other_cell_idx)?.clone()) + .expect("Has same capacity"); + new_proofs + .push(*other.kzg_proofs.get(other_cell_idx)?) + .expect("Has same capacity"); + other_cell_idx = other_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + } + + Some(Self { + cells_present_bitmap: new_bitmap, + column: new_column, + kzg_proofs: new_proofs, + }) + } +} + +// TODO(dknopik): More specific error cases - e.g. internal inconsistency +pub struct MissingCellError; + +impl From> for PartialDataColumnSidecar { + fn from(value: DataColumnSidecar) -> Self { + // Create a bitmap with all cells marked as present + let mut cells_present_bitmap = BitList::with_capacity(value.column.len()) + .expect("Bitmap and cell list are both bounded by `MaxBlobCommitmentsPerBlock`"); + for idx in 0..value.column.len() { + cells_present_bitmap.set(idx, true).expect( + "Bitmap was created with column length, so we should be able to push a value", + ); + } + + Self { + cells_present_bitmap, + column: value.column, + kzg_proofs: value.kzg_proofs, + } + } +} + +// TODO(dknopik): Name? +#[derive(Debug, Clone, PartialEq)] +pub struct DanglingPartialDataColumn { + pub block_root: Hash256, + pub index: ColumnIndex, + pub sidecar: PartialDataColumnSidecar, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VerifiablePartialDataColumn { + pub column: Arc>, + pub kzg_commitments: KzgCommitments, + pub slot: Slot, +} + +// TODO(dknopik): Is there an existing approriate error type? +#[derive(Debug)] +pub enum PartialDataColumnMatchingError { + MismatchingBlock, + InvalidForkBlock, +} + +impl VerifiablePartialDataColumn { + pub fn from_dangling_and_block>( + column: Arc>, + block: &SignedBeaconBlock, + ) -> Result { + if column.block_root != block.canonical_root() { + return Err(PartialDataColumnMatchingError::MismatchingBlock); + } + + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_| PartialDataColumnMatchingError::InvalidForkBlock)? + .clone(); + + Ok(VerifiablePartialDataColumn { + column, + kzg_commitments, + slot: block.slot(), + }) + } + + pub fn clone_filter(&self, filter: F) -> Option + where + F: Fn(usize) -> bool, + { + Some(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + sidecar: self.column.sidecar.clone_filter(filter)?, + block_root: self.column.block_root, + index: self.column.index, + }), + kzg_commitments: self.kzg_commitments.clone(), + slot: self.slot, + }) + } +} + +impl DasColumn for VerifiablePartialDataColumn { + fn slot(&self) -> Slot { + self.slot + } + + fn index(&self) -> ColumnIndex { + self.column.index + } + + fn cell_count_total(&self) -> usize { + self.column.sidecar.cells_present_bitmap.len() + } + + fn cells_present(&self) -> impl Iterator { + self.column + .sidecar + .cells_present_bitmap + .iter() + .enumerate() + .filter_map(|(idx, bit)| bit.then_some(idx)) + } + + fn column(&self) -> &DataColumn { + &self.column.sidecar.column + } + + fn kzg_proofs(&self) -> &VariableList { + &self.column.sidecar.kzg_proofs + } + + fn kzg_commitments(&self) -> &KzgCommitments { + &self.kzg_commitments + } + + fn block_root(&self) -> Hash256 { + self.column.block_root.tree_hash_root() + } + + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + None + } + + fn into_partial(self) -> VerifiablePartialDataColumn { + self + } + + fn as_full( + &self, + block: Option<&SignedBeaconBlock>, + ) -> Option>> { + // we definitely require the block + let block = block?; + + // we need to have all columns + if !self.column.sidecar.is_complete() { + return None; + } + + // we need to have the correct amount of everything + let expected = self.column.sidecar.cells_present_bitmap.len(); + if self.column.sidecar.column.len() != expected + || self.column.sidecar.kzg_proofs.len() != expected + || self.kzg_commitments.len() != expected + { + return None; + } + + let (signed_block_header, kzg_commitments_inclusion_proof) = + block.signed_block_header_and_kzg_commitments_proof().ok()?; + Some(Cow::Owned(DataColumnSidecar { + kzg_commitments_inclusion_proof, + index: self.column.index, + column: self.column.sidecar.column.clone(), + kzg_commitments: self.kzg_commitments.clone(), + kzg_proofs: self.column.sidecar.kzg_proofs.clone(), + signed_block_header, + })) + } + + fn iter(&self) -> impl Iterator>> { + let sidecar = &self.column.sidecar; + let mut present_iterator = sidecar + .column + .iter() + .zip(self.kzg_commitments.iter()) + .zip(sidecar.kzg_proofs.iter()) + .map(|((cell, commitment), proof)| CellWithMetadata { + cell, + commitment, + proof, + }); + sidecar.cells_present_bitmap.iter().map(move |present| { + if present { + present_iterator.next() + } else { + None + } + }) + } +} diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 47b99023455..0163c6b779e 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -530,11 +530,12 @@ impl Tester { }) .collect(); - let result = self.block_on_dangerous( - self.harness - .chain - .process_gossip_data_columns(gossip_verified_data_columns, || Ok(())), - )?; + let result = + self.block_on_dangerous(self.harness.chain.process_gossip_data_columns( + gossip_verified_data_columns, + || Ok(()), + |_| (), + ))?; if valid { assert!(result.is_ok()); } From 303c6387e76a662634985f3d7bae13dcb0172ebc Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 14:21:27 +0100 Subject: [PATCH 002/146] fix compilation --- beacon_node/beacon_chain/src/kzg_utils.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 64183e7154a..49dbed69479 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -397,7 +397,8 @@ pub(crate) fn build_partial_data_columns( .get(col) .ok_or(format!("Missing blob cell at index {col}"))?; let cell: Vec = cell.to_vec(); - let cell = Cell::::from(cell); + let cell = + Cell::::try_from(cell).map_err(|e| format!("BytesPerCell exceeded: {e:?}"))?; let proof = blob_cell_proofs .get(col) @@ -410,33 +411,35 @@ pub(crate) fn build_partial_data_columns( .get_mut(col) .ok_or(format!("Missing data column proofs at index {col}"))?; - column.push(cell); + column.push(cell.clone()); column_proofs.push(*proof); } } - let sidecars: Vec>> = columns + let sidecars: Result>>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() .map(|(index, (col, proofs))| { - Arc::new(VerifiablePartialDataColumn { + Ok(Arc::new(VerifiablePartialDataColumn { column: Arc::new(DanglingPartialDataColumn { block_root: signed_block_header.message.canonical_root(), index: index as u64, sidecar: PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), - column: DataColumn::::from(col), - kzg_proofs: VariableList::from(proofs), + column: DataColumn::::try_from(col) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, + kzg_proofs: VariableList::try_from(proofs) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, }, }), kzg_commitments: kzg_commitments.clone(), slot: signed_block_header.message.slot, - }) + })) }) .collect(); - Ok(sidecars) + sidecars } /// Reconstruct blobs from a subset of data column sidecars (requires at least 50%). From ad850598caffa081c35b818e4719d25fb3210ad9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 14:41:11 +0100 Subject: [PATCH 003/146] fix some lints --- .../overflow_lru_cache.rs | 7 +++++-- .../beacon_chain/src/data_column_verification.rs | 3 +++ .../src/fetch_blobs/fetch_blobs_beacon_adapter.rs | 4 ++-- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 15 +++++---------- beacon_node/http_api/src/publish_blocks.rs | 6 ++++-- .../lighthouse_network/src/service/utils.rs | 2 +- .../network/src/network_beacon_processor/mod.rs | 1 - 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 5a5f0350a5d..5a3c8fb4423 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -645,7 +645,7 @@ impl DataAvailabilityCheckerInner { fn check_availability_and_cache_components( &self, block_root: Hash256, - pending_components: MappedRwLockReadGuard<'_, PendingComponents>, + pending_components: ComponentsLock<'_, T>, num_expected_columns_opt: Option, ) -> Result, AvailabilityCheckError> { if let Some(available_block) = pending_components.make_available( @@ -682,7 +682,7 @@ impl DataAvailabilityCheckerInner { block_root: Hash256, epoch: Epoch, update_fn: F, - ) -> Result<(MappedRwLockReadGuard<'_, PendingComponents>, R), AvailabilityCheckError> + ) -> Result<(ComponentsLock<'_, T>, R), AvailabilityCheckError> where F: FnOnce(&mut PendingComponents) -> Result, { @@ -887,6 +887,9 @@ impl DataAvailabilityCheckerInner { } } +type ComponentsLock<'a, T> = + MappedRwLockReadGuard<'a, PendingComponents<::EthSpec>>; + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index d8077f93854..b9e6d686426 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -325,6 +325,9 @@ impl } } +pub type GossipVerifiedFullDataColumn = + GossipVerifiedDataColumn::EthSpec>>; + /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. #[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 4e98e81df24..98de21b2fbd 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -1,7 +1,7 @@ use crate::fetch_blobs::{EngineGetBlobsOutput, FetchEngineBlobError}; use crate::observed_block_producers::ProposalKey; use crate::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes}; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use kzg::Kzg; #[cfg(test)] use mockall::automock; @@ -71,7 +71,7 @@ impl FetchBlobsBeaconAdapter { pub(crate) async fn get_blobs_v3( &self, versioned_hashes: Vec, - ) -> Result>>>, FetchEngineBlobError> { + ) -> Result>>, FetchEngineBlobError> { let execution_layer = self .chain .execution_layer diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 55f43289900..560887591f4 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -25,7 +25,7 @@ use crate::{ metrics, }; use execution_layer::Error as ExecutionLayerError; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use metrics::{TryExt, inc_counter}; #[cfg(test)] use mockall_double::double; @@ -38,8 +38,8 @@ use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ - BeaconStateError, Blob, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, KzgProofs, - SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, + BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, SignedBeaconBlock, + SignedBeaconBlockHeader, VersionedHash, }; /// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the @@ -265,11 +265,6 @@ async fn fetch_and_process_blobs_v2( return Ok(None); }; - let blobs_and_proofs: Vec<_> = blobs_and_proofs - .into_iter() - .map(|blob_and_proof| blob_and_proof.map(|BlobAndProofV2 { blob, proofs }| (blob, proofs))) - .collect(); - let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); @@ -326,7 +321,7 @@ async fn compute_custody_columns_to_import( chain_adapter: &Arc>, block_root: Hash256, block: Arc>>, - blobs_and_proofs: Vec, KzgProofs)>>, + blobs_and_proofs: Vec>, custody_columns_indices: &[ColumnIndex], ) -> Result< Vec>>, @@ -352,7 +347,7 @@ async fn compute_custody_columns_to_import( .map(|option| { option .as_ref() - .map(|(blob, proofs)| (blob, proofs.as_ref())) + .map(|BlobAndProofV2 { blob, proofs }| (blob, proofs.as_ref())) }) .collect::>(); let data_columns_result = diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index a1412debc28..2c97aba5e70 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,7 +3,9 @@ use std::future::Future; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; -use beacon_chain::data_column_verification::GossipVerifiedDataColumn; +use beacon_chain::data_column_verification::{ + GossipVerifiedDataColumn, GossipVerifiedFullDataColumn, +}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, @@ -407,7 +409,7 @@ fn build_data_columns( block: &SignedBeaconBlock>, blobs: BlobsList, kzg_cell_proofs: KzgProofs, -) -> Result>>, Rejection> { +) -> Result>, Rejection> { let slot = block.slot(); let data_column_sidecars = build_blob_data_column_sidecars(chain, block, blobs, kzg_cell_proofs).map_err(|e| { diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index a0026837e37..63f22be5e2c 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -41,7 +41,7 @@ pub fn build_transport( quic_support: bool, ) -> std::io::Result { // mplex config - let mut mplex_config = libp2p_mplex::MplexConfig::new(); + let mut mplex_config = libp2p_mplex::Config::new(); mplex_config.set_max_buffer_size(256); mplex_config.set_max_buffer_behaviour(libp2p_mplex::MaxBufferBehaviour::Block); diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 3310b9c72ec..7bb4884c851 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -12,7 +12,6 @@ use beacon_processor::{ BeaconProcessorSend, DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, }; -use futures::FutureExt; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, From 7b8cb0d99a9c22a9d2de9a81eeb257705a869681 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:05:28 +0100 Subject: [PATCH 004/146] impl get_blobs_v2 fallback --- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 15 ++++++++- .../beacon_chain/src/fetch_blobs/mod.rs | 31 +++++++++++++------ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 98de21b2fbd..e95cc9dfe13 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -51,7 +51,6 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } - // TODO(dknopik): fall back to this pub(crate) async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -138,4 +137,18 @@ impl FetchBlobsBeaconAdapter { .fork_choice_read_lock() .contains_block(block_root) } + + pub(crate) async fn supports_get_blobs_v3(&self) -> Result { + let execution_layer = self + .chain + .execution_layer + .as_ref() + .ok_or(FetchEngineBlobError::ExecutionLayerMissing)?; + + execution_layer + .get_engine_capabilities(None) + .await + .map_err(FetchEngineBlobError::RequestFailed) + .map(|caps| caps.get_blobs_v3) + } } diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 560887591f4..fb70cbc8e89 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -124,7 +124,7 @@ async fn fetch_and_process_engine_blobs_inner( .spec() .is_peer_das_enabled_for_epoch(block.epoch()) { - fetch_and_process_blobs_v2( + fetch_and_process_blobs_v2_or_v3( chain_adapter, block_root, block, @@ -239,7 +239,7 @@ async fn fetch_and_process_blobs_v1( } #[instrument(skip_all, level = "debug")] -async fn fetch_and_process_blobs_v2( +async fn fetch_and_process_blobs_v2_or_v3( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, block: Arc>, @@ -250,14 +250,25 @@ async fn fetch_and_process_blobs_v2( let num_expected_blobs = versioned_hashes.len(); metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); - // TODO(dknopik): implement fallback to get_blobs_v2 - debug!(num_expected_blobs, "Fetching blobs from the EL"); - let response = chain_adapter - .get_blobs_v3(versioned_hashes) - .await - .inspect_err(|_| { - inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); - })?; + + let response = if chain_adapter.supports_get_blobs_v3().await? { + debug!(num_expected_blobs, "Fetching available blobs from the EL"); + chain_adapter + .get_blobs_v3(versioned_hashes) + .await + .inspect_err(|_| { + inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); + })? + } else { + debug!(num_expected_blobs, "Fetching all blobs from the EL"); + chain_adapter + .get_blobs_v2(versioned_hashes) + .await + .inspect_err(|_| { + inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); + })? + .map(|vec| vec.into_iter().map(Some).collect()) + }; let Some(blobs_and_proofs) = response else { debug!(num_expected_blobs, "No blobs fetched from the EL"); From efa9366d4e3d4c3339455643693df81f676ffc70 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:46:47 +0100 Subject: [PATCH 005/146] update gossipsub --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e638a561ab8..64430c3cf72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "pin-project", From 56f7a1f8bb3778ebeaedfb904c1c5918f356e820 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:48:07 +0100 Subject: [PATCH 006/146] remove redundant log --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index fb70cbc8e89..ea77b9eb9d4 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -115,11 +115,6 @@ async fn fetch_and_process_engine_blobs_inner( return Ok(None); }; - debug!( - num_expected_blobs = versioned_hashes.len(), - "Fetching blobs from the EL" - ); - if chain_adapter .spec() .is_peer_das_enabled_for_epoch(block.epoch()) From 2a721d72a75d0cc2aed5d10516f580e6e67b86f0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 16:00:09 +0100 Subject: [PATCH 007/146] try to upgrade column using block immediately --- .../overflow_lru_cache.rs | 4 ++-- .../src/data_column_verification.rs | 18 +++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 5a3c8fb4423..e1cc6eac7c1 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -217,7 +217,7 @@ impl PendingComponents { continue; } - if let Some(full) = data_column.try_as_full() { + if let Some(full) = data_column.try_as_full(self.block.as_ref().map(|b| b.as_block())) { self.verified_partial_columns .retain(|col| col.index() != full.index()); self.verified_data_columns.push(full); @@ -234,7 +234,7 @@ impl PendingComponents { let did_merge = cached_partial.merge(&partial); if did_merge { if let Some(block) = &self.block - && let Some(full) = cached_partial.try_upgrade_full(block.as_block()) + && let Some(full) = cached_partial.try_as_full(Some(block.as_block())) { merged_data.completed_columns.push(full.clone_arc()); self.verified_data_columns.push(full); diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index b9e6d686426..643916e3210 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -453,23 +453,11 @@ impl> KzgVerifiedCustodyDataColumn { self.data.index() } - // TODO(dknopik): below three functions are baaaad. They clone in far too many cases + // TODO(dknopik): below two functions are baaaad. They clone in far too many cases - pub fn try_as_full(&self) -> Option>> { + pub fn try_as_full(&self, block: Option<&SignedBeaconBlock>) -> Option>> { self.data - .as_full(None) - .map(|full| KzgVerifiedCustodyDataColumn { - data: Arc::new(full.into_owned()), - _phantom: PhantomData, - }) - } - - pub fn try_upgrade_full( - &self, - block: &SignedBeaconBlock, - ) -> Option>> { - self.data - .as_full(Some(block)) + .as_full(block) .map(|full| KzgVerifiedCustodyDataColumn { data: Arc::new(full.into_owned()), _phantom: PhantomData, From 57094088ace561caf90ddb56467f0ed1c57ca506 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 16:31:58 +0100 Subject: [PATCH 008/146] publish data after block availability --- beacon_node/beacon_chain/src/beacon_chain.rs | 10 +++- .../src/data_availability_checker.rs | 13 +++-- .../overflow_lru_cache.rs | 51 ++++++++++--------- .../src/data_column_verification.rs | 5 +- beacon_node/beacon_chain/src/test_utils.rs | 2 + beacon_node/http_api/src/publish_blocks.rs | 39 ++++++++++++++ .../gossip_methods.rs | 1 + .../network_beacon_processor/sync_methods.rs | 1 + beacon_node/network/src/sync/tests/lookups.rs | 2 +- beacon_node/network/src/sync/tests/range.rs | 1 + testing/ef_tests/src/cases/fork_choice.rs | 2 + 11 files changed, 93 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 0d6d0e30a2c..11694acb24c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2929,6 +2929,7 @@ impl BeaconChain { notify_execution_layer, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await { @@ -3360,6 +3361,7 @@ impl BeaconChain { notify_execution_layer: NotifyExecutionLayer, block_source: BlockImportSource, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let block_slot = unverified_block.block().slot(); @@ -3429,7 +3431,8 @@ impl BeaconChain { self.import_available_block(Box::new(block)).await } ExecutedBlock::AvailabilityPending(block) => { - self.check_block_availability_and_import(block).await + self.check_block_availability_and_import(block, data_publish_fn) + .await } } }; @@ -3539,9 +3542,12 @@ impl BeaconChain { async fn check_block_availability_and_import( self: &Arc, block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let slot = block.block.slot(); - let availability = self.data_availability_checker.put_executed_block(block)?; + let availability = self + .data_availability_checker + .put_executed_block(block, data_publish_fn)?; self.process_availability(slot, availability, || Ok(())) .await } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9c8172ebb0c..b43e0826f7b 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -339,7 +339,7 @@ impl DataAvailabilityChecker { block_root: Hash256, slot: Slot, data_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = slot.epoch(T::EthSpec::slots_per_epoch()); let sampling_columns = self @@ -354,7 +354,7 @@ impl DataAvailabilityChecker { self.availability_cache.put_kzg_verified_data_columns( block_root, custody_columns, - publish_fn, + data_publish_fn, ) } @@ -366,12 +366,12 @@ impl DataAvailabilityChecker { &self, block_root: Hash256, custody_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { self.availability_cache.put_kzg_verified_data_columns( block_root, custody_columns, - publish_fn, + data_publish_fn, ) } @@ -380,8 +380,10 @@ impl DataAvailabilityChecker { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache.put_executed_block(executed_block) + self.availability_cache + .put_executed_block(executed_block, data_publish_fn) } /// Inserts a pre-execution block into the cache. @@ -901,6 +903,7 @@ impl MaybeAvailableBlock { } } +#[must_use = "Publish the data within"] pub struct MergedData { pub completed_columns: Vec>>, pub updated_partials: Vec>>, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index e1cc6eac7c1..ea7e6295b6f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -13,6 +13,7 @@ use lighthouse_tracing::SPAN_PENDING_COMPONENTS; use lru::LruCache; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::cmp::Ordering; +use std::mem; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{Span, debug, debug_span}; @@ -202,13 +203,11 @@ impl PendingComponents { } /// Merges a given set of data columns into the cache. - fn merge_data_columns< + fn merge_data_columns(&mut self, kzg_verified_data_columns: I) -> MergedData + where I: IntoIterator>, C: DasColumn, - >( - &mut self, - kzg_verified_data_columns: I, - ) -> Result, AvailabilityCheckError> { + { let mut merged_data = MergedData::empty(); for data_column in kzg_verified_data_columns { @@ -251,16 +250,18 @@ impl PendingComponents { } } - Ok(merged_data) + merged_data } /// Inserts a new block and revalidates the existing blobs against it. /// /// Blobs that don't match the new block's commitments are evicted. - pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) { + pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) -> MergedData { self.insert_executed_block(block); let reinsert = self.get_cached_blobs_mut().take(); self.merge_blobs(reinsert); + let reinsert = mem::take(&mut self.verified_partial_columns); + self.merge_data_columns(reinsert) } /// Returns Some if the block has received all its required data for import. The return value @@ -578,7 +579,6 @@ impl DataAvailabilityCheckerInner { let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_blobs(fixed_blobs); - Ok(()) })?; pending_components.span.in_scope(|| { @@ -600,7 +600,7 @@ impl DataAvailabilityCheckerInner { &self, block_root: Hash256, kzg_verified_data_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); let Some(epoch) = kzg_verified_data_columns.peek().map(|verified_blob| { @@ -621,7 +621,7 @@ impl DataAvailabilityCheckerInner { pending_components.merge_data_columns(kzg_verified_data_columns) })?; - publish_fn(merged_data); + data_publish_fn(merged_data); let num_expected_columns = self .custody_context @@ -684,7 +684,7 @@ impl DataAvailabilityCheckerInner { update_fn: F, ) -> Result<(ComponentsLock<'_, T>, R), AvailabilityCheckError> where - F: FnOnce(&mut PendingComponents) -> Result, + F: FnOnce(&mut PendingComponents) -> R, { let mut write_lock = self.critical.write(); @@ -692,7 +692,7 @@ impl DataAvailabilityCheckerInner { let pending_components = write_lock.get_or_insert_mut(block_root, || { PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) }); - update_fn(pending_components)? + update_fn(pending_components) }; RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { @@ -774,7 +774,6 @@ impl DataAvailabilityCheckerInner { let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.insert_pre_execution_block(block, source); - Ok(()) })?; let num_expected_columns_opt = self.get_num_expected_columns(epoch); @@ -804,6 +803,7 @@ impl DataAvailabilityCheckerInner { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = executed_block.as_block().epoch(); let block_root = executed_block.import_data.block_root; @@ -813,12 +813,13 @@ impl DataAvailabilityCheckerInner { .state_cache .register_pending_executed_block(executed_block); - let (pending_components, ()) = + let (pending_components, merged_data) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { - pending_components.merge_block(diet_executed_block); - Ok(()) + pending_components.merge_block(diet_executed_block) })?; + data_publish_fn(merged_data); + let num_expected_columns_opt = self.get_num_expected_columns(epoch); pending_components.span.in_scope(|| { @@ -1125,7 +1126,7 @@ mod test { ); assert!(cache.critical.read().is_empty(), "cache should be empty"); let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); if blobs_expected == 0 { assert!( @@ -1192,7 +1193,7 @@ mod test { ); } let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); assert!( matches!(availability, Availability::Available(_)), @@ -1254,7 +1255,7 @@ mod test { // put the block in the cache let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); // grab the diet block from the cache for later testing @@ -1449,7 +1450,7 @@ mod pending_components_tests { setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); @@ -1464,7 +1465,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(blobs); assert_cache_consistent(cache, max_len); @@ -1480,7 +1481,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); assert_empty_blob_cache(cache); } @@ -1493,7 +1494,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); @@ -1509,7 +1510,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); assert_cache_consistent(cache, max_len); @@ -1525,7 +1526,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); assert_cache_consistent(cache, max_len); } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 643916e3210..cdfd6957539 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -455,7 +455,10 @@ impl> KzgVerifiedCustodyDataColumn { // TODO(dknopik): below two functions are baaaad. They clone in far too many cases - pub fn try_as_full(&self, block: Option<&SignedBeaconBlock>) -> Option>> { + pub fn try_as_full( + &self, + block: Option<&SignedBeaconBlock>, + ) -> Option>> { self.data .as_full(block) .map(|full| KzgVerifiedCustodyDataColumn { diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4e1d0002087..58e0a62680a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2374,6 +2374,7 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await? .try_into() @@ -2398,6 +2399,7 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await? .try_into() diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 2c97aba5e70..c5f9b133696 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,6 +3,7 @@ use std::future::Future; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{ GossipVerifiedDataColumn, GossipVerifiedFullDataColumn, }; @@ -29,6 +30,7 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tracing::{Span, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; +use types::das_column::DasColumn; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, @@ -201,6 +203,41 @@ pub async fn publish_block>( Ok(()) }; + let sender_clone = network_tx.clone(); + let spec = chain.spec.clone(); + let data_publish_fn = move |merged_data: MergedData| { + debug!( + partial = merged_data.updated_partials.len(), + full = merged_data.completed_columns.len(), + "Sending merged data after block availability" + ); + let messages: Vec<_> = merged_data + .updated_partials + .into_iter() + .map(|partial| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index(partial.index(), &spec), + partial.column.clone().into(), + ))) + }) + .chain(merged_data.completed_columns.into_iter().flat_map(|full| { + let subnet = DataColumnSubnetId::from_column_index(full.index, &spec); + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*full).clone().into_partial().column.clone().into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), + ] + })) + .collect(); + if !messages.is_empty() + && let Err(err) = crate::publish_pubsub_messages(&sender_clone, messages) + { + warn!(?err, "Publishing data after block availability failed") + } + }; + // Wait for blobs/columns to get gossip verified before proceeding further as we need them for import. let (gossip_verified_blobs, gossip_verified_columns) = build_sidecar_task_handle.await?; @@ -275,6 +312,7 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, + data_publish_fn, )) .await; post_block_import_logging_and_response( @@ -325,6 +363,7 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, + data_publish_fn, )) .await; post_block_import_logging_and_response( diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index be6fb20f785..500842ff149 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1800,6 +1800,7 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Gossip, || Ok(()), + |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "block"); diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 41b12fa01b3..a7cff747230 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -178,6 +178,7 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Rpc, "block"); diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index fc641861754..cb8028c5874 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1079,7 +1079,7 @@ impl TestRig { .harness .chain .data_availability_checker - .put_executed_block(executed_block) + .put_executed_block(executed_block, |_| ()) .unwrap() { Availability::Available(_) => panic!("block removed from da_checker, available"), diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index cb728a90c1b..8b9c876907c 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -431,6 +431,7 @@ impl TestRig { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap() diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 0163c6b779e..f394459439d 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -549,6 +549,7 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = data_column_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); @@ -639,6 +640,7 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = blob_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); From 02194da47f88890ad7394438941231159c335ae6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 17:45:02 +0100 Subject: [PATCH 009/146] fix remaining lints and release test compilation --- beacon_node/beacon_chain/tests/blob_verification.rs | 1 + beacon_node/beacon_chain/tests/block_verification.rs | 9 ++++++++- beacon_node/beacon_chain/tests/column_verification.rs | 1 + beacon_node/beacon_chain/tests/payload_invalidation.rs | 4 ++++ beacon_node/beacon_chain/tests/store_tests.rs | 3 +++ beacon_node/beacon_chain/tests/tests.rs | 1 + .../lighthouse_network/src/service/gossip_cache.rs | 4 ++++ .../network/src/network_beacon_processor/tests.rs | 10 +++++++++- 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/tests/blob_verification.rs b/beacon_node/beacon_chain/tests/blob_verification.rs index c42a2828c01..a6a1aeef519 100644 --- a/beacon_node/beacon_chain/tests/blob_verification.rs +++ b/beacon_node/beacon_chain/tests/blob_verification.rs @@ -86,6 +86,7 @@ async fn rpc_blobs_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 3d1fa8f4af4..508a7105102 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -504,6 +504,7 @@ async fn assert_invalid_signature( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; assert!( @@ -575,6 +576,7 @@ async fn invalid_signature_gossip_block() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; assert!( @@ -1018,6 +1020,7 @@ async fn block_gossip_verification() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect("should import valid gossip verified block"); @@ -1327,7 +1330,7 @@ async fn verify_and_process_gossip_data_sidecars( harness .chain - .process_gossip_data_columns(gossip_verified, || Ok(())) + .process_gossip_data_columns(gossip_verified, || Ok(()), |_| ()) .await .expect("should import valid gossip verified columns"); } @@ -1377,6 +1380,7 @@ async fn verify_block_for_gossip_slashing_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -1413,6 +1417,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -1577,6 +1582,7 @@ async fn add_base_block_to_altair_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect_err("should error when processing base block"), @@ -1714,6 +1720,7 @@ async fn add_altair_block_to_base_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect_err("should error when processing altair block"), diff --git a/beacon_node/beacon_chain/tests/column_verification.rs b/beacon_node/beacon_chain/tests/column_verification.rs index 229ae1e1998..491969fc663 100644 --- a/beacon_node/beacon_chain/tests/column_verification.rs +++ b/beacon_node/beacon_chain/tests/column_verification.rs @@ -90,6 +90,7 @@ async fn rpc_columns_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 5bd43835e33..81da193574c 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -695,6 +695,7 @@ async fn invalidates_all_descendants() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap() @@ -797,6 +798,7 @@ async fn switches_heads() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap() @@ -1063,6 +1065,7 @@ async fn invalid_parent() { assert!(matches!( rig.harness.chain.process_block(rpc_block.block_root(), rpc_block, NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ).await, Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root }) if invalid_root == parent_root @@ -1393,6 +1396,7 @@ async fn recover_from_invalid_head_by_importing_blocks() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 0a261e36cef..04efa34f83c 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2937,6 +2937,7 @@ async fn weak_subjectivity_sync_test( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -3502,6 +3503,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap_err(); @@ -3517,6 +3519,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index ec0e607d00a..02790a02eb6 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -712,6 +712,7 @@ async fn run_skip_slot_test(skip_slots: u64) { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 120b9e6c245..f72bf3b8a52 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -13,6 +13,8 @@ use tokio_util::time::delay_queue::{DelayQueue, Key}; /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. +// TODO(dknopik) +#[allow(dead_code)] pub struct GossipCache { /// Expire timeouts for each topic-msg pair. expirations: DelayQueue<(GossipTopic, Vec)>, @@ -196,6 +198,8 @@ impl GossipCache { } // Insert a message to be sent later. + // TODO(dknopik) + #[allow(dead_code)] pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index a9794cb5c42..d1f8147e7b0 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -1,6 +1,7 @@ #![cfg(not(debug_assertions))] // Tests are too slow in debug. #![cfg(test)] +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::{ network_beacon_processor::{ ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor, @@ -32,6 +33,7 @@ use lighthouse_network::{ types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}, }; use matches::assert_matches; +use parking_lot::Mutex; use slot_clock::SlotClock; use std::collections::HashSet; use std::iter::Iterator; @@ -265,6 +267,7 @@ impl TestRig { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send: beacon_processor_tx.clone(), duplicate_cache: duplicate_cache.clone(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: harness.chain.clone(), network_tx, sync_tx, @@ -1134,7 +1137,12 @@ async fn accept_processed_gossip_data_columns_without_import() { let block_root = rig.next_block.canonical_root(); rig.chain .data_availability_checker - .put_gossip_verified_data_columns(block_root, rig.next_block.slot(), verified_data_columns) + .put_gossip_verified_data_columns( + block_root, + rig.next_block.slot(), + verified_data_columns, + |_| (), + ) .expect("should put data columns into availability cache"); // WHEN an already processed but unobserved data column is received via gossip From bd8000d1d289819f61a107e1cc0a63c2054469c2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 01:48:07 +0100 Subject: [PATCH 010/146] implement caching of partial messages --- .../src/service/gossip_cache.rs | 80 +++++++++++-------- .../lighthouse_network/src/service/mod.rs | 74 ++++++++++++----- .../lighthouse_network/src/types/mod.rs | 2 +- .../lighthouse_network/src/types/pubsub.rs | 34 +++++--- 4 files changed, 126 insertions(+), 64 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index f72bf3b8a52..5aa655def96 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -1,25 +1,27 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::marker::PhantomData; use std::pin::Pin; use std::task::{Context, Poll}; use std::time::Duration; use crate::GossipTopic; -use crate::types::GossipKind; +use crate::types::{EncodedPubsubMessage, GossipKind}; use tokio_util::time::delay_queue::{DelayQueue, Key}; +use types::EthSpec; + +type TopicMsg = (Key, EncodedPubsubMessage); /// Store of gossip messages that we failed to publish and will try again later. By default, all /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. -// TODO(dknopik) -#[allow(dead_code)] -pub struct GossipCache { - /// Expire timeouts for each topic-msg pair. +pub struct GossipCache { + /// Expire timeouts for each topic-msgid triple. expirations: DelayQueue<(GossipTopic, Vec)>, /// Messages cached for each topic. - topic_msgs: HashMap, Key>>, + topic_msgs: HashMap, TopicMsg>>, /// Timeout for blocks. beacon_block: Option, /// Timeout for blobs. @@ -49,7 +51,7 @@ pub struct GossipCache { } #[derive(Default)] -pub struct GossipCacheBuilder { +pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, @@ -77,10 +79,11 @@ pub struct GossipCacheBuilder { light_client_finality_update: Option, /// Timeout for light client optimistic updates. light_client_optimistic_update: Option, + _phantom: PhantomData, } #[allow(dead_code)] -impl GossipCacheBuilder { +impl GossipCacheBuilder { /// By default, all timeouts all disabled. Setting a default timeout will enable all timeout /// that are not already set. pub fn default_timeout(mut self, timeout: Duration) -> Self { @@ -153,7 +156,7 @@ impl GossipCacheBuilder { self } - pub fn build(self) -> GossipCache { + pub fn build(self) -> GossipCache { let GossipCacheBuilder { default_timeout, beacon_block, @@ -169,6 +172,7 @@ impl GossipCacheBuilder { bls_to_execution_change, light_client_finality_update, light_client_optimistic_update, + .. } = self; GossipCache { expirations: DelayQueue::default(), @@ -190,17 +194,15 @@ impl GossipCacheBuilder { } } -impl GossipCache { +impl GossipCache { /// Get a builder of a `GossipCache`. Topic kinds for which no timeout is defined will be /// ignored if added in `insert`. - pub fn builder() -> GossipCacheBuilder { + pub fn builder() -> GossipCacheBuilder { GossipCacheBuilder::default() } // Insert a message to be sent later. - // TODO(dknopik) - #[allow(dead_code)] - pub fn insert(&mut self, topic: GossipTopic, data: Vec) { + pub fn insert(&mut self, topic: GossipTopic, id: Vec, data: EncodedPubsubMessage) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, @@ -223,45 +225,54 @@ impl GossipCache { .topic_msgs .entry(topic.clone()) .or_default() - .entry(data.clone()) + .entry(id.clone()) { - Entry::Occupied(key) => self.expirations.reset(key.get(), expire_timeout), + Entry::Occupied(key) => self.expirations.reset(&key.get().0, expire_timeout), Entry::Vacant(entry) => { - let key = self.expirations.insert((topic, data), expire_timeout); - entry.insert(key); + let key = self.expirations.insert((topic, id), expire_timeout); + entry.insert((key, data)); } } } - // Get the registered messages for this topic. - pub fn retrieve(&mut self, topic: &GossipTopic) -> Option> + '_> { - if let Some(msgs) = self.topic_msgs.remove(topic) { - for (_, key) in msgs.iter() { - self.expirations.remove(key); - } - Some(msgs.into_keys()) - } else { - None - } + // Get the registered messages for this topic, removing them + pub fn retrieve_all( + &mut self, + topic: &GossipTopic, + ) -> Option> + '_> { + self.topic_msgs.remove(topic).map(|msgs| { + msgs.into_values().map(|(key, msg)| { + self.expirations.remove(&key); + msg + }) + }) + } + + // Get a registered message, NOT removing it. + pub fn retrieve_one(&self, topic: &GossipTopic, id: &[u8]) -> Option<&EncodedPubsubMessage> { + self.topic_msgs + .get(topic) + .and_then(|msgs| msgs.get(id)) + .map(|(_, msg)| msg) } } -impl futures::stream::Stream for GossipCache { +impl futures::stream::Stream for GossipCache { type Item = Result; // We don't care to retrieve the expired data. fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.expirations.poll_expired(cx) { Poll::Ready(Some(expired)) => { let expected_key = expired.key(); - let (topic, data) = expired.into_inner(); + let (topic, id) = expired.into_inner(); let topic_msg = self.topic_msgs.get_mut(&topic); debug_assert!( topic_msg.is_some(), "Topic for registered message is not present." ); if let Some(msgs) = topic_msg { - let key = msgs.remove(&data); - debug_assert_eq!(key, Some(expected_key)); + let key_and_data = msgs.remove(&id); + debug_assert_eq!(key_and_data.map(|(key, _)| key), Some(expected_key)); if msgs.is_empty() { // no more messages for this topic. self.topic_msgs.remove(&topic); @@ -279,10 +290,11 @@ impl futures::stream::Stream for GossipCache { mod tests { use super::*; use futures::stream::StreamExt; + use types::MainnetEthSpec; #[tokio::test] async fn test_stream() { - let mut cache = GossipCache::builder() + let mut cache = GossipCache::::builder() .default_timeout(Duration::from_millis(300)) .build(); let test_topic = GossipTopic::new( @@ -290,7 +302,7 @@ mod tests { crate::types::GossipEncoding::SSZSnappy, [0u8; 4], ); - cache.insert(test_topic, vec![]); + cache.insert(test_topic, vec![], EncodedPubsubMessage::Full(vec![])); tokio::time::sleep(Duration::from_millis(300)).await; while cache.next().await.is_some() {} assert!(cache.expirations.is_empty()); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 64120b38f2a..2dd7ebbe5f3 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -15,15 +15,16 @@ use crate::rpc::{ RequestType, ResponseTermination, RpcResponse, RpcSuccessResponse, }; use crate::types::{ - GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, SubnetDiscovery, - all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, subnet_from_topic_hash, + EncodedPubsubMessage, GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, + SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, + subnet_from_topic_hash, }; use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, - TopicScoreParams, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, Partial, + PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; use libp2p::multiaddr::{self, Multiaddr, Protocol as MProtocol}; @@ -33,6 +34,7 @@ use libp2p::upnp::tokio::Behaviour as Upnp; use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; +use sha2::{Digest, Sha256}; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; use std::pin::Pin; @@ -160,7 +162,7 @@ pub struct Network { score_settings: PeerScoreSettings, /// The interval for updating gossipsub scores update_gossipsub_scores: tokio::time::Interval, - gossip_cache: GossipCache, + gossip_cache: GossipCache, /// This node's PeerId. pub local_peer_id: PeerId, } @@ -839,7 +841,8 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let result = message.publish(self.gossipsub_mut(), topic.clone().into()); + let message_data = message.encode(); + let result = message_data.do_publish(self.gossipsub_mut(), topic.clone().into()); if let Err(e) = result { match e { PublishError::Duplicate => { @@ -877,10 +880,17 @@ impl Network { } } - if let PublishError::NoPeersSubscribedToTopic = e { - // TODO(dknopik): fix gossip cache - //self.gossip_cache.insert(gossip_topic, message); + if let PublishError::NoPeersSubscribedToTopic = e + && let EncodedPubsubMessage::Full(bytes) = &message_data + { + let id = Sha256::digest(bytes)[..].to_vec(); + self.gossip_cache.insert(topic, id, message_data); } + } else if let EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) = + &message_data + { + let id = partial.group_id().as_ref().to_vec(); + self.gossip_cache.insert(topic, id, message_data); } } } @@ -1297,8 +1307,38 @@ impl Network { message, metadata: _, } => { - // TODO(dknopik): take a look at the metadata and publish if we have something for them - // maybe by reusing the gossip cache?! + let topic = GossipTopic::decode(topic_id.as_str()) + .inspect_err(|error| { + debug!( + topic = ?topic_id, + error, + "Could not decode gossipsub partial message topic" + ); + }) + .ok()?; + + // TODO(dknopik): maybe do this after validation? + if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { + // We do not bother checking here if that cached msg contains relevant data, + // as gossipsub will do that anyway. + let result = cached_partial + .do_publish(&mut self.swarm.behaviour_mut().gossipsub, topic.into()); + match result { + Ok(_) => { + debug!( + group_id = hex::encode(&group_id), + "Replied with cached partial message" + ) + } + Err(err) => { + warn!( + group_id = hex::encode(&group_id), + ?err, + "Failed to reply with cached partial message" + ); + } + } + } if let Some(message) = message { match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { @@ -1337,15 +1377,13 @@ impl Network { .add_subscription(&peer_id, subnet_id); } // Try to send the cached messages for this topic - if let Some(msgs) = self.gossip_cache.retrieve(&topic) { + if let Some(msgs) = self.gossip_cache.retrieve_all(&topic) { for data in msgs { let topic_str: &str = topic.kind().as_ref(); - match self - .swarm - .behaviour_mut() - .gossipsub - .publish(Topic::from(topic.clone()), data) - { + match data.do_publish( + &mut self.swarm.behaviour_mut().gossipsub, + topic.clone().into(), + ) { Ok(_) => { debug!(topic = topic_str, "Gossip message published on retry"); metrics::inc_counter_vec( diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 748df3cb15c..b2abd1d38a2 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -13,7 +13,7 @@ pub type Enr = discv5::enr::Enr; pub use eth2::lighthouse::sync_state::{BackFillState, CustodyBackFillState, SyncState}; pub use globals::NetworkGlobals; -pub use pubsub::{PubsubMessage, SnappyTransform}; +pub use pubsub::{EncodedPubsubMessage, PubsubMessage, SnappyTransform}; pub use subnet::{Subnet, SubnetDiscovery}; pub use topics::{ GossipEncoding, GossipKind, GossipTopic, TopicConfig, all_topics_at_fork, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index a784eae2e9a..2c2c110a2be 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,13 +1,13 @@ //! Handles the encoding and decoding of pubsub messages. +use crate::{Gossipsub, TopicHash}; use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; -use crate::{Gossipsub, TopicHash}; -use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; +use gossipsub::{IdentTopic, PublishError}; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, @@ -425,13 +425,8 @@ impl PubsubMessage { } } - /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If - /// no encoding is known, and error is returned. - pub fn publish( - &self, - gossipsub: &mut Gossipsub, - topic: IdentTopic, - ) -> Result<(), PublishError> { + /// Encodes a `PubsubMessage`. + pub fn encode(&self) -> EncodedPubsubMessage { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the @@ -441,7 +436,7 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::PartialDataColumnSidecar(data) => { - return gossipsub.publish_partial(topic, &data.1); + return EncodedPubsubMessage::PartialDataColumnSidecarMessage(data.1.clone()); } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), @@ -454,7 +449,7 @@ impl PubsubMessage { PubsubMessage::LightClientFinalityUpdate(data) => data.as_ssz_bytes(), PubsubMessage::LightClientOptimisticUpdate(data) => data.as_ssz_bytes(), }; - gossipsub.publish(topic, bytes).map(|_| ()) + EncodedPubsubMessage::Full(bytes) } } @@ -529,3 +524,20 @@ impl std::fmt::Display for PubsubMessage { } } } + +pub enum EncodedPubsubMessage { + Full(Vec), + PartialDataColumnSidecarMessage(PartialDataColumnSidecarMessage), +} + +impl EncodedPubsubMessage { + pub fn do_publish(&self, gossipsub: &mut Gossipsub, topic: IdentTopic) -> Result<(), PublishError> { + match self { + EncodedPubsubMessage::Full(bytes) => gossipsub + .publish(topic, bytes.clone()) + .map(|_| ()), + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => gossipsub + .publish_partial(topic, partial), + } + } +} From bb575adc3a92a39985a6af97aa929048a40e3165 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 01:50:29 +0100 Subject: [PATCH 011/146] update gossipsub --- Cargo.lock | 46 +++++++++---------- .../lighthouse_network/src/service/mod.rs | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64430c3cf72..a1e5c52fb16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 2.0.100", + "syn 1.0.109", ] [[package]] @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 2dd7ebbe5f3..5dde1a55b95 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -810,7 +810,7 @@ impl Network { let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic, partial) { + match self.gossipsub_mut().subscribe(&topic, partial, partial) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false From eeaf03b0222fff7cc7baccb0e68e862d7b9de0ce Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 02:05:06 +0100 Subject: [PATCH 012/146] add CLI flags for disable request and disable support --- beacon_node/lighthouse_network/src/config.rs | 8 +++++++ .../lighthouse_network/src/service/mod.rs | 11 +++++++++- .../lighthouse_network/src/types/pubsub.rs | 21 ++++++++++++------- beacon_node/src/cli.rs | 16 ++++++++++++++ beacon_node/src/config.rs | 8 +++++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 89c6c58d4f6..e022fcbed1e 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -143,6 +143,12 @@ pub struct Config { /// Flag for advertising a fake CGC to peers for testing ONLY. pub advertise_false_custody_group_count: Option, + + /// Whether to disable signaling partial message support. Implies disable_partial_messages_request + pub disable_partial_messages_support: bool, + + /// Whether to disable requesting partial messages + pub disable_partial_messages_request: bool, } impl Config { @@ -368,6 +374,8 @@ impl Default for Config { inbound_rate_limiter_config: None, idontwant_message_size_threshold: DEFAULT_IDONTWANT_MESSAGE_SIZE_THRESHOLD, advertise_false_custody_group_count: None, + disable_partial_messages_support: false, + disable_partial_messages_request: false, } } } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 5dde1a55b95..0f54d524d03 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -807,10 +807,19 @@ impl Network { .write() .insert(topic.clone()); + let config = &self.network_globals.config; + let disable_request = + config.disable_partial_messages_request || config.disable_partial_messages_support; + let disable_support = config.disable_partial_messages_support; + let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic, partial, partial) { + match self.gossipsub_mut().subscribe( + &topic, + partial && !disable_request, + partial && !disable_support, + ) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 2c2c110a2be..b8c4aac6612 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,13 +1,13 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::{Gossipsub, TopicHash}; use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use crate::{Gossipsub, TopicHash}; +use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use gossipsub::{IdentTopic, PublishError}; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, @@ -531,13 +531,18 @@ pub enum EncodedPubsubMessage { } impl EncodedPubsubMessage { - pub fn do_publish(&self, gossipsub: &mut Gossipsub, topic: IdentTopic) -> Result<(), PublishError> { + pub fn do_publish( + &self, + gossipsub: &mut Gossipsub, + topic: IdentTopic, + ) -> Result<(), PublishError> { match self { - EncodedPubsubMessage::Full(bytes) => gossipsub - .publish(topic, bytes.clone()) - .map(|_| ()), - EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => gossipsub - .publish_partial(topic, partial), + EncodedPubsubMessage::Full(bytes) => { + gossipsub.publish(topic, bytes.clone()).map(|_| ()) + } + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { + gossipsub.publish_partial(topic, partial) + } } } } diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index e4c7c6ff1fe..1bf6303b3ab 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -670,6 +670,22 @@ pub fn cli_app() -> Command { .hide(true) .display_order(0) ) + .arg( + Arg::new("disable-partial-messages-request") + .long("disable-partial-messages-request") + .help("Do not request partial messages for data columns.") + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) + .arg( + Arg::new("disable-partial-messages-support") + .long("disable-partial-messages-support") + .help("Do not support receiving partial messages for data columns.") + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) /* * Monitoring metrics */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 3b0e80e0b7d..05051fcbe0f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1492,6 +1492,14 @@ pub fn set_network_config( })?; } + if parse_flag(cli_args, "disable-partial-messages-request") { + config.disable_partial_messages_request = true; + } + + if parse_flag(cli_args, "disable-partial-messages-support") { + config.disable_partial_messages_support = true; + } + Ok(()) } From 47a411962c22eb5a1003ad27f55436930636d196 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 10:46:00 +0100 Subject: [PATCH 013/146] update CLI docs --- book/src/help_bn.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 5f3c43a7e42..9965938006c 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -477,6 +477,10 @@ Flags: --disable-packet-filter Disables the discovery packet filter. Useful for testing in smaller networks + --disable-partial-messages-request + Do not request partial messages for data columns. + --disable-partial-messages-support + Do not support receiving partial messages for data columns. --disable-proposer-reorgs Do not attempt to reorg late blocks from other validators when proposing. From 65c236a3a64060193146f92ed1b293aef01bcf4e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 11:19:26 +0100 Subject: [PATCH 014/146] do not error for column on failed observation check --- beacon_node/beacon_chain/src/data_column_verification.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index cdfd6957539..3791c9e2009 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -602,11 +602,10 @@ pub fn validate_data_column_sidecar_for_gossip Date: Fri, 31 Oct 2025 11:55:03 +0100 Subject: [PATCH 015/146] Fix get blobs v2 tests --- .../beacon_chain/src/fetch_blobs/tests.rs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index cbe2f78fbda..059f4e07138 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -25,7 +25,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_in_block() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, _s) = mock_publish_fn(); let block = SignedBeaconBlock::::Fulu(SignedBeaconBlockFulu { message: BeaconBlockFulu::empty(mock_adapter.spec()), @@ -53,7 +53,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, _) = mock_publish_fn(); let (block, _blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -78,7 +78,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_partial_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, mut blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -111,7 +111,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_block_imported_after_el_response() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -144,7 +144,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_new_columns_to_import() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -184,7 +184,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_success() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -259,7 +259,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_blobs_in_block() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let spec = mock_adapter.spec(); let (publish_fn, _s) = mock_publish_fn(); let block_no_blobs = @@ -287,7 +287,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, _) = mock_publish_fn(); let (block, _blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -314,7 +314,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_partial_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let blob_count = 2; let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); @@ -372,7 +372,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_block_imported_after_el_response() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -405,7 +405,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_new_blobs_to_import() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -453,7 +453,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_success() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let blob_count = 2; let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); @@ -606,7 +606,7 @@ fn mock_publish_fn() -> ( (publish_fn, captured_args) } -fn mock_beacon_adapter(fork_name: ForkName) -> MockFetchBlobsBeaconAdapter { +fn mock_beacon_adapter(fork_name: ForkName, get_blobs_v3: bool) -> MockFetchBlobsBeaconAdapter { let test_runtime = TestRuntime::default(); let spec = Arc::new(fork_name.make_genesis_spec(E::default_spec())); let kzg = get_kzg(&spec); @@ -618,4 +618,7 @@ fn mock_beacon_adapter(fork_name: ForkName) -> MockFetchBlobsBeaconAdapter { .expect_executor() .return_const(test_runtime.task_executor.clone()); mock_adapter + .expect_supports_get_blobs_v3() + .returning(move || Ok(get_blobs_v3)); + mock_adapter } From 8442bf3db13edd418d64daa94b53628e41e5e843 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 11:55:32 +0100 Subject: [PATCH 016/146] Reintroduce sanity check for get blobs v2 --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index ea77b9eb9d4..048203ff15f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -32,7 +32,7 @@ use mockall_double::double; use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use tracing::{Span, debug, instrument}; +use tracing::{Span, debug, instrument, warn}; use types::blob_sidecar::BlobSidecarError; use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; @@ -246,7 +246,8 @@ async fn fetch_and_process_blobs_v2_or_v3( metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); - let response = if chain_adapter.supports_get_blobs_v3().await? { + let get_blobs_v3 = chain_adapter.supports_get_blobs_v3().await?; + let response = if get_blobs_v3 { debug!(num_expected_blobs, "Fetching available blobs from the EL"); chain_adapter .get_blobs_v3(versioned_hashes) @@ -274,6 +275,18 @@ async fn fetch_and_process_blobs_v2_or_v3( let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); + if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { + // This scenario is not supposed to happen if the EL is spec compliant. + // It should either return all requested blobs or none, but NOT partial responses. + // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. + warn!( + num_fetched_blobs, + num_expected_blobs, "The EL did not return all requested blobs" + ); + inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); + return Ok(None); + } + debug!(num_fetched_blobs, "Blobs received from the EL"); inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); From 7c68ba449d761e8270fac3070e9838f4d64c5148 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:20:13 +0100 Subject: [PATCH 017/146] Only count actually received blobs --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 048203ff15f..7b0295dacca 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -272,7 +272,7 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); }; - let num_fetched_blobs = blobs_and_proofs.len(); + let num_fetched_blobs = response.iter().filter(|opt| opt.is_some()).count(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { From 09e220b2922f85f8dabcc0fd79d62367d863ff4a Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:20:55 +0100 Subject: [PATCH 018/146] Log full vs partial --- beacon_node/lighthouse_network/src/types/pubsub.rs | 4 ++++ beacon_node/network/src/service.rs | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index b8c4aac6612..0d04995ec93 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -451,6 +451,10 @@ impl PubsubMessage { }; EncodedPubsubMessage::Full(bytes) } + + pub fn is_partial(&self) -> bool { + matches!(self, PubsubMessage::PartialDataColumnSidecar(_)) + } } impl std::fmt::Display for PubsubMessage { diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index dd9448983c6..23d662c9366 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -637,8 +637,14 @@ impl NetworkService { NetworkMessage::Publish { messages } => { let mut topic_kinds = Vec::new(); for message in &messages { - if !topic_kinds.contains(&message.kind()) { - topic_kinds.push(message.kind()); + let message_kind = if message.is_partial() { + "partial" + } else { + "full" + }; + let kind = (message.kind(), message_kind); + if !topic_kinds.contains(&kind) { + topic_kinds.push(kind); } } debug!( From 7cbb5c652e80a94ed64f92292b2bcbd1ed0d41e6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:44:23 +0100 Subject: [PATCH 019/146] fix and improve logging --- .../beacon_chain/src/fetch_blobs/mod.rs | 2 +- .../lighthouse_network/src/service/mod.rs | 28 +++++++++++++++++-- .../lighthouse_network/src/types/pubsub.rs | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 7b0295dacca..66e82f548b7 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -272,7 +272,7 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); }; - let num_fetched_blobs = response.iter().filter(|opt| opt.is_some()).count(); + let num_fetched_blobs = blobs_and_proofs.iter().filter(|opt| opt.is_some()).count(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 0a34bffb1dd..5f2dc7083a7 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -35,12 +35,14 @@ use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; use sha2::{Digest, Sha256}; +use ssz::Decode; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; +use types::partial_data_column_sidecar::CellBitmap; use types::{ChainSpec, ForkName}; use types::{ EnrForkId, EthSpec, ForkContext, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, @@ -1314,8 +1316,21 @@ impl Network { propagation_source, group_id, message, - metadata: _, + metadata, } => { + // TODO(dknopik): this currently hardcodes the data column metadata format + if let Some(metadata) = metadata { + if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { + let metadata = hex::encode(metadata.as_slice()); + debug!(%metadata, "Got metadata") + } else { + warn!(?metadata, "Got weird metadata"); + } + } else { + debug!("Got no metadata") + } + + let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { debug!( @@ -1335,18 +1350,20 @@ impl Network { match result { Ok(_) => { debug!( - group_id = hex::encode(&group_id), + group_id = hex_group_id, "Replied with cached partial message" ) } Err(err) => { warn!( - group_id = hex::encode(&group_id), + group_id = hex_group_id, ?err, "Failed to reply with cached partial message" ); } } + } else { + debug!(group_id = hex_group_id, "Did not reply to partial message"); } if let Some(message) = message { @@ -1366,6 +1383,11 @@ impl Network { //); } Ok(message) => { + debug!( + %message, + %propagation_source, + "Decoded partial message" + ); // Notify the network return Some(NetworkEvent::PubsubMessage { id: MessageId::new(&[]), // TODO(dknopik): waht to send diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 0d04995ec93..d9f9997d2d0 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -482,7 +482,7 @@ impl std::fmt::Display for PubsubMessage { f, "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", data.1.partial_column.block_root, - data.1.partial_column.block_root, + data.1.partial_column.index, hex::encode( data.1 .partial_column From b6626f610e436290f6d5d1ae7337458a05621387 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 11:24:51 +0100 Subject: [PATCH 020/146] update gossipsub --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1e5c52fb16..e1b8f41038d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "pin-project", From f0b7d8fd4ee6330f0cb0dd5bb7ca9db533b36d34 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 12:10:29 +0100 Subject: [PATCH 021/146] look for block in data availability checker --- .../gossip_methods.rs | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 500842ff149..3aef2f6cae6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -10,8 +10,9 @@ use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use beacon_chain::store::Error; use beacon_chain::{ - AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, - GossipVerifiedBlock, IntoExecutionPendingBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, + BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, IntoExecutionPendingBlock, + NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -791,26 +792,17 @@ impl NetworkBeaconProcessor { ) { let partial_column = match self .chain - .store - .try_get_full_block(&column_sidecar.block_root) + .data_availability_checker + .get_cached_block(&column_sidecar.block_root) { - Ok(Some(DatabaseBlock::Full(block))) => { - Some(VerifiablePartialDataColumn::from_dangling_and_block( - column_sidecar.clone(), - &block, - )) - } - Ok(Some(DatabaseBlock::Blinded(block))) => { - Some(VerifiablePartialDataColumn::from_dangling_and_block( - column_sidecar.clone(), - &block, - )) - } - Ok(None) => None, - Err(err) => { - warn!(?err, "Error getting block for partial data column"); - None - } + Some( + BlockProcessStatus::ExecutionValidated(block) + | BlockProcessStatus::NotValidated(block, _), + ) => Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )), + None | Some(BlockProcessStatus::Unknown) => None, }; match partial_column { From 99d3568161cf1cb29deb26353fc8702d38092151 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 13:46:11 +0100 Subject: [PATCH 022/146] re-enable SSE --- beacon_node/beacon_chain/src/beacon_chain.rs | 38 ++++++++++--------- .../gossip_methods.rs | 1 - 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 11694acb24c..5f2bc2a3005 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3089,11 +3089,10 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - // TODO(dknopik): impl SSE - //self.emit_sse_data_column_sidecar_events( - // &block_root, - // data_columns.iter().map(|column| column.as_data_column()), - //); + self.emit_sse_data_column_sidecar_events( + &block_root, + data_columns.iter().map(|column| column.as_data_column()), + ); self.check_gossip_data_columns_availability_and_import( slot, @@ -3166,12 +3165,11 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CustodyColumns(_columns) => { - // TODO(dknopik): impl SSE - //self.emit_sse_data_column_sidecar_events( - // &block_root, - // columns.iter().map(|column| column.as_data_column()), - //); + EngineGetBlobsOutput::CustodyColumns(columns) => { + self.emit_sse_data_column_sidecar_events( + &block_root, + columns.iter().map(|column| column.as_data_column()), + ); } } @@ -3200,26 +3198,30 @@ impl BeaconChain { } } - fn emit_sse_data_column_sidecar_events<'a, I>( + fn emit_sse_data_column_sidecar_events<'a, I, C>( self: &Arc, block_root: &Hash256, data_columns_iter: I, ) where - I: Iterator>, + I: Iterator, + C: DasColumn + 'a, { if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_data_column_sidecar_subscribers() { let imported_data_columns = self .data_availability_checker - .cached_data_column_indexes(block_root) + .get_data_columns(*block_root) .unwrap_or_default(); - let new_data_columns = - data_columns_iter.filter(|b| !imported_data_columns.contains(&b.index)); - for data_column in new_data_columns { + let new_data_columns_indices = data_columns_iter.map(|c| c.index()).collect::>(); + + for data_column in imported_data_columns + .into_iter() + .filter(|c| new_data_columns_indices.contains(&c.index)) + { event_handler.register(EventKind::DataColumnSidecar( - SseDataColumnSidecar::from_data_column_sidecar(data_column), + SseDataColumnSidecar::from_data_column_sidecar(data_column.as_ref()), )); } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 3aef2f6cae6..9333a6ef9d6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -56,7 +56,6 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use store::DatabaseBlock; use types::das_column::DasColumn; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; From 646fef59f994a1e24aade34dcbb44d2be3c24e11 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 15:25:54 +0100 Subject: [PATCH 023/146] Properly handly observing in new_for_block_publishing --- .../beacon_chain/src/data_column_verification.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 3791c9e2009..e5df753a91c 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -292,12 +292,11 @@ impl // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; - // TODO(dknopik): is proper handling of none? if chain .data_availability_checker .determine_missing_cells(&column_sidecar.block_root(), column_sidecar.as_ref()) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)? - .is_empty() + .map(|cells| cells.is_empty()) + .unwrap_or(false) { // Observe this data column so we don't process it again. if O::observe() { @@ -680,12 +679,6 @@ pub fn validate_partial_data_column_sidecar_for_gossip< let kzg_verified_data_column = verify_kzg_for_data_column(filtered_column, kzg) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; - // TODO(dknopik): observe?! I do not think so, as publishing works differently anyway... - // Basically we publish if we update our internal view, but this is merged elsewhere - //if O::observe() { - // observe_gossip_data_column(&data_column, chain)?; - //} - Ok(GossipVerifiedDataColumn { block_root: data_column.block_root(), data_column: kzg_verified_data_column, From 6794e5fddc98e4146fe7a5245b1361d24e7802e9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 16:57:48 +0100 Subject: [PATCH 024/146] Fix cache and better logging --- .../src/service/gossip_cache.rs | 6 +++ .../lighthouse_network/src/service/mod.rs | 52 +++++++++++++------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 5aa655def96..8259fa0701d 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -96,6 +96,12 @@ impl GossipCacheBuilder { self } + /// Timeout for data column sidecars. + pub fn data_column_sidecar(mut self, timeout: Duration) -> Self { + self.data_column_sidecar = Some(timeout); + self + } + /// Timeout for aggregate attestations. pub fn aggregates_timeout(mut self, timeout: Duration) -> Self { self.aggregates = Some(timeout); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 5f2dc7083a7..7672782fb81 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -259,6 +259,7 @@ impl Network { // .signed_contribution_and_proof_timeout(timeout) // Do not retry // .sync_committee_message_timeout(timeout) // Do not retry .bls_to_execution_change_timeout(half_epoch * 2) + .data_column_sidecar(slot_duration) .build() }; @@ -901,6 +902,11 @@ impl Network { &message_data { let id = partial.group_id().as_ref().to_vec(); + debug!( + kind = %topic.kind(), + group_id = hex::encode(&id), + "Adding partial message to cache" + ); self.gossip_cache.insert(topic, id, message_data); } } @@ -1318,18 +1324,6 @@ impl Network { message, metadata, } => { - // TODO(dknopik): this currently hardcodes the data column metadata format - if let Some(metadata) = metadata { - if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { - let metadata = hex::encode(metadata.as_slice()); - debug!(%metadata, "Got metadata") - } else { - warn!(?metadata, "Got weird metadata"); - } - } else { - debug!("Got no metadata") - } - let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { @@ -1341,29 +1335,55 @@ impl Network { }) .ok()?; + // TODO(dknopik): this currently hardcodes the data column metadata format + if let Some(metadata) = metadata { + if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { + let mut metadata_string = String::with_capacity(metadata.len()); + for bit in metadata.iter() { + if bit { + metadata_string.push('1'); + } else { + metadata_string.push('0'); + } + } + debug!(metadata = metadata_string, %topic, %propagation_source, "Got metadata") + } else { + warn!(?metadata, %topic, %propagation_source, "Got weird metadata"); + } + } else { + debug!(%propagation_source, "Got no metadata") + } + // TODO(dknopik): maybe do this after validation? if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { // We do not bother checking here if that cached msg contains relevant data, // as gossipsub will do that anyway. - let result = cached_partial - .do_publish(&mut self.swarm.behaviour_mut().gossipsub, topic.into()); + let result = cached_partial.do_publish( + &mut self.swarm.behaviour_mut().gossipsub, + topic.clone().into(), + ); match result { Ok(_) => { debug!( group_id = hex_group_id, - "Replied with cached partial message" + %topic, %propagation_source, "Replied with cached partial message" ) } Err(err) => { warn!( group_id = hex_group_id, + %topic, + %propagation_source, ?err, "Failed to reply with cached partial message" ); } } } else { - debug!(group_id = hex_group_id, "Did not reply to partial message"); + debug!( + group_id = hex_group_id, + %topic, %propagation_source, "Did not reply to partial message" + ); } if let Some(message) = message { From e98dce0f75ace74dfdf80e2e306e185f5ec19f7b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 16:59:45 +0100 Subject: [PATCH 025/146] Fix invalid `UnexpectedDataColumn` error --- .../src/data_availability_checker.rs | 5 +++- .../src/data_column_verification.rs | 26 +++++++++++-------- consensus/types/src/das_column.rs | 1 + 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index b43e0826f7b..51d9382877f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -195,7 +195,10 @@ impl DataAvailabilityChecker { match cached.compare(data_column) { ColumnComparison::Equal => Some(vec![]), ColumnComparison::MissingCells { missing_in_lhs, .. } => Some(missing_in_lhs), - _ => None, + comparison => { + debug!(?comparison, "Unexpected columns comparison"); + None + } } } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index e5df753a91c..0ccdcef6afd 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -655,22 +655,26 @@ pub fn validate_partial_data_column_sidecar_for_gossip< > { // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply let column_slot = data_column.slot(); + let block_root = data_column.block_root(); verify_sidecar_not_from_future_slot(chain, column_slot)?; verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; - let missing_cells = chain + let filtered_column = if let Some(missing_cells) = chain .data_availability_checker - .determine_missing_cells(&data_column.block_root(), data_column.as_ref()) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?; - if missing_cells.is_empty() { - return Err(GossipDataColumnError::PriorKnownUnpublished); - } + .determine_missing_cells(&block_root, data_column.as_ref()) + { + if missing_cells.is_empty() { + return Err(GossipDataColumnError::PriorKnownUnpublished); + } - let filtered_column = Arc::new( + Arc::new( + data_column + .clone_filter(|idx| missing_cells.contains(&idx)) + .ok_or_else(|| GossipDataColumnError::PriorKnownUnpublished)?, + ) + } else { data_column - .clone_filter(|idx| missing_cells.contains(&idx)) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?, - ); + }; // We do not have to check block related data here, as we create the verifiable column from // gossip accepted block @@ -680,7 +684,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip< .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; Ok(GossipVerifiedDataColumn { - block_root: data_column.block_root(), + block_root, data_column: kzg_verified_data_column, _phantom: PhantomData, }) diff --git a/consensus/types/src/das_column.rs b/consensus/types/src/das_column.rs index 3e25576fdac..a1597dbd8d2 100644 --- a/consensus/types/src/das_column.rs +++ b/consensus/types/src/das_column.rs @@ -74,6 +74,7 @@ pub trait DasColumn: Clone { } } +#[derive(Debug)] pub enum ColumnComparison { DifferentColumns, DataConflict, From 5e1f05dd2fb231360ad8091c4bb31e1c7d221a84 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 17:01:42 +0100 Subject: [PATCH 026/146] remove custom cache in favour of reprocessing queue --- beacon_node/beacon_processor/src/lib.rs | 7 +- .../src/scheduler/work_reprocessing_queue.rs | 117 ++++++++++++++++- beacon_node/network/src/lib.rs | 1 - .../gossip_methods.rs | 122 +++++++++++------- .../src/network_beacon_processor/mod.rs | 7 +- .../network_beacon_processor/sync_methods.rs | 15 +++ .../network/src/partial_data_column_cache.rs | 76 ----------- beacon_node/network/src/router.rs | 6 +- 8 files changed, 217 insertions(+), 134 deletions(-) delete mode 100644 beacon_node/network/src/partial_data_column_cache.rs diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index c3dfd0a9126..21f98175445 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -39,7 +39,8 @@ //! task. use crate::work_reprocessing_queue::{ - QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, ReprocessQueueMessage, + QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, QueuedPartialColumn, + ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; @@ -493,6 +494,10 @@ impl From for WorkEvent { work: Work::ColumnReconstruction(process_fn), } } + ReadyWork::PartialColumn(QueuedPartialColumn { process_fn, .. }) => Self { + drop_during_sync: true, + work: Work::GossipPartialDataColumnSidecar(process_fn), + }, } } } diff --git a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs index c99388287c0..04286714ec0 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs @@ -30,7 +30,7 @@ use strum::AsRefStr; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; -use tracing::{debug, error, trace, warn}; +use tracing::{debug, error, info, trace, warn}; use types::{EthSpec, Hash256, Slot}; const TASK_NAME: &str = "beacon_processor_reprocess_queue"; @@ -60,6 +60,9 @@ pub const QUEUED_SAMPLING_REQUESTS_DELAY: Duration = Duration::from_secs(12); /// For how long to queue delayed column reconstruction. pub const QUEUED_RECONSTRUCTION_DELAY: Duration = Duration::from_millis(150); +/// For how long to queue partial columns. +pub const QUEUED_PARTIAL_COLUMN_DELAY: Duration = Duration::from_secs(4); + /// Set an arbitrary upper-bound on the number of queued blocks to avoid DoS attacks. The fact that /// we signature-verify blocks before putting them in the queue *should* protect against this, but /// it's nice to have extra protection. @@ -71,6 +74,9 @@ const MAXIMUM_QUEUED_ATTESTATIONS: usize = 16_384; /// How many light client updates we keep before new ones get dropped. const MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES: usize = 128; +/// How many partial column messages we keep before new ones get dropped. +const MAXIMUM_QUEUED_PARTIAL_COLUMNS: usize = 256; + // Process backfill batch 50%, 60%, 80% through each slot. // // Note: use caution to set these fractions in a way that won't cause panic-y @@ -115,6 +121,10 @@ pub enum ReprocessQueueMessage { BackfillSync(QueuedBackfillBatch), /// A delayed column reconstruction that needs checking DelayColumnReconstruction(QueuedColumnReconstruction), + /// A block with pending data availability was added. + BlockPending { block_root: Hash256 }, + /// A partial data column that references an unknown block. + UnknownPartialColumn(QueuedPartialColumn), } /// Events sent by the scheduler once they are ready for re-processing. @@ -127,6 +137,7 @@ pub enum ReadyWork { LightClientUpdate(QueuedLightClientUpdate), BackfillSync(QueuedBackfillBatch), ColumnReconstruction(QueuedColumnReconstruction), + PartialColumn(QueuedPartialColumn), } /// An Attestation for which the corresponding block was not seen while processing, queued for @@ -182,6 +193,11 @@ pub struct QueuedColumnReconstruction { pub process_fn: AsyncFn, } +pub struct QueuedPartialColumn { + pub beacon_block_root: Hash256, + pub process_fn: AsyncFn, +} + impl TryFrom> for QueuedBackfillBatch { type Error = WorkEvent; @@ -220,6 +236,8 @@ enum InboundEvent { ReadyBackfillSync(QueuedBackfillBatch), /// A column reconstruction that was queued is ready for processing. ReadyColumnReconstruction(QueuedColumnReconstruction), + /// A partial column that was queued has timed out. + ReadyPartialColumn((DelayKey, QueuedPartialColumn)), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -242,6 +260,8 @@ struct ReprocessQueue { lc_updates_delay_queue: DelayQueue, /// Queue to manage scheduled column reconstructions. column_reconstructions_delay_queue: DelayQueue, + /// Queue to manage scheduled partial columns. + partial_columns_delay_queue: DelayQueue, /* Queued items */ /// Queued blocks. @@ -260,6 +280,8 @@ struct ReprocessQueue { queued_column_reconstructions: HashMap, /// Queued backfill batches queued_backfill_batches: Vec, + /// Partial columns per root. + awaiting_partial_columns_per_root: HashMap>, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations @@ -355,6 +377,16 @@ impl Stream for ReprocessQueue { Poll::Ready(None) | Poll::Pending => (), } + match self.partial_columns_delay_queue.poll_expired(cx) { + Poll::Ready(Some(partial_column)) => { + return Poll::Ready(Some(InboundEvent::ReadyPartialColumn(( + partial_column.key(), + partial_column.into_inner(), + )))); + } + Poll::Ready(None) | Poll::Pending => (), + } + if let Some(next_backfill_batch_event) = self.next_backfill_batch_event.as_mut() { match next_backfill_batch_event.as_mut().poll(cx) { Poll::Ready(_) => { @@ -422,6 +454,7 @@ impl ReprocessQueue { attestations_delay_queue: DelayQueue::new(), lc_updates_delay_queue: DelayQueue::new(), column_reconstructions_delay_queue: DelayQueue::new(), + partial_columns_delay_queue: DelayQueue::new(), queued_gossip_block_roots: HashSet::new(), queued_lc_updates: FnvHashMap::default(), queued_aggregates: FnvHashMap::default(), @@ -430,6 +463,7 @@ impl ReprocessQueue { awaiting_lc_updates_per_parent_root: HashMap::new(), queued_backfill_batches: Vec::new(), queued_column_reconstructions: HashMap::new(), + awaiting_partial_columns_per_root: HashMap::new(), next_attestation: 0, next_lc_update: 0, early_block_debounce: TimeLatch::default(), @@ -702,6 +736,15 @@ impl ReprocessQueue { ); } } + + // Remove waiting partial columns - as the block is imported they will not be needed + if let Some(queued_partials) = + self.awaiting_partial_columns_per_root.remove(&block_root) + { + for key in queued_partials { + self.partial_columns_delay_queue.remove(&key); + } + } } InboundEvent::Msg(NewLightClientOptimisticUpdate { parent_root }) => { // Unqueue the light client optimistic updates we have for this root, if any. @@ -784,6 +827,62 @@ impl ReprocessQueue { } } } + InboundEvent::Msg(BlockPending { block_root }) => { + if let Some(queued_partials) = + self.awaiting_partial_columns_per_root.remove(&block_root) + { + let mut sent_count = 0; + let mut failed_count = 0; + for key in queued_partials { + let partial_column = self.partial_columns_delay_queue.remove(&key); + + if self + .ready_work_tx + .try_send(ReadyWork::PartialColumn(partial_column.into_inner())) + .is_ok() + { + sent_count += 1; + } else { + failed_count += 1; + } + } + + if failed_count > 0 { + error!( + hint = "system may be overloaded", + ?block_root, + failed_count, + sent_count, + "Ignored scheduled partial column(s) for block" + ); + } else { + info!(?block_root, sent_count, "Sent partial column(s) for block") + } + } + } + InboundEvent::Msg(UnknownPartialColumn(queued_partial_column)) => { + if self.partial_columns_delay_queue.len() >= MAXIMUM_QUEUED_PARTIAL_COLUMNS { + error!( + queue_size = MAXIMUM_QUEUED_PARTIAL_COLUMNS, + msg = "system resources may be saturated", + "Partial columns delay queue is full" + ); + return; + } + + let block_root = queued_partial_column.beacon_block_root; + + // Register the delay. + let delay_key = self + .partial_columns_delay_queue + .insert(queued_partial_column, QUEUED_ATTESTATION_DELAY); + + // Register this attestation for the corresponding root. + self.awaiting_partial_columns_per_root + .entry(block_root) + .or_default() + .push(delay_key); + } // A block that was queued for later processing is now ready to be processed. InboundEvent::ReadyGossipBlock(ready_block) => { let block_root = ready_block.beacon_block_root; @@ -934,6 +1033,22 @@ impl ReprocessQueue { ); } } + InboundEvent::ReadyPartialColumn((key, partial_column)) => { + if let Entry::Occupied(mut queued_cols) = self + .awaiting_partial_columns_per_root + .entry(partial_column.beacon_block_root) + && let Some(index) = queued_cols.get().iter().position(|&k| k == key) + { + let queued_atts_mut = queued_cols.get_mut(); + queued_atts_mut.swap_remove(index); + + if queued_atts_mut.is_empty() { + queued_cols.remove_entry(); + } + } + + // Do not send partial column, we timed out already. + } } metrics::set_gauge_vec( diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index db2fd876287..2a7fedb53e9 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,7 +4,6 @@ pub mod service; mod metrics; mod nat; mod network_beacon_processor; -mod partial_data_column_cache; mod persisted_dht; mod router; mod status; diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9333a6ef9d6..ce2aad392bb 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -11,8 +11,7 @@ use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifi use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, - BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, IntoExecutionPendingBlock, - NotifyExecutionLayer, + BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -48,7 +47,7 @@ use types::{ Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, beacon_block::BlockImportSource, }; -use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; +use beacon_processor::work_reprocessing_queue::{QueuedColumnReconstruction, QueuedPartialColumn}; use beacon_processor::{ DuplicateCache, GossipAggregatePackage, GossipAttestationBatch, work_reprocessing_queue::{ @@ -785,9 +784,9 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_dangling_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - _subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_duration: Duration, + allow_reprocess: bool, ) { let partial_column = match self .chain @@ -817,17 +816,65 @@ impl NetworkBeaconProcessor { Some(Err(err)) => { warn!(?err, "Error creating verifiable partial data column"); } - None => { - debug!("Received partial while not having block"); - metrics::inc_counter( - &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, - ); - self.partial_data_column_cache.lock().insert( - column_sidecar, - peer_id, - seen_duration, - ); - } + None => self.handle_dangling_partial_data_column_sidecar_missing_block( + peer_id, + column_sidecar, + seen_duration, + allow_reprocess, + ), + } + } + + fn handle_dangling_partial_data_column_sidecar_missing_block( + self: &Arc, + peer_id: PeerId, + column_sidecar: Arc>, + seen_duration: Duration, + allow_reprocess: bool, + ) { + // TODO(dknopik): is this method of checking adequate?... + if self + .chain + .store + .block_exists(&column_sidecar.block_root) + .unwrap_or(false) + { + debug!("Received partial for already imported block"); + return; + } + if !allow_reprocess { + debug!("Not reprocessing"); + return; + } + debug!("Received partial while not having block"); + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, + ); + // TODO(dknopik): self.send_sync_message() ? + let cloned_self = self.clone(); + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::UnknownPartialColumn( + QueuedPartialColumn { + beacon_block_root: column_sidecar.block_root, + process_fn: Box::pin(async move { + cloned_self + .process_gossip_dangling_partial_data_column_sidecar( + peer_id, + column_sidecar, + seen_duration, + false, + ) + .await; + }), + }, + )), + }) + .is_err() + { + error!("Failed to send attestation for re-processing") } } @@ -1421,35 +1468,6 @@ impl NetworkBeaconProcessor { Span::current().record("block_root", block_root.to_string()); if let Some(handle) = duplicate_cache.check_and_insert(block_root) { - // First, get the partial columns that have been waiting. Do this first in order to gossip them quickly (i guess?) TODO(dknopik): check if good - let partials = { - let mut partial_cache = self.partial_data_column_cache.lock(); - let partials = partial_cache.get_for_block(block_root); - // Opportunistically clean the cache as we are holding the lock anyway. TODO(dknopik): do somewhere else - partial_cache.clean(); - partials - }; - - for cached_partial in partials { - // TODO(dknopik): Do this in parallel (queueing it maybe) - match VerifiablePartialDataColumn::from_dangling_and_block( - cached_partial.sidecar, - gossip_verified_block.block(), - ) { - Ok(partial_data_column) => { - self.process_gossip_verifiable_partial_data_column_sidecar( - cached_partial.peer_id, - Arc::new(partial_data_column), - cached_partial.seen_duration, - ) - .await; - } - Err(err) => { - warn!(?err, "Unable to process cached partial column") - } - } - } - self.process_gossip_verified_block( peer_id, gossip_verified_block, @@ -1835,6 +1853,22 @@ impl NetworkBeaconProcessor { %block_root, "Processed block, waiting for other components" ); + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::BlockPending { + block_root: *block_root, + }), + }) + .is_err() + { + error!( + source = "gossip", + ?block_root, + "Failed to inform block pending" + ) + }; } Err(BlockError::ParentUnknown { .. }) => { // This should not occur. It should be checked by `should_forward_block`. diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7bb4884c851..ed607406020 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1,4 +1,3 @@ -use crate::partial_data_column_cache::PartialDataColumnCache; use crate::sync::manager::BlockProcessType; use crate::{service::NetworkMessage, sync::manager::SyncMessage}; use beacon_chain::blob_verification::{GossipBlobError, observe_gossip_blob}; @@ -22,7 +21,6 @@ use lighthouse_network::{ Client, MessageId, NetworkGlobals, PeerId, PubsubMessage, rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, }; -use parking_lot::Mutex; use rand::prelude::SliceRandom; use std::path::PathBuf; use std::sync::Arc; @@ -58,7 +56,6 @@ pub enum InvalidBlockStorage { pub struct NetworkBeaconProcessor { pub beacon_processor_send: BeaconProcessorSend, pub duplicate_cache: DuplicateCache, - pub partial_data_column_cache: Mutex>, pub chain: Arc>, pub network_tx: mpsc::UnboundedSender>, pub sync_tx: mpsc::UnboundedSender>, @@ -257,7 +254,6 @@ impl NetworkBeaconProcessor { pub fn send_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_timestamp: Duration, ) -> Result<(), Error> { @@ -266,9 +262,9 @@ impl NetworkBeaconProcessor { processor .process_gossip_dangling_partial_data_column_sidecar( peer_id, - subnet_id, column_sidecar, seen_timestamp, + true, ) .await }; @@ -1123,7 +1119,6 @@ impl NetworkBeaconProcessor> { let network_beacon_processor = Self { beacon_processor_send: beacon_processor_tx, duplicate_cache: DuplicateCache::default(), - partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain, network_tx, sync_tx, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index a7cff747230..06f6f13f0b7 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -221,6 +221,21 @@ impl NetworkBeaconProcessor { self.chain.recompute_head_at_current_slot().await; } Ok(AvailabilityProcessingStatus::MissingComponents(..)) => { + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::BlockPending { block_root }), + }) + .is_err() + { + error!( + source = "rpc", + ?block_root, + "Failed to inform block pending" + ) + }; + // Block is valid, we can now attempt fetching blobs from EL using version hashes // derived from kzg commitments from the block, without having to wait for all blobs // to be sent from the peers if we already have them. diff --git a/beacon_node/network/src/partial_data_column_cache.rs b/beacon_node/network/src/partial_data_column_cache.rs deleted file mode 100644 index 07f7cccabcc..00000000000 --- a/beacon_node/network/src/partial_data_column_cache.rs +++ /dev/null @@ -1,76 +0,0 @@ -use lighthouse_network::PeerId; -use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; -use types::{EthSpec, Hash256}; - -const BLOCK_LIMIT: usize = 16; -const SIDECAR_PER_BLOCK_LIMIT: usize = 16; -const EXPIRATION_TIME: Duration = Duration::from_secs(24); - -/// Really dumb hacky implementation of a cache for partial data column sidecars. -/// -/// The issue is that (with the current spec draft) we have to match up the dangling partial data -/// columns sidecars with the corresponding block. Of course, usually we have to take great care to -/// not be exploitable, but this cache design (for now) assumes no malicious behaviour. -/// -/// Do not take anything in this file as an actual implementation suggestion, it is just a hack -/// while we discuss the spec! -pub struct PartialDataColumnCache { - per_block: HashMap>>, -} - -pub struct CachedPartial { - pub sidecar: Arc>, - pub peer_id: PeerId, - pub seen_duration: Duration, -} - -impl PartialDataColumnCache { - pub fn new() -> Self { - Self { - per_block: HashMap::new(), - } - } - - pub fn insert( - &mut self, - sidecar: Arc>, - peer_id: PeerId, - seen_duration: Duration, - ) { - let len = self.per_block.len(); - let entry = self.per_block.entry(sidecar.block_root); - if matches!(entry, Entry::Vacant(_)) && len >= BLOCK_LIMIT { - return; - } - - let sidecars = entry.or_default(); - if sidecars.len() < SIDECAR_PER_BLOCK_LIMIT { - sidecars.push(CachedPartial { - sidecar, - peer_id, - seen_duration, - }); - } - } - - pub fn get_for_block(&mut self, block_root: Hash256) -> Vec> { - self.per_block.remove(&block_root).unwrap_or_default() - } - - pub fn clean(&mut self) { - if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { - self.per_block.retain(|_, sidecars| { - sidecars - .iter() - .map(|cached| cached.seen_duration) - .max() - .map(|last_seen| now.saturating_sub(last_seen) < EXPIRATION_TIME) - .unwrap_or(false) - }); - } - } -} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 30ccb4319ae..a39bfb8c4e3 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -6,7 +6,6 @@ #![allow(clippy::unit_arg)] use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; -use crate::partial_data_column_cache::PartialDataColumnCache; use crate::service::NetworkMessage; use crate::status::status_message; use crate::sync::SyncMessage; @@ -20,7 +19,6 @@ use lighthouse_network::{ }; use logging::TimeLatch; use logging::crit; -use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; @@ -99,7 +97,6 @@ impl Router { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send, duplicate_cache: DuplicateCache::default(), - partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: beacon_chain.clone(), network_tx: network_send.clone(), sync_tx: sync_send.clone(), @@ -388,12 +385,11 @@ impl Router { ) } PubsubMessage::PartialDataColumnSidecar(data) => { - let (subnet_id, column_sidecar) = *data; + let (_, column_sidecar) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor .send_gossip_partial_data_column_sidecar( peer_id, - subnet_id, column_sidecar.partial_column, timestamp_now(), ), From 695c648a4bc491a0ae21177e2a826755b5b2b3b9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 17:12:02 +0100 Subject: [PATCH 027/146] publish partials on http API block --- beacon_node/http_api/src/publish_blocks.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index c5f9b133696..f8d680cc86b 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -564,9 +564,15 @@ fn publish_column_sidecars( } let pubsub_messages = data_column_sidecars .into_iter() - .map(|data_col| { + .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); - PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))) + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*data_col).clone().into_partial().column.into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), + ] }) .collect::>(); crate::publish_pubsub_messages(sender_clone, pubsub_messages) From 6fb3721e8ea2bbe797110c2e646081a5831b20fe Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 20:34:54 +0100 Subject: [PATCH 028/146] experimentally full eager publish for API blocks --- beacon_node/http_api/src/publish_blocks.rs | 15 +++++--- .../lighthouse_network/src/types/partial.rs | 32 ++++++++++------- .../lighthouse_network/src/types/pubsub.rs | 36 ++++++++++++------- .../gossip_methods.rs | 6 ++-- .../src/network_beacon_processor/mod.rs | 3 +- beacon_node/network/src/router.rs | 4 +-- 6 files changed, 60 insertions(+), 36 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index f8d680cc86b..8f6ea9d632e 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -209,7 +209,7 @@ pub async fn publish_block>( debug!( partial = merged_data.updated_partials.len(), full = merged_data.completed_columns.len(), - "Sending merged data after block availability" + "Sending merged data after block publish (should not happen?)" ); let messages: Vec<_> = merged_data .updated_partials @@ -217,7 +217,8 @@ pub async fn publish_block>( .map(|partial| { PubsubMessage::PartialDataColumnSidecar(Box::new(( DataColumnSubnetId::from_column_index(partial.index(), &spec), - partial.column.clone().into(), + partial.column.clone(), + None, ))) }) .chain(merged_data.completed_columns.into_iter().flat_map(|full| { @@ -225,7 +226,8 @@ pub async fn publish_block>( [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*full).clone().into_partial().column.clone().into(), + (*full).clone().into_partial().column.clone(), + None, ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), ] @@ -234,7 +236,7 @@ pub async fn publish_block>( if !messages.is_empty() && let Err(err) = crate::publish_pubsub_messages(&sender_clone, messages) { - warn!(?err, "Publishing data after block availability failed") + warn!(?err, "Publishing data after block publish") } }; @@ -566,10 +568,13 @@ fn publish_column_sidecars( .into_iter() .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); + let column = (*data_col).clone().into_partial().column; + let all_cells = column.sidecar.cells_present_bitmap.clone(); [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*data_col).clone().into_partial().column.into(), + column, + Some(all_cells), ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 3a165127e1e..da712e4daf8 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -9,29 +9,35 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, - pub send_eager: Option>, + send_eager: Option>, } -impl From> for PartialDataColumnSidecarMessage { - fn from(value: DanglingPartialDataColumn) -> Self { - Self { - partial_column: Arc::new(value), +impl PartialDataColumnSidecarMessage { + pub fn new(partial_column: Arc>) -> Self { + PartialDataColumnSidecarMessage { + partial_column, send_eager: None, } } -} -impl From>> for PartialDataColumnSidecarMessage { - fn from(value: Arc>) -> Self { - Self { - partial_column: value, - send_eager: None, - } + pub fn eagerly_send(&mut self, cells: &CellBitmap) { + let Some(eager) = self + .partial_column + .sidecar + .clone_filter(|idx| cells.get(idx).unwrap_or(false)) + else { + return; + }; + + self.send_eager = Some(SendEager { + data: eager.as_ssz_bytes(), + metadata: eager.cells_present_bitmap.into(), + }) } } #[derive(Debug, Clone, PartialEq)] -pub struct SendEager { +struct SendEager { /// The encoded message to send eagerly, i.e. when we have no metadata for that peer. data: Vec, /// The metadata to associate with a peer after sending it the eager message. diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index d9f9997d2d0..db475213ab6 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -8,7 +8,9 @@ use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; +use types::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, +}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, @@ -21,6 +23,12 @@ use types::{ SyncCommitteeMessage, SyncSubnetId, }; +type PartialDataColumnSidecarTuple = ( + DataColumnSubnetId, + Arc>, + Option>, +); + #[derive(Debug, Clone, PartialEq)] pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. @@ -30,7 +38,8 @@ pub enum PubsubMessage { /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a [`PartialDataColumnSidecar`] along with the subnet id where it was received. - PartialDataColumnSidecar(Box<(DataColumnSubnetId, PartialDataColumnSidecarMessage)>), // TODO(dknopik): review `Arc` situation + /// TODO(dknopik) - it is time for this to move into its own enum. + PartialDataColumnSidecar(Box>), // TODO(dknopik): review `Arc` situation /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a `SingleAttestation` with its subnet id. @@ -417,7 +426,8 @@ impl PubsubMessage { }; Ok(Self::PartialDataColumnSidecar(Box::new(( *id, - data_column.into(), + Arc::new(data_column), + None, )))) } other => Err(format!("Partial message unsupported for topic: {other}")), @@ -436,7 +446,13 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::PartialDataColumnSidecar(data) => { - return EncodedPubsubMessage::PartialDataColumnSidecarMessage(data.1.clone()); + let sidecar = &data.1; + let eager_cells = &data.2; + let mut message = PartialDataColumnSidecarMessage::new(sidecar.clone()); + if let Some(eager_cells) = eager_cells { + message.eagerly_send(eager_cells); + } + return EncodedPubsubMessage::PartialDataColumnSidecarMessage(message); } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), @@ -481,15 +497,9 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::PartialDataColumnSidecar(data) => write!( f, "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", - data.1.partial_column.block_root, - data.1.partial_column.index, - hex::encode( - data.1 - .partial_column - .sidecar - .cells_present_bitmap - .as_slice() - ), + data.1.block_root, + data.1.index, + hex::encode(data.1.sidecar.cells_present_bitmap.as_slice()), ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 1ddfe8c6580..9e0fd816373 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1307,7 +1307,8 @@ impl NetworkBeaconProcessor { partial.index(), &cloned_self.chain.spec, ), - partial.column.clone().into(), + partial.column.clone(), + None, ))) }) .chain(merged_data.completed_columns.into_iter().flat_map(|full| { @@ -1316,7 +1317,8 @@ impl NetworkBeaconProcessor { [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*full).clone().into_partial().column.clone().into(), + (*full).clone().into_partial().column.clone(), + None, ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), ] diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 535f587514b..5ac99d3e612 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -823,7 +823,8 @@ impl NetworkBeaconProcessor { c.index(), &self_cloned.chain.spec, ), - c.into_partial().into_inner().column.clone().into(), + c.into_partial().into_inner().column.clone(), + None, ))) }) .collect(), diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index a39bfb8c4e3..2659a2e8283 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -385,12 +385,12 @@ impl Router { ) } PubsubMessage::PartialDataColumnSidecar(data) => { - let (_, column_sidecar) = *data; + let (_, column_sidecar, _) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor .send_gossip_partial_data_column_sidecar( peer_id, - column_sidecar.partial_column, + column_sidecar, timestamp_now(), ), ) From 870fa0d2522fccdab45475128b7b6b8895a4e34b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 10 Nov 2025 17:04:41 +0100 Subject: [PATCH 029/146] experimental libp2p changes --- Cargo.lock | 46 ++++---- .../src/service/gossip_cache.rs | 82 +++++--------- .../lighthouse_network/src/service/mod.rs | 78 ++++--------- .../lighthouse_network/src/types/partial.rs | 105 +++++++++--------- .../lighthouse_network/src/types/pubsub.rs | 4 +- 5 files changed, 127 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4101a2c224a..77c26f9054c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 2.0.100", + "syn 1.0.109", ] [[package]] @@ -5143,7 +5143,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "bytes", "either", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5185,7 +5185,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "async-trait", "futures", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "either", @@ -5304,7 +5304,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "hickory-proto", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "libp2p-core", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5413,7 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "fnv", @@ -5434,7 +5434,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "heck 0.5.0", "quote", @@ -5444,7 +5444,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-rustls", @@ -5477,7 +5477,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "futures", @@ -6163,7 +6163,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "bytes", "futures", @@ -7382,7 +7382,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -8195,7 +8195,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 8259fa0701d..120b9e6c245 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -1,27 +1,23 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::marker::PhantomData; use std::pin::Pin; use std::task::{Context, Poll}; use std::time::Duration; use crate::GossipTopic; -use crate::types::{EncodedPubsubMessage, GossipKind}; +use crate::types::GossipKind; use tokio_util::time::delay_queue::{DelayQueue, Key}; -use types::EthSpec; - -type TopicMsg = (Key, EncodedPubsubMessage); /// Store of gossip messages that we failed to publish and will try again later. By default, all /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. -pub struct GossipCache { - /// Expire timeouts for each topic-msgid triple. +pub struct GossipCache { + /// Expire timeouts for each topic-msg pair. expirations: DelayQueue<(GossipTopic, Vec)>, /// Messages cached for each topic. - topic_msgs: HashMap, TopicMsg>>, + topic_msgs: HashMap, Key>>, /// Timeout for blocks. beacon_block: Option, /// Timeout for blobs. @@ -51,7 +47,7 @@ pub struct GossipCache { } #[derive(Default)] -pub struct GossipCacheBuilder { +pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, @@ -79,11 +75,10 @@ pub struct GossipCacheBuilder { light_client_finality_update: Option, /// Timeout for light client optimistic updates. light_client_optimistic_update: Option, - _phantom: PhantomData, } #[allow(dead_code)] -impl GossipCacheBuilder { +impl GossipCacheBuilder { /// By default, all timeouts all disabled. Setting a default timeout will enable all timeout /// that are not already set. pub fn default_timeout(mut self, timeout: Duration) -> Self { @@ -96,12 +91,6 @@ impl GossipCacheBuilder { self } - /// Timeout for data column sidecars. - pub fn data_column_sidecar(mut self, timeout: Duration) -> Self { - self.data_column_sidecar = Some(timeout); - self - } - /// Timeout for aggregate attestations. pub fn aggregates_timeout(mut self, timeout: Duration) -> Self { self.aggregates = Some(timeout); @@ -162,7 +151,7 @@ impl GossipCacheBuilder { self } - pub fn build(self) -> GossipCache { + pub fn build(self) -> GossipCache { let GossipCacheBuilder { default_timeout, beacon_block, @@ -178,7 +167,6 @@ impl GossipCacheBuilder { bls_to_execution_change, light_client_finality_update, light_client_optimistic_update, - .. } = self; GossipCache { expirations: DelayQueue::default(), @@ -200,15 +188,15 @@ impl GossipCacheBuilder { } } -impl GossipCache { +impl GossipCache { /// Get a builder of a `GossipCache`. Topic kinds for which no timeout is defined will be /// ignored if added in `insert`. - pub fn builder() -> GossipCacheBuilder { + pub fn builder() -> GossipCacheBuilder { GossipCacheBuilder::default() } // Insert a message to be sent later. - pub fn insert(&mut self, topic: GossipTopic, id: Vec, data: EncodedPubsubMessage) { + pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, @@ -231,54 +219,45 @@ impl GossipCache { .topic_msgs .entry(topic.clone()) .or_default() - .entry(id.clone()) + .entry(data.clone()) { - Entry::Occupied(key) => self.expirations.reset(&key.get().0, expire_timeout), + Entry::Occupied(key) => self.expirations.reset(key.get(), expire_timeout), Entry::Vacant(entry) => { - let key = self.expirations.insert((topic, id), expire_timeout); - entry.insert((key, data)); + let key = self.expirations.insert((topic, data), expire_timeout); + entry.insert(key); } } } - // Get the registered messages for this topic, removing them - pub fn retrieve_all( - &mut self, - topic: &GossipTopic, - ) -> Option> + '_> { - self.topic_msgs.remove(topic).map(|msgs| { - msgs.into_values().map(|(key, msg)| { - self.expirations.remove(&key); - msg - }) - }) - } - - // Get a registered message, NOT removing it. - pub fn retrieve_one(&self, topic: &GossipTopic, id: &[u8]) -> Option<&EncodedPubsubMessage> { - self.topic_msgs - .get(topic) - .and_then(|msgs| msgs.get(id)) - .map(|(_, msg)| msg) + // Get the registered messages for this topic. + pub fn retrieve(&mut self, topic: &GossipTopic) -> Option> + '_> { + if let Some(msgs) = self.topic_msgs.remove(topic) { + for (_, key) in msgs.iter() { + self.expirations.remove(key); + } + Some(msgs.into_keys()) + } else { + None + } } } -impl futures::stream::Stream for GossipCache { +impl futures::stream::Stream for GossipCache { type Item = Result; // We don't care to retrieve the expired data. fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.expirations.poll_expired(cx) { Poll::Ready(Some(expired)) => { let expected_key = expired.key(); - let (topic, id) = expired.into_inner(); + let (topic, data) = expired.into_inner(); let topic_msg = self.topic_msgs.get_mut(&topic); debug_assert!( topic_msg.is_some(), "Topic for registered message is not present." ); if let Some(msgs) = topic_msg { - let key_and_data = msgs.remove(&id); - debug_assert_eq!(key_and_data.map(|(key, _)| key), Some(expected_key)); + let key = msgs.remove(&data); + debug_assert_eq!(key, Some(expected_key)); if msgs.is_empty() { // no more messages for this topic. self.topic_msgs.remove(&topic); @@ -296,11 +275,10 @@ impl futures::stream::Stream for GossipCache { mod tests { use super::*; use futures::stream::StreamExt; - use types::MainnetEthSpec; #[tokio::test] async fn test_stream() { - let mut cache = GossipCache::::builder() + let mut cache = GossipCache::builder() .default_timeout(Duration::from_millis(300)) .build(); let test_topic = GossipTopic::new( @@ -308,7 +286,7 @@ mod tests { crate::types::GossipEncoding::SSZSnappy, [0u8; 4], ); - cache.insert(test_topic, vec![], EncodedPubsubMessage::Full(vec![])); + cache.insert(test_topic, vec![]); tokio::time::sleep(Duration::from_millis(300)).await; while cache.next().await.is_some() {} assert!(cache.expirations.is_empty()); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 7672782fb81..0eee3cfca81 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -23,7 +23,7 @@ use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, Partial, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; @@ -34,7 +34,6 @@ use libp2p::upnp::tokio::Behaviour as Upnp; use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; -use sha2::{Digest, Sha256}; use ssz::Decode; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; @@ -164,7 +163,7 @@ pub struct Network { score_settings: PeerScoreSettings, /// The interval for updating gossipsub scores update_gossipsub_scores: tokio::time::Interval, - gossip_cache: GossipCache, + gossip_cache: GossipCache, /// This node's PeerId. pub local_peer_id: PeerId, } @@ -259,7 +258,6 @@ impl Network { // .signed_contribution_and_proof_timeout(timeout) // Do not retry // .sync_committee_message_timeout(timeout) // Do not retry .bls_to_execution_change_timeout(half_epoch * 2) - .data_column_sidecar(slot_duration) .build() }; @@ -854,7 +852,17 @@ impl Network { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { let message_data = message.encode(); - let result = message_data.do_publish(self.gossipsub_mut(), topic.clone().into()); + let gossipsub = self.gossipsub_mut(); + let publish_topic: Topic = topic.clone().into(); + let (cache, result) = match message_data { + EncodedPubsubMessage::Full(bytes) => ( + Some(bytes.clone()), + gossipsub.publish(publish_topic, bytes).map(|_| ()), + ), + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { + (None, gossipsub.publish_partial(publish_topic, partial)) + } + }; if let Err(e) = result { match e { PublishError::Duplicate => { @@ -893,21 +901,10 @@ impl Network { } if let PublishError::NoPeersSubscribedToTopic = e - && let EncodedPubsubMessage::Full(bytes) = &message_data + && let Some(bytes) = cache { - let id = Sha256::digest(bytes)[..].to_vec(); - self.gossip_cache.insert(topic, id, message_data); + self.gossip_cache.insert(topic, bytes); } - } else if let EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) = - &message_data - { - let id = partial.group_id().as_ref().to_vec(); - debug!( - kind = %topic.kind(), - group_id = hex::encode(&id), - "Adding partial message to cache" - ); - self.gossip_cache.insert(topic, id, message_data); } } } @@ -1324,7 +1321,6 @@ impl Network { message, metadata, } => { - let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { debug!( @@ -1354,38 +1350,6 @@ impl Network { debug!(%propagation_source, "Got no metadata") } - // TODO(dknopik): maybe do this after validation? - if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { - // We do not bother checking here if that cached msg contains relevant data, - // as gossipsub will do that anyway. - let result = cached_partial.do_publish( - &mut self.swarm.behaviour_mut().gossipsub, - topic.clone().into(), - ); - match result { - Ok(_) => { - debug!( - group_id = hex_group_id, - %topic, %propagation_source, "Replied with cached partial message" - ) - } - Err(err) => { - warn!( - group_id = hex_group_id, - %topic, - %propagation_source, - ?err, - "Failed to reply with cached partial message" - ); - } - } - } else { - debug!( - group_id = hex_group_id, - %topic, %propagation_source, "Did not reply to partial message" - ); - } - if let Some(message) = message { match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { Err(error) => { @@ -1428,13 +1392,15 @@ impl Network { .add_subscription(&peer_id, subnet_id); } // Try to send the cached messages for this topic - if let Some(msgs) = self.gossip_cache.retrieve_all(&topic) { + if let Some(msgs) = self.gossip_cache.retrieve(&topic) { for data in msgs { let topic_str: &str = topic.kind().as_ref(); - match data.do_publish( - &mut self.swarm.behaviour_mut().gossipsub, - topic.clone().into(), - ) { + match self + .swarm + .behaviour_mut() + .gossipsub + .publish(Topic::from(topic.clone()), data) + { Ok(_) => { debug!(topic = topic_str, "Gossip message published on retry"); metrics::inc_counter_vec( diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index da712e4daf8..f61604d7694 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,6 +1,7 @@ -use gossipsub::partial::{Metadata, PublishAction}; +use gossipsub::partial::Metadata; use gossipsub::{Partial, PartialMessageError}; use ssz::{Decode, Encode}; +use std::cmp::Ordering; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; @@ -9,12 +10,14 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, + metadata: CellBitmapMetadata, send_eager: Option>, } impl PartialDataColumnSidecarMessage { pub fn new(partial_column: Arc>) -> Self { PartialDataColumnSidecarMessage { + metadata: partial_column.sidecar.cells_present_bitmap.clone().into(), partial_column, send_eager: None, } @@ -45,85 +48,77 @@ struct SendEager { } #[derive(Debug, Clone, PartialEq, Eq)] -struct CellBitmapMetadata { +pub struct CellBitmapMetadata { bitmap: CellBitmap, - encoded: Vec, } impl Metadata for CellBitmapMetadata { - fn as_slice(&self) -> &[u8] { - &self.encoded + fn decode(bytes: &[u8]) -> Result + where + Self: Sized, + { + Ok(CellBitmapMetadata { + bitmap: CellBitmap::::from_ssz_bytes(bytes) + .map_err(|_| PartialMessageError::InvalidFormat)?, + }) } - fn update(&mut self, data: &[u8]) -> Result { - let data = CellBitmap::::from_ssz_bytes(data) - .map_err(|_| PartialMessageError::InvalidFormat)?; - if data.len() != self.bitmap.len() { - return Err(PartialMessageError::OutOfRange); + fn compare(&self, other: &Self) -> Option { + let self_subset = self.bitmap.is_subset(&other.bitmap); + let other_subset = other.bitmap.is_subset(&self.bitmap); + match (self_subset, other_subset) { + (true, true) => Some(Ordering::Equal), + (true, false) => Some(Ordering::Less), + (false, true) => Some(Ordering::Greater), + (false, false) => None, } - let new_bitmap = self.bitmap.union(&data); + } + + fn encode(&self) -> Vec { + self.bitmap.as_ssz_bytes() + } + + fn update(&mut self, data: &Self) -> Result { + let new_bitmap = self.bitmap.union(&data.bitmap); if self.bitmap == new_bitmap { return Ok(false); } self.bitmap = new_bitmap; - self.encoded = self.bitmap.as_ssz_bytes(); Ok(true) } } impl From> for CellBitmapMetadata { - fn from(value: CellBitmap) -> Self { - Self { - encoded: value.as_ssz_bytes(), - bitmap: value, - } + fn from(bitmap: CellBitmap) -> Self { + Self { bitmap } } } impl Partial for PartialDataColumnSidecarMessage { - fn group_id(&self) -> impl AsRef<[u8]> { - &self.partial_column.block_root + type Metadata = CellBitmapMetadata; + + fn group_id(&self) -> Vec { + self.partial_column.block_root.to_vec() } - fn parts_metadata(&self) -> impl AsRef<[u8]> { - self.partial_column - .sidecar - .cells_present_bitmap - .as_ssz_bytes() + fn metadata(&self) -> &Self::Metadata { + &self.metadata } fn partial_message_bytes_from_metadata( &self, - metadata: Option>, - ) -> Result { - match metadata { - None => { - // Send the eager message if any - match &self.send_eager { - None => Ok(PublishAction::NothingToSend), - Some(send) => Ok(PublishAction::Send { - message: send.data.clone(), - metadata: Box::new(send.metadata.clone()), - }), - } - } - Some(metadata) => { - let peer_has = CellBitmap::::from_ssz_bytes(metadata.as_ref()) - .map_err(|_| PartialMessageError::InvalidFormat)?; - if peer_has == self.partial_column.sidecar.cells_present_bitmap { - return Ok(PublishAction::SameMetadata); - } - - let Some(send) = self.partial_column.sidecar.with_missing_cells(&peer_has) else { - return Ok(PublishAction::NothingToSend); - }; - - let new_metadata = peer_has.union(&send.cells_present_bitmap); - Ok(PublishAction::Send { - message: send.as_ssz_bytes().to_vec(), - metadata: Box::new(CellBitmapMetadata::::from(new_metadata)), - }) - } - } + metadata: &Self::Metadata, + ) -> Result>, PartialMessageError> { + Ok(self + .partial_column + .sidecar + .with_missing_cells(&metadata.bitmap) + .map(|message| message.as_ssz_bytes())) + } + + fn data_for_eager_push( + &self, + ) -> Result, Self::Metadata)>, PartialMessageError> { + Ok(self.send_eager.clone().map(|eager| (eager.data, eager.metadata))) } } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index db475213ab6..54eb76d4ae5 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -546,13 +546,13 @@ pub enum EncodedPubsubMessage { impl EncodedPubsubMessage { pub fn do_publish( - &self, + self, gossipsub: &mut Gossipsub, topic: IdentTopic, ) -> Result<(), PublishError> { match self { EncodedPubsubMessage::Full(bytes) => { - gossipsub.publish(topic, bytes.clone()).map(|_| ()) + gossipsub.publish(topic, bytes).map(|_| ()) } EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { gossipsub.publish_partial(topic, partial) From 995344eacdfbd48e9f465973fb3238ce78ff02ee Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 12 Nov 2025 16:21:35 +0100 Subject: [PATCH 030/146] joaos libp2p changes --- Cargo.lock | 46 ++++---- .../lighthouse_network/src/types/partial.rs | 105 ++++++++++-------- 2 files changed, 79 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77c26f9054c..c33d7bdec17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -5143,7 +5143,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "bytes", "either", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5185,7 +5185,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "async-trait", "futures", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "either", @@ -5304,7 +5304,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "hickory-proto", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "libp2p-core", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5413,7 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "fnv", @@ -5434,7 +5434,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "heck 0.5.0", "quote", @@ -5444,7 +5444,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-rustls", @@ -5477,7 +5477,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "futures", @@ -6163,7 +6163,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "bytes", "futures", @@ -7382,7 +7382,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -8195,7 +8195,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index f61604d7694..556dc147e9e 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,7 +1,6 @@ -use gossipsub::partial::Metadata; +use gossipsub::partial::{Metadata, PublishAction}; use gossipsub::{Partial, PartialMessageError}; use ssz::{Decode, Encode}; -use std::cmp::Ordering; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; @@ -10,14 +9,12 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, - metadata: CellBitmapMetadata, send_eager: Option>, } impl PartialDataColumnSidecarMessage { pub fn new(partial_column: Arc>) -> Self { PartialDataColumnSidecarMessage { - metadata: partial_column.sidecar.cells_present_bitmap.clone().into(), partial_column, send_eager: None, } @@ -48,77 +45,87 @@ struct SendEager { } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct CellBitmapMetadata { +struct CellBitmapMetadata { bitmap: CellBitmap, + encoded: Vec, } impl Metadata for CellBitmapMetadata { - fn decode(bytes: &[u8]) -> Result - where - Self: Sized, - { - Ok(CellBitmapMetadata { - bitmap: CellBitmap::::from_ssz_bytes(bytes) - .map_err(|_| PartialMessageError::InvalidFormat)?, - }) + fn as_slice(&self) -> &[u8] { + &self.encoded } - fn compare(&self, other: &Self) -> Option { - let self_subset = self.bitmap.is_subset(&other.bitmap); - let other_subset = other.bitmap.is_subset(&self.bitmap); - match (self_subset, other_subset) { - (true, true) => Some(Ordering::Equal), - (true, false) => Some(Ordering::Less), - (false, true) => Some(Ordering::Greater), - (false, false) => None, + fn update(&mut self, data: &[u8]) -> Result { + let data = CellBitmap::::from_ssz_bytes(data) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if data.len() != self.bitmap.len() { + return Err(PartialMessageError::OutOfRange); } - } - - fn encode(&self) -> Vec { - self.bitmap.as_ssz_bytes() - } - - fn update(&mut self, data: &Self) -> Result { - let new_bitmap = self.bitmap.union(&data.bitmap); + let new_bitmap = self.bitmap.union(&data); if self.bitmap == new_bitmap { return Ok(false); } self.bitmap = new_bitmap; + self.encoded = self.bitmap.as_ssz_bytes(); Ok(true) } } impl From> for CellBitmapMetadata { - fn from(bitmap: CellBitmap) -> Self { - Self { bitmap } + fn from(value: CellBitmap) -> Self { + Self { + encoded: value.as_ssz_bytes(), + bitmap: value, + } } } impl Partial for PartialDataColumnSidecarMessage { - type Metadata = CellBitmapMetadata; - fn group_id(&self) -> Vec { - self.partial_column.block_root.to_vec() + self.partial_column.block_root.as_slice().to_vec() } - fn metadata(&self) -> &Self::Metadata { - &self.metadata - } - - fn partial_message_bytes_from_metadata( - &self, - metadata: &Self::Metadata, - ) -> Result>, PartialMessageError> { - Ok(self - .partial_column + fn metadata(&self) -> Vec { + self.partial_column .sidecar - .with_missing_cells(&metadata.bitmap) - .map(|message| message.as_ssz_bytes())) + .cells_present_bitmap + .as_ssz_bytes() } - fn data_for_eager_push( + fn partial_message_bytes_from_metadata( &self, - ) -> Result, Self::Metadata)>, PartialMessageError> { - Ok(self.send_eager.clone().map(|eager| (eager.data, eager.metadata))) + metadata: Option<&[u8]>, + ) -> Result { + match metadata { + None => Ok(PublishAction { + need: false, + send: self.send_eager.clone().map(|eager| { + ( + eager.data, + Box::new(eager.metadata) as Box, + ) + }), + }), + Some(metadata) => { + let peer_has = CellBitmap::::from_ssz_bytes(metadata) + .map_err(|_| PartialMessageError::InvalidFormat)?; + let need = !peer_has.is_subset(&self.partial_column.sidecar.cells_present_bitmap); + + let send = self + .partial_column + .sidecar + .with_missing_cells(&peer_has) + .map(|sidecar| { + ( + sidecar.as_ssz_bytes(), + Box::new(CellBitmapMetadata::::from( + peer_has.union(&sidecar.cells_present_bitmap), + )) as Box, + ) + }); + + Ok(PublishAction { need, send }) + } + } } } From af1499cadea9e3364058c721a621483796f8e579 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 5 Dec 2025 11:14:06 +0100 Subject: [PATCH 031/146] fix test --- beacon_node/network/src/network_beacon_processor/tests.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index ef1c51425fa..5c6f00266d2 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -1,7 +1,6 @@ #![cfg(not(debug_assertions))] // Tests are too slow in debug. #![cfg(test)] -use crate::partial_data_column_cache::PartialDataColumnCache; use crate::{ network_beacon_processor::{ ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor, @@ -33,7 +32,6 @@ use lighthouse_network::{ types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}, }; use matches::assert_matches; -use parking_lot::Mutex; use slot_clock::SlotClock; use ssz_types::RuntimeVariableList; use std::collections::HashSet; @@ -268,7 +266,6 @@ impl TestRig { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send: beacon_processor_tx.clone(), duplicate_cache: duplicate_cache.clone(), - partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: harness.chain.clone(), network_tx, sync_tx, From 90266e7510fa9423a06be44fac5a028ddb9be212 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 8 Dec 2025 11:55:01 +0100 Subject: [PATCH 032/146] fix function path --- beacon_node/http_api/src/publish_blocks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index c09e23e5e6c..b863738110d 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -235,7 +235,7 @@ pub async fn publish_block>( })) .collect(); if !messages.is_empty() - && let Err(err) = crate::publish_pubsub_messages(&sender_clone, messages) + && let Err(err) = crate::utils::publish_pubsub_messages(&sender_clone, messages) { warn!(?err, "Publishing data after block publish") } From 4429de41ddf6c3f7df1526a0d32efbf4e6504538 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 10 Dec 2025 16:06:12 +0100 Subject: [PATCH 033/146] update libp2p --- Cargo.lock | 62 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 226fd82cd1c..5269bb6c2a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3118,7 +3118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4755,7 +4755,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5021,7 +5021,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "bytes", "either", @@ -5053,7 +5053,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5063,7 +5063,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5073,7 +5073,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "either", "fnv", @@ -5097,7 +5097,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "async-trait", "futures", @@ -5112,7 +5112,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5142,7 +5142,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "asynchronous-codec", "either", @@ -5161,9 +5161,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3104e13b51e4711ff5738caa1fb54467c8604c2e94d607e27745bcf709068774" +checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" dependencies = [ "asn1_der", "bs58 0.5.1", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "hickory-proto", @@ -5200,7 +5200,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "libp2p-core", @@ -5215,7 +5215,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "asynchronous-codec", "bytes", @@ -5233,7 +5233,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "asynchronous-codec", "bytes", @@ -5255,7 +5255,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "asynchronous-codec", "bytes", @@ -5270,7 +5270,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "futures-timer", @@ -5291,7 +5291,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "either", "fnv", @@ -5312,7 +5312,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "heck 0.5.0", "quote", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "futures-timer", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "futures-rustls", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "futures-timer", @@ -5369,7 +5369,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "either", "futures", @@ -6058,7 +6058,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "bytes", "futures", @@ -6323,7 +6323,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7180,7 +7180,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "asynchronous-codec", "bytes", @@ -7242,7 +7242,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7764,7 +7764,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7870,7 +7870,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" dependencies = [ "futures", "pin-project", @@ -8850,7 +8850,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10180,7 +10180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 7f190fe2428433729fc4c3f9af42e6efa54ff556 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 11 Dec 2025 14:26:04 +0100 Subject: [PATCH 034/146] do not eagerly publish all columns on proposal --- beacon_node/http_api/src/publish_blocks.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b863738110d..70691909833 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -569,12 +569,11 @@ fn publish_column_sidecars( .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); let column = (*data_col).clone().into_partial().column; - let all_cells = column.sidecar.cells_present_bitmap.clone(); [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, column, - Some(all_cells), + None, ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] From 2df27a144557fc978026feda2151ea3020732dbe Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 11 Dec 2025 14:26:04 +0100 Subject: [PATCH 035/146] update libp2p again --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5269bb6c2a5..a0a3c679547 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3118,7 +3118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4755,7 +4755,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5021,7 +5021,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "bytes", "either", @@ -5053,7 +5053,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5063,7 +5063,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5073,7 +5073,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "either", "fnv", @@ -5097,7 +5097,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "async-trait", "futures", @@ -5112,7 +5112,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5142,7 +5142,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "asynchronous-codec", "either", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "hickory-proto", @@ -5200,7 +5200,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "libp2p-core", @@ -5215,7 +5215,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "asynchronous-codec", "bytes", @@ -5233,7 +5233,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "asynchronous-codec", "bytes", @@ -5255,7 +5255,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "asynchronous-codec", "bytes", @@ -5270,7 +5270,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "futures-timer", @@ -5291,7 +5291,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "either", "fnv", @@ -5312,7 +5312,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "heck 0.5.0", "quote", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "futures-timer", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "futures-rustls", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "futures-timer", @@ -5369,7 +5369,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "either", "futures", @@ -6058,7 +6058,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "bytes", "futures", @@ -6323,7 +6323,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7180,7 +7180,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "asynchronous-codec", "bytes", @@ -7242,7 +7242,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7764,7 +7764,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7870,7 +7870,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#cdad5cb405bf8d1c0e2f96725e3747a25e094fb2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" dependencies = [ "futures", "pin-project", @@ -8850,7 +8850,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10180,7 +10180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From bd62eaa7232e1bdedc0e4356adb08ba845670fee Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 10:02:56 +0100 Subject: [PATCH 036/146] once again update libp2p --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0a3c679547..1b645f34969 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3118,7 +3118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4755,7 +4755,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5021,7 +5021,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "bytes", "either", @@ -5053,7 +5053,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5063,7 +5063,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5073,7 +5073,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "either", "fnv", @@ -5097,7 +5097,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "async-trait", "futures", @@ -5112,7 +5112,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5142,7 +5142,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "asynchronous-codec", "either", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "hickory-proto", @@ -5200,7 +5200,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "libp2p-core", @@ -5215,7 +5215,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "asynchronous-codec", "bytes", @@ -5233,7 +5233,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "asynchronous-codec", "bytes", @@ -5255,7 +5255,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "asynchronous-codec", "bytes", @@ -5270,7 +5270,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "futures-timer", @@ -5291,7 +5291,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "either", "fnv", @@ -5312,7 +5312,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "heck 0.5.0", "quote", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "futures-timer", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "futures-rustls", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "futures-timer", @@ -5369,7 +5369,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "either", "futures", @@ -6058,7 +6058,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "bytes", "futures", @@ -6323,7 +6323,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7180,7 +7180,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "asynchronous-codec", "bytes", @@ -7242,7 +7242,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7764,7 +7764,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7870,7 +7870,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#3de7fc0d2253bb0c824616606345e7d39896d02c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" dependencies = [ "futures", "pin-project", @@ -8850,7 +8850,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10180,7 +10180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From b725f58cf26b14a2dc6c91d1ea4afbbbdd663ba6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 12:36:08 +0100 Subject: [PATCH 037/146] fix import --- consensus/types/src/data/partial_data_column_sidecar.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index e13a0a8b408..56b77931f96 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -2,8 +2,9 @@ use crate::beacon_block_body::KzgCommitments; use crate::data::das_column::{CellWithMetadata, DasColumn}; use crate::data_column_sidecar::{Cell, DataColumn}; use crate::test_utils::TestRandom; -use crate::{AbstractExecPayload, ColumnIndex, DataColumnSidecar, context_deserialize}; +use crate::{AbstractExecPayload, ColumnIndex, DataColumnSidecar}; use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot}; +use context_deserialize::context_deserialize; use educe::Educe; use kzg::KzgProof; use serde::{Deserialize, Serialize}; From 79c17e797b654a3344200869ee56d4cc0ddd018b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 14:13:27 +0100 Subject: [PATCH 038/146] update for libp2p fix --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c2e5d9b4f0..a2a0bd677c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3132,7 +3132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4782,7 +4782,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5049,7 +5049,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "bytes", "either", @@ -5081,7 +5081,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5091,7 +5091,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5101,7 +5101,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "either", "fnv", @@ -5125,7 +5125,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "async-trait", "futures", @@ -5140,7 +5140,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "asynchronous-codec", "either", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "hickory-proto", @@ -5228,7 +5228,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "libp2p-core", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "asynchronous-codec", "bytes", @@ -5261,7 +5261,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "asynchronous-codec", "bytes", @@ -5283,7 +5283,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "asynchronous-codec", "bytes", @@ -5298,7 +5298,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "futures-timer", @@ -5319,7 +5319,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "either", "fnv", @@ -5340,7 +5340,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "heck 0.5.0", "quote", @@ -5350,7 +5350,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "futures-timer", @@ -5365,7 +5365,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "futures-rustls", @@ -5383,7 +5383,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "futures-timer", @@ -5397,7 +5397,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "either", "futures", @@ -6090,7 +6090,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "bytes", "futures", @@ -6357,7 +6357,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7219,7 +7219,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "asynchronous-codec", "bytes", @@ -7281,7 +7281,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7803,7 +7803,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7909,7 +7909,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6e0c2a55d8cdd9a740aadd6df05030eecc2758d9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" dependencies = [ "futures", "pin-project", @@ -8904,7 +8904,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10243,7 +10243,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From 96b391f96b2f5b87d8daa78a4ca622dd1092b603 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 14:26:11 +0100 Subject: [PATCH 039/146] Fix inverted condition --- consensus/types/src/data/das_column.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/types/src/data/das_column.rs b/consensus/types/src/data/das_column.rs index bbd47b4e6f6..9445ee62558 100644 --- a/consensus/types/src/data/das_column.rs +++ b/consensus/types/src/data/das_column.rs @@ -38,9 +38,9 @@ pub trait DasColumn: Clone { fn iter(&self) -> impl Iterator>>; fn compare>(&self, rhs: &C) -> ColumnComparison { - if self.slot() == rhs.slot() - && self.index() == rhs.index() - && self.block_root() == rhs.block_root() + if self.slot() != rhs.slot() + || self.index() != rhs.index() + || self.block_root() != rhs.block_root() { return ColumnComparison::DifferentColumns; } From 4b4f92e0340872bc22fd8494d25b7fb5e66c4f7a Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 14:26:11 +0100 Subject: [PATCH 040/146] Another update for libp2p to fix received msgs --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2a0bd677c0..37dad33a707 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3132,7 +3132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4782,7 +4782,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5049,7 +5049,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "bytes", "either", @@ -5081,7 +5081,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5091,7 +5091,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5101,7 +5101,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "either", "fnv", @@ -5125,7 +5125,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "async-trait", "futures", @@ -5140,7 +5140,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "asynchronous-codec", "either", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "hickory-proto", @@ -5228,7 +5228,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "libp2p-core", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "asynchronous-codec", "bytes", @@ -5261,7 +5261,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "asynchronous-codec", "bytes", @@ -5283,7 +5283,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "asynchronous-codec", "bytes", @@ -5298,7 +5298,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "futures-timer", @@ -5319,7 +5319,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "either", "fnv", @@ -5340,7 +5340,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "heck 0.5.0", "quote", @@ -5350,7 +5350,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "futures-timer", @@ -5365,7 +5365,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "futures-rustls", @@ -5383,7 +5383,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "futures-timer", @@ -5397,7 +5397,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "either", "futures", @@ -6090,7 +6090,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "bytes", "futures", @@ -6357,7 +6357,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7219,7 +7219,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "asynchronous-codec", "bytes", @@ -7281,7 +7281,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7803,7 +7803,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7909,7 +7909,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#62a9fc41146801ac230f93d3c9e69b36300567cb" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" dependencies = [ "futures", "pin-project", @@ -8904,7 +8904,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10243,7 +10243,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 5b2ee006e93e7f521a95f7fc9f7250a4058d29af Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 19:38:40 +0100 Subject: [PATCH 041/146] fix kzg verification --- beacon_node/beacon_chain/src/kzg_utils.rs | 14 ++++---------- .../network_beacon_processor/gossip_methods.rs | 2 +- .../src/data/partial_data_column_sidecar.rs | 17 ++++++++--------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index b3fc8dadd32..e4ae778c6c8 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -72,17 +72,11 @@ where return Err((Some(col_index), KzgError::KzgVerificationFailed)); } - for cell in data_column.column() { - cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); + for data in data_column.iter().flatten() { + cells.push(ssz_cell_to_crypto_cell::(data.cell).map_err(|e| (Some(col_index), e))?); column_indices.push(col_index); - } - - for &proof in data_column.kzg_proofs() { - proofs.push(Bytes48::from(proof)); - } - - for &commitment in data_column.kzg_commitments() { - commitments.push(Bytes48::from(commitment)); + proofs.push(Bytes48::from(*data.proof)); + commitments.push(Bytes48::from(*data.commitment)); } let expected_len = column_indices.len(); diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9e0fd816373..8ebe499b2f0 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -980,7 +980,7 @@ impl NetworkBeaconProcessor { %slot, %block_root, %index, - "Could not verify column sidecar for gossip. Rejecting the column sidecar" + "Could not verify partial column for gossip. Rejecting the column sidecar" ); // Prevent recurring behaviour by penalizing the peer slightly. self.gossip_penalize_peer( diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 56b77931f96..4d4ee769a8d 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -345,16 +345,15 @@ impl DasColumn for VerifiablePartialDataColumn { let mut present_iterator = sidecar .column .iter() - .zip(self.kzg_commitments.iter()) - .zip(sidecar.kzg_proofs.iter()) - .map(|((cell, commitment), proof)| CellWithMetadata { - cell, - commitment, - proof, - }); - sidecar.cells_present_bitmap.iter().map(move |present| { + .zip(sidecar.kzg_proofs.iter()); + sidecar.cells_present_bitmap.iter().zip(self.kzg_commitments.iter()).map(move |(present, commitment)| { if present { - present_iterator.next() + let (cell, proof) = present_iterator.next()?; + Some(CellWithMetadata { + cell, + proof, + commitment, + }) } else { None } From 65854ef295ce2ab6abdd873de560c8f458af7bef Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 12 Dec 2025 20:33:43 +0100 Subject: [PATCH 042/146] actually dequeue partials --- beacon_node/beacon_processor/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index cc44c4cffc6..e583f67ba10 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -1067,6 +1067,8 @@ impl BeaconProcessor { Some(item) } else if let Some(item) = gossip_data_column_queue.pop() { Some(item) + } else if let Some(item) = gossip_partial_data_column_queue.pop() { + Some(item) } else if let Some(item) = column_reconstruction_queue.pop() { Some(item) // Check the priority 0 API requests after blocks and blobs, but before attestations. From 3484cf6ccfbd7bf668218be956c47f422f314967 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 17 Dec 2025 23:13:03 +0100 Subject: [PATCH 043/146] update libp2p --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37dad33a707..93d1f51f8e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3132,7 +3132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4782,7 +4782,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5049,7 +5049,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "bytes", "either", @@ -5081,7 +5081,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5091,7 +5091,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5101,7 +5101,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "either", "fnv", @@ -5125,7 +5125,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "async-trait", "futures", @@ -5140,7 +5140,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "asynchronous-codec", "either", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "hickory-proto", @@ -5228,7 +5228,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "libp2p-core", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "asynchronous-codec", "bytes", @@ -5261,7 +5261,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "asynchronous-codec", "bytes", @@ -5283,7 +5283,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "asynchronous-codec", "bytes", @@ -5298,7 +5298,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "futures-timer", @@ -5319,7 +5319,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "either", "fnv", @@ -5340,7 +5340,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "heck 0.5.0", "quote", @@ -5350,7 +5350,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "futures-timer", @@ -5365,7 +5365,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "futures-rustls", @@ -5383,7 +5383,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "futures-timer", @@ -5397,7 +5397,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "either", "futures", @@ -6090,7 +6090,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "bytes", "futures", @@ -6357,7 +6357,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7219,7 +7219,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "asynchronous-codec", "bytes", @@ -7281,7 +7281,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7803,7 +7803,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7909,7 +7909,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#7751293725f666e7efb75be865bb07ea8c43991e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" dependencies = [ "futures", "pin-project", @@ -8904,7 +8904,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10243,7 +10243,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From 002a5bc055da8c27fca21f522ee9ccdb160dcec4 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 6 Jan 2026 14:24:46 +0100 Subject: [PATCH 044/146] Cargo fmt --- beacon_node/http_api/src/publish_blocks.rs | 6 +--- .../src/data/partial_data_column_sidecar.rs | 31 ++++++++++--------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 70691909833..ca838e2feef 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -570,11 +570,7 @@ fn publish_column_sidecars( let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); let column = (*data_col).clone().into_partial().column; [ - PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - column, - None, - ))), + PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, column, None))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] }) diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 4d4ee769a8d..c6b08fac8cf 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -342,21 +342,22 @@ impl DasColumn for VerifiablePartialDataColumn { fn iter(&self) -> impl Iterator>> { let sidecar = &self.column.sidecar; - let mut present_iterator = sidecar - .column + let mut present_iterator = sidecar.column.iter().zip(sidecar.kzg_proofs.iter()); + sidecar + .cells_present_bitmap .iter() - .zip(sidecar.kzg_proofs.iter()); - sidecar.cells_present_bitmap.iter().zip(self.kzg_commitments.iter()).map(move |(present, commitment)| { - if present { - let (cell, proof) = present_iterator.next()?; - Some(CellWithMetadata { - cell, - proof, - commitment, - }) - } else { - None - } - }) + .zip(self.kzg_commitments.iter()) + .map(move |(present, commitment)| { + if present { + let (cell, proof) = present_iterator.next()?; + Some(CellWithMetadata { + cell, + proof, + commitment, + }) + } else { + None + } + }) } } From 88c9c080be8f82f63d105455ad6130fa6a7d9ef4 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Thu, 8 Jan 2026 08:59:09 -0800 Subject: [PATCH 045/146] Refactor --- beacon_node/beacon_chain/src/beacon_chain.rs | 138 ++++-- .../src/data_availability_checker.rs | 260 ++++++++--- .../overflow_lru_cache.rs | 156 ++----- .../src/data_column_verification.rs | 439 +++++++++++------- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 6 + .../beacon_chain/src/fetch_blobs/mod.rs | 74 ++- .../beacon_chain/src/fetch_blobs/tests.rs | 2 +- beacon_node/beacon_chain/src/kzg_utils.rs | 98 +++- beacon_node/beacon_chain/src/lib.rs | 1 + .../src/partial_data_column_assembler.rs | 234 ++++++++++ beacon_node/beacon_chain/src/test_utils.rs | 4 +- .../beacon_chain/tests/blob_verification.rs | 1 - .../beacon_chain/tests/block_verification.rs | 9 +- .../beacon_chain/tests/column_verification.rs | 1 - beacon_node/beacon_chain/tests/events.rs | 2 +- .../tests/payload_invalidation.rs | 4 - beacon_node/beacon_chain/tests/store_tests.rs | 3 - beacon_node/beacon_chain/tests/tests.rs | 1 - beacon_node/http_api/src/publish_blocks.rs | 90 ++-- .../gossip_methods.rs | 154 ++++-- .../src/network_beacon_processor/mod.rs | 30 +- .../network_beacon_processor/sync_methods.rs | 1 - .../src/network_beacon_processor/tests.rs | 7 +- beacon_node/network/src/sync/tests/lookups.rs | 2 +- beacon_node/network/src/sync/tests/range.rs | 1 - consensus/types/src/data/das_column.rs | 106 ----- .../types/src/data/data_column_sidecar.rs | 113 ++--- consensus/types/src/data/mod.rs | 1 - .../src/data/partial_data_column_sidecar.rs | 72 +-- testing/ef_tests/src/cases/fork_choice.rs | 13 +- 30 files changed, 1140 insertions(+), 883 deletions(-) create mode 100644 beacon_node/beacon_chain/src/partial_data_column_assembler.rs delete mode 100644 consensus/types/src/data/das_column.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 74a41595ff6..07039dcfdd0 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -23,9 +23,11 @@ use crate::chain_config::ChainConfig; use crate::custody_context::CustodyContextSsz; use crate::data_availability_checker::{ Availability, AvailabilityCheckError, AvailableBlock, AvailableBlockData, - DataAvailabilityChecker, DataColumnReconstructionResult, MergedData, + DataAvailabilityChecker, DataColumnReconstructionResult, +}; +use crate::data_column_verification::{ + GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumn, }; -use crate::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::events::ServerSentEventHandler; @@ -133,7 +135,6 @@ use tokio_stream::Stream; use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace, warn}; use tree_hash::TreeHash; use types::blob_sidecar::FixedBlobSidecarList; -use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::payload::BlockProductionVersion; @@ -2189,8 +2190,7 @@ impl BeaconChain { self: &Arc, data_column_sidecar: Arc>, subnet_id: DataColumnSubnetId, - ) -> Result>, GossipDataColumnError> - { + ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); GossipVerifiedDataColumn::new(data_column_sidecar, subnet_id, self).inspect(|_| { @@ -2202,14 +2202,11 @@ impl BeaconChain { pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, data_column_sidecar: Arc>, - ) -> Result< - GossipVerifiedDataColumn>, - GossipDataColumnError, - > { + ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); - GossipVerifiedDataColumn::new_partial(data_column_sidecar, self).inspect(|_| { + GossipVerifiedPartialDataColumn::new(data_column_sidecar, self).inspect(|_| { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); }) } @@ -2922,7 +2919,6 @@ impl BeaconChain { notify_execution_layer, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await { @@ -3054,12 +3050,12 @@ impl BeaconChain { /// Cache the data columns in the processing cache, process it, then evict it from the cache if it was /// imported or errors. + /// Only accepts full columns. Partials are handled via PartialDataColumnAssembler. #[instrument(skip_all, level = "debug")] - pub async fn process_gossip_data_columns>( + pub async fn process_gossip_data_columns( self: &Arc, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, - data_publish_fn: impl FnOnce(MergedData), ) -> Result { let Ok((slot, block_root)) = data_columns .iter() @@ -3092,11 +3088,62 @@ impl BeaconChain { block_root, data_columns, publish_fn, - data_publish_fn, ) .await } + /// Process a gossip-verified partial data column by attempting to merge it in the assembler. + /// Returns the merge result which indicates if a column was completed. + #[instrument(skip_all, level = "debug")] + pub async fn process_gossip_partial_data_column( + self: &Arc, + verified_partial: GossipVerifiedPartialDataColumn, + ) -> Result, BlockError> + { + let block_root = verified_partial.block_root(); + let slot = verified_partial.slot(); + + // If this block has already been imported to forkchoice it must have been available + if self + .canonical_head + .fork_choice_read_lock() + .contains_block(&block_root) + { + return Err(BlockError::DuplicateFullyImported(block_root)); + } + + // Merge the partial into the assembler + let partial_arc = verified_partial.clone_data_column(); + let merge_result = self + .data_availability_checker + .partial_assembler() + .merge_partial(block_root, partial_arc) + .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; + + // If we completed a column, process it through the DA checker + if let crate::partial_data_column_assembler::PartialMergeResult::Completed { + ref full_column, + .. + } = merge_result + { + let availability = self + .data_availability_checker + .put_full_data_columns(block_root, vec![full_column.clone()])?; + + // If we achieved full availability, process it + if let Availability::Available(available_block) = availability { + self.process_availability( + slot, + Availability::Available(available_block), + || Ok(()), + ) + .await?; + } + } + + Ok(merge_result) + } + /// Cache the blobs in the processing cache, process it, then evict it from the cache if it was /// imported or errors. #[instrument(skip_all, level = "debug")] @@ -3158,12 +3205,15 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CustodyColumns(columns) => { + EngineGetBlobsOutput::CompleteColumns(columns) => { self.emit_sse_data_column_sidecar_events( &block_root, - columns.iter().map(|column| column.as_data_column()), + columns.iter().map(|column| column.as_ref()), ); } + EngineGetBlobsOutput::PartialColumns(_) => { + // Partials don't emit SSE events until they complete + } } self.check_engine_blobs_availability_and_import(slot, block_root, engine_get_blobs_output) @@ -3191,14 +3241,11 @@ impl BeaconChain { } } - fn emit_sse_data_column_sidecar_events<'a, I, C>( + fn emit_sse_data_column_sidecar_events<'a>( self: &Arc, block_root: &Hash256, - data_columns_iter: I, - ) where - I: Iterator, - C: DasColumn + 'a, - { + data_columns_iter: impl Iterator>, + ) { if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_data_column_sidecar_subscribers() { @@ -3207,7 +3254,7 @@ impl BeaconChain { .get_data_columns(*block_root) .unwrap_or_default(); - let new_data_columns_indices = data_columns_iter.map(|c| c.index()).collect::>(); + let new_data_columns_indices = data_columns_iter.map(|c| c.index).collect::>(); for data_column in imported_data_columns .into_iter() @@ -3356,7 +3403,6 @@ impl BeaconChain { notify_execution_layer: NotifyExecutionLayer, block_source: BlockImportSource, publish_fn: impl FnOnce() -> Result<(), BlockError>, - data_publish_fn: impl FnOnce(MergedData), ) -> Result { let block_slot = unverified_block.block().slot(); @@ -3426,8 +3472,7 @@ impl BeaconChain { self.import_available_block(Box::new(block)).await } ExecutedBlock::AvailabilityPending(block) => { - self.check_block_availability_and_import(block, data_publish_fn) - .await + self.check_block_availability_and_import(block).await } } }; @@ -3537,12 +3582,9 @@ impl BeaconChain { async fn check_block_availability_and_import( self: &Arc, block: AvailabilityPendingExecutedBlock, - data_publish_fn: impl FnOnce(MergedData), ) -> Result { let slot = block.block.slot(); - let availability = self - .data_availability_checker - .put_executed_block(block, data_publish_fn)?; + let availability = self.data_availability_checker.put_executed_block(block)?; self.process_availability(slot, availability, || Ok(())) .await } @@ -3567,23 +3609,24 @@ impl BeaconChain { /// Checks if the provided data column can make any cached blocks available, and imports immediately /// if so, otherwise caches the data column in the data availability checker. - async fn check_gossip_data_columns_availability_and_import>( + /// Check gossip data columns for availability and import. Only accepts full columns. + /// Partials are handled separately via PartialDataColumnAssembler. + async fn check_gossip_data_columns_availability_and_import( self: &Arc, slot: Slot, block_root: Hash256, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, - data_publish_fn: impl FnOnce(MergedData), ) -> Result { if let Some(slasher) = self.slasher.as_ref() { - for header in data_columns.iter().filter_map(|c| c.signed_block_header()) { + for header in data_columns.iter().map(|c| c.signed_block_header()) { slasher.accept_block_header(header.clone()); } } let availability = self .data_availability_checker - .put_gossip_verified_data_columns(block_root, slot, data_columns, data_publish_fn)?; + .put_gossip_verified_data_columns(block_root, slot, data_columns)?; self.process_availability(slot, availability, publish_fn) .await @@ -3654,13 +3697,21 @@ impl BeaconChain { self.data_availability_checker .put_kzg_verified_blobs(block_root, blobs)? } - EngineGetBlobsOutput::CustodyColumns(data_columns) => { + EngineGetBlobsOutput::CompleteColumns(data_columns) => { self.check_data_column_sidecar_header_signature_and_slashability( block_root, - data_columns.iter().map(|c| c.as_data_column()), + data_columns.iter().map(|c| c.as_ref()), )?; self.data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, data_columns, |_| ())? + .put_full_data_columns(block_root, data_columns)? + } + EngineGetBlobsOutput::PartialColumns(_partials) => { + // Partials have already been initialized in the assembler during fetch_and_process + // They don't go through put_kzg_verified - they're assembled via gossip + // Return MissingComponents to indicate we're waiting for more data + return Ok(AvailabilityProcessingStatus::MissingComponents( + slot, block_root, + )); } }; @@ -3693,13 +3744,10 @@ impl BeaconChain { .await } - fn check_data_column_sidecar_header_signature_and_slashability< - 'a, - C: DasColumn + 'a, - >( + fn check_data_column_sidecar_header_signature_and_slashability<'a>( self: &Arc, block_root: Hash256, - custody_columns: impl IntoIterator, + custody_columns: impl IntoIterator>, ) -> Result<(), BlockError> { let mut slashable_cache = self.observed_slashable.write(); // Process all unique block headers - previous logic assumed all headers were identical and @@ -3707,7 +3755,7 @@ impl BeaconChain { // from RPC. for header in custody_columns .into_iter() - .filter_map(|c| c.signed_block_header()) + .map(|c| &c.signed_block_header) .unique() { // Return an error if *any* header signature is invalid, we do not want to import this diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index c6a479ac7c1..13be9eb111d 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -7,6 +7,7 @@ use crate::block_verification_types::{ use crate::data_availability_checker::overflow_lru_cache::{ DataAvailabilityCheckerInner, ReconstructColumnsDecision, }; +use crate::partial_data_column_assembler::PartialDataColumnAssembler; use crate::{ BeaconChain, BeaconChainTypes, BeaconStore, BlockProcessStatus, CustodyContext, metrics, }; @@ -21,8 +22,8 @@ use task_executor::TaskExecutor; use tracing::{debug, error, instrument}; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{ - BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecar, DataColumnSidecarList, Epoch, - EthSpec, Hash256, SignedBeaconBlock, Slot, + BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecarList, Epoch, EthSpec, Hash256, + SignedBeaconBlock, Slot, }; mod error; @@ -39,7 +40,7 @@ use crate::metrics::{ }; use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; -use types::das_column::{ColumnComparison, DasColumn}; +use types::DataColumnSidecar; use types::non_zero_usize::new_non_zero_usize; use types::partial_data_column_sidecar::VerifiablePartialDataColumn; @@ -84,6 +85,7 @@ const STATE_LRU_CAPACITY_NON_ZERO: NonZeroUsize = new_non_zero_usize(32); pub struct DataAvailabilityChecker { complete_blob_backfill: bool, availability_cache: Arc>, + partial_assembler: Arc>, slot_clock: T::SlotClock, kzg: Arc, custody_context: Arc>, @@ -134,8 +136,12 @@ impl DataAvailabilityChecker { custody_context.clone(), spec.clone(), )?; + let partial_assembler = Arc::new(PartialDataColumnAssembler::new( + OVERFLOW_LRU_CAPACITY_NON_ZERO, + )); Ok(Self { complete_blob_backfill, + partial_assembler, availability_cache: Arc::new(inner), slot_clock, kzg, @@ -148,6 +154,10 @@ impl DataAvailabilityChecker { &self.custody_context } + pub fn partial_assembler(&self) -> &Arc> { + &self.partial_assembler + } + /// Checks if the block root is currently in the availability cache awaiting import because /// of missing components. /// @@ -180,44 +190,77 @@ impl DataAvailabilityChecker { }) } - /// Check if the (potentially partial) data column is in the availability cache. + /// Check if the full data column is in the availability cache. /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells - /// otherwise. - pub fn determine_missing_cells>( + /// (cells in the incoming column that aren't cached yet) otherwise. + pub fn determine_missing_cells_full( &self, block_root: &Hash256, - data_column: &C, + data_column: &DataColumnSidecar, ) -> Option> { - fn do_compare, C2: DasColumn>( - cached: &C1, - data_column: &C2, - ) -> Option> { - match cached.compare(data_column) { - ColumnComparison::Equal => Some(vec![]), - ColumnComparison::MissingCells { missing_in_lhs, .. } => Some(missing_in_lhs), - comparison => { - debug!(?comparison, "Unexpected columns comparison"); - None - } - } + let column_index = data_column.index; + let cell_count = data_column.kzg_commitments.len(); + + // Check DA checker cache first - if we have a full column cached, nothing is missing + if self + .availability_cache + .peek_pending_components(block_root, |components| { + components + .and_then(|c| c.get_cached_data_column(column_index)) + .is_some() + }) + { + return Some(vec![]); } - self.availability_cache + // Check assembler for partial columns + if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { + // Compare: find which cells from incoming full column aren't in cached partial + return compare_full_to_partial(cell_count, cached_partial.as_ref()); + } + + // No cached data, all cells are "missing" (new data we want) + Some((0..cell_count).collect()) + } + + /// Check if the partial data column is in the availability cache. + /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells + /// (cells in the incoming column that aren't cached yet) otherwise. + pub fn determine_missing_cells_partial( + &self, + block_root: &Hash256, + data_column: &VerifiablePartialDataColumn, + ) -> Option> { + let column_index = data_column.column.index; + + // Check DA checker cache first - if we have a full column cached, nothing is missing + if self + .availability_cache .peek_pending_components(block_root, |components| { - if let Some(components) = components { - if let Some(cached_column) = - components.get_cached_data_column(data_column.index()) - { - return do_compare(cached_column.as_ref(), data_column); - } - if let Some(cached_column) = - components.get_cached_partial_data_column(data_column.index()) - { - return do_compare(cached_column.as_ref(), data_column); - } - } - Some(data_column.cells_present().collect()) + components + .and_then(|c| c.get_cached_data_column(column_index)) + .is_some() }) + { + return Some(vec![]); + } + + // Check assembler for partial columns + if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { + // Compare: find which cells from incoming partial aren't in cached partial + return compare_partial_to_partial(data_column, cached_partial.as_ref()); + } + + // No cached data, return all present cells as "missing" (new data we want) + let incoming_cells: Vec = data_column + .column + .sidecar + .cells_present_bitmap + .iter() + .enumerate() + .filter_map(|(idx, present)| present.then_some(idx)) + .collect(); + Some(incoming_cells) } /// Get a blob from the availability cache. @@ -292,11 +335,8 @@ impl DataAvailabilityChecker { .map(KzgVerifiedCustodyDataColumn::from_asserted_custody) .collect::>(); - self.availability_cache.put_kzg_verified_data_columns( - block_root, - verified_custody_columns, - |_| (), - ) + self.availability_cache + .put_kzg_verified_data_columns(block_root, verified_custody_columns) } /// Check if we've cached other blobs for this block. If it completes a set and we also @@ -331,18 +371,17 @@ impl DataAvailabilityChecker { /// have a block cached, return the `Availability` variant triggering block import. /// Otherwise cache the data column sidecar. /// - /// This should only accept gossip verified data columns, so we should not have to worry about dupes. + /// This should only accept gossip verified full data columns (not partials). + /// Partials are assembled in PartialDataColumnAssembler. #[instrument(skip_all, level = "trace")] pub fn put_gossip_verified_data_columns< O: ObservationStrategy, - C: DasColumn, - I: IntoIterator>, + I: IntoIterator>, >( &self, block_root: Hash256, slot: Slot, data_columns: I, - data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = slot.epoch(T::EthSpec::slots_per_epoch()); let sampling_columns = self @@ -354,28 +393,55 @@ impl DataAvailabilityChecker { .map(|c| KzgVerifiedCustodyDataColumn::from_asserted_custody(c.into_inner())) .collect::>(); - self.availability_cache.put_kzg_verified_data_columns( - block_root, - custody_columns, - data_publish_fn, - ) + self.availability_cache + .put_kzg_verified_data_columns(block_root, custody_columns) } + /// Put KZG-verified full custody data columns. + /// Only accepts full columns. Partials are assembled in PartialDataColumnAssembler. #[instrument(skip_all, level = "trace")] pub fn put_kzg_verified_custody_data_columns< - I: IntoIterator>, - C: DasColumn, + I: IntoIterator>, >( &self, block_root: Hash256, custody_columns: I, - data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache.put_kzg_verified_data_columns( - block_root, - custody_columns, - data_publish_fn, - ) + self.availability_cache + .put_kzg_verified_data_columns(block_root, custody_columns) + } + + /// Put complete data columns directly into the DA checker (no assembly needed) + #[instrument(skip_all, level = "trace")] + pub fn put_full_data_columns( + &self, + block_root: Hash256, + data_columns: Vec>>, + ) -> Result, AvailabilityCheckError> { + // Convert to custody columns - filter by custody requirement + let Some(first_column) = data_columns.first() else { + // No columns to process - return missing components + return Ok(Availability::MissingComponents(block_root)); + }; + let epoch = first_column.slot().epoch(T::EthSpec::slots_per_epoch()); + + let sampling_columns = self + .custody_context + .sampling_columns_for_epoch(epoch, &self.spec); + + let custody_columns = data_columns + .into_iter() + .filter(|col| sampling_columns.contains(&col.index)) + .map(|c| { + // These are from EL, already KZG verified + KzgVerifiedCustodyDataColumn::from_asserted_custody( + KzgVerifiedDataColumn::from_execution_verified(c), + ) + }) + .collect::>(); + + self.availability_cache + .put_kzg_verified_data_columns(block_root, custody_columns) } /// Check if we have all the blobs for a block. Returns `Availability` which has information @@ -383,10 +449,8 @@ impl DataAvailabilityChecker { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, - data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache - .put_executed_block(executed_block, data_publish_fn) + self.availability_cache.put_executed_block(executed_block) } /// Inserts a pre-execution block into the cache. @@ -692,11 +756,7 @@ impl DataAvailabilityChecker { ); self.availability_cache - .put_kzg_verified_data_columns( - *block_root, - data_columns_to_import_and_publish.clone(), - |_| (), - ) + .put_kzg_verified_data_columns(*block_root, data_columns_to_import_and_publish.clone()) .map(|availability| { DataColumnReconstructionResult::Success(( availability, @@ -906,19 +966,67 @@ impl MaybeAvailableBlock { } } -#[must_use = "Publish the data within"] -pub struct MergedData { - pub completed_columns: Vec>>, - pub updated_partials: Vec>>, +/// Compare an incoming full column (all cells present, indices 0..cell_count) against a cached partial. +/// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. +fn compare_full_to_partial( + incoming_cell_count: usize, + cached_partial: &VerifiablePartialDataColumn, +) -> Option> { + let cached_bitmap = &cached_partial.column.sidecar.cells_present_bitmap; + + // Check that cell counts match + if cached_bitmap.len() != incoming_cell_count { + debug!( + cached_len = cached_bitmap.len(), + incoming_len = incoming_cell_count, + "Cell count mismatch in column comparison" + ); + return None; + } + + // Find cells that are in incoming (all of them for a full column) but not in cached + let missing_cells: Vec = (0..incoming_cell_count) + .filter(|&idx| { + !cached_bitmap + .get(idx) + .expect("idx within bounds due to length check above") + }) + .collect(); + + Some(missing_cells) } -impl MergedData { - pub fn empty() -> Self { - MergedData { - completed_columns: vec![], - updated_partials: vec![], - } +/// Compare an incoming partial column against a cached partial. +/// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. +fn compare_partial_to_partial( + incoming: &VerifiablePartialDataColumn, + cached_partial: &VerifiablePartialDataColumn, +) -> Option> { + let incoming_bitmap = &incoming.column.sidecar.cells_present_bitmap; + let cached_bitmap = &cached_partial.column.sidecar.cells_present_bitmap; + + // Check that cell counts match + if incoming_bitmap.len() != cached_bitmap.len() { + debug!( + incoming_len = incoming_bitmap.len(), + cached_len = cached_bitmap.len(), + "Cell count mismatch in column comparison" + ); + return None; } + + // Find cells that are in incoming but not in cached + let missing_cells: Vec = incoming_bitmap + .iter() + .zip(cached_bitmap.iter()) + .enumerate() + .filter_map(|(idx, (in_incoming, in_cached))| { + // Cell is "missing" (from cache) if it's in incoming but not in cached + (in_incoming && !in_cached).then_some(idx) + }) + .collect(); + + Some(missing_cells) } #[cfg(test)] @@ -1071,10 +1179,10 @@ mod test { let gossip_columns = data_columns .into_iter() .filter(|d| requested_columns.contains(&d.index)) - .map(GossipVerifiedDataColumn::>::__new_for_testing) + .map(GossipVerifiedDataColumn::::__new_for_testing) .collect::>(); da_checker - .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns, |_| ()) + .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns) .expect("should put gossip custody columns"); // THEN the sampling size for the end slot of the same epoch remains unchanged @@ -1200,7 +1308,7 @@ mod test { da_checker .availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns, |_| ()) + .put_kzg_verified_data_columns(block_root, custody_columns) .expect("should put custody columns"); // Try reconstrucing diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index c2e71fcd1bf..776fb50f619 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1,5 +1,5 @@ +use super::AvailableBlockData; use super::state_lru_cache::{DietAvailabilityPendingExecutedBlock, StateLRUCache}; -use super::{AvailableBlockData, MergedData}; use crate::CustodyContext; use crate::beacon_chain::BeaconStore; use crate::blob_verification::KzgVerifiedBlob; @@ -14,14 +14,11 @@ use lru::LruCache; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use ssz_types::{RuntimeFixedVector, RuntimeVariableList}; use std::cmp::Ordering; -use std::mem; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{Span, debug, debug_span}; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobIdentifier; -use types::das_column::DasColumn; -use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ BlobSidecar, BlockImportSource, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, @@ -76,10 +73,7 @@ impl CachedBlock { pub struct PendingComponents { pub block_root: Hash256, pub verified_blobs: RuntimeFixedVector>>, - // TODO(dknopik): four options: two fields, one field containing an enum, converting everything to partial, or refactor for cell level storage - pub verified_partial_columns: - Vec>>, - pub verified_data_columns: Vec>>, + pub verified_data_columns: Vec>, pub block: Option>, pub reconstruction_started: bool, span: Span, @@ -99,7 +93,7 @@ impl PendingComponents { }) } - /// Returns an immutable reference to the full cached data column. + /// Returns an immutable reference to the cached data column. pub fn get_cached_data_column( &self, data_column_index: u64, @@ -110,17 +104,6 @@ impl PendingComponents { .map(|d| d.clone_arc()) } - /// Returns an immutable reference to the partial cached data column. - pub fn get_cached_partial_data_column( - &self, - data_column_index: u64, - ) -> Option>> { - self.verified_partial_columns - .iter() - .find(|d| d.index() == data_column_index) - .map(|d| d.clone_arc()) - } - /// Returns a mutable reference to the fixed vector of cached blobs. pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector>> { &mut self.verified_blobs @@ -203,65 +186,26 @@ impl PendingComponents { } /// Merges a given set of data columns into the cache. - fn merge_data_columns(&mut self, kzg_verified_data_columns: I) -> MergedData - where - I: IntoIterator>, - C: DasColumn, - { - let mut merged_data = MergedData::empty(); - + fn merge_data_columns>>( + &mut self, + kzg_verified_data_columns: I, + ) -> Result<(), AvailabilityCheckError> { for data_column in kzg_verified_data_columns { - if self.get_cached_data_column(data_column.index()).is_some() { - // already have the full column - continue; - } - - if let Some(full) = data_column.try_as_full(self.block.as_ref().map(|b| b.as_block())) { - self.verified_partial_columns - .retain(|col| col.index() != full.index()); - self.verified_data_columns.push(full); - continue; - } - - let partial = data_column.into_partial(); - if let Some((idx, cached_partial)) = self - .verified_partial_columns - .iter_mut() - .enumerate() - .find(|d| d.1.index() == partial.index()) - { - let did_merge = cached_partial.merge(&partial); - if did_merge { - if let Some(block) = &self.block - && let Some(full) = cached_partial.try_as_full(Some(block.as_block())) - { - merged_data.completed_columns.push(full.clone_arc()); - self.verified_data_columns.push(full); - self.verified_partial_columns.remove(idx); - } else { - merged_data - .updated_partials - .push(cached_partial.clone_arc()); - } - } - } else { - merged_data.updated_partials.push(partial.clone_arc()); - self.verified_partial_columns.push(partial); + if self.get_cached_data_column(data_column.index()).is_none() { + self.verified_data_columns.push(data_column); } } - merged_data + Ok(()) } /// Inserts a new block and revalidates the existing blobs against it. /// /// Blobs that don't match the new block's commitments are evicted. - pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) -> MergedData { + pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) { self.insert_executed_block(block); let reinsert = self.get_cached_blobs_mut().take(); self.merge_blobs(reinsert); - let reinsert = mem::take(&mut self.verified_partial_columns); - self.merge_data_columns(reinsert) } /// Returns Some if the block has received all its required data for import. The return value @@ -303,12 +247,7 @@ impl PendingComponents { let data_columns = self .verified_data_columns .iter() - .filter_map(|d| { - d.clone() - .into_inner() - .as_full(Some(block.as_block())) - .map(|c| Arc::new(c.into_owned())) - }) + .map(|d| d.clone().into_inner()) .collect::>(); Some(AvailableBlockData::DataColumns(data_columns)) } @@ -401,7 +340,6 @@ impl PendingComponents { block_root, verified_blobs: RuntimeFixedVector::new(vec![None; max_len]), verified_data_columns: vec![], - verified_partial_columns: vec![], block: None, reconstruction_started: false, span, @@ -474,7 +412,7 @@ pub struct DataAvailabilityCheckerInner { // the current usage, as it's deconstructed immediately. #[allow(clippy::large_enum_variant)] pub(crate) enum ReconstructColumnsDecision { - Yes(Vec>>), + Yes(Vec>), No(&'static str), } @@ -576,9 +514,10 @@ impl DataAvailabilityCheckerInner { *blob_opt = Some(blob); } } - let (pending_components, ()) = + let pending_components = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_blobs(fixed_blobs); + Ok(()) })?; pending_components.span.in_scope(|| { @@ -594,21 +533,17 @@ impl DataAvailabilityCheckerInner { #[allow(clippy::type_complexity)] pub fn put_kzg_verified_data_columns< - I: IntoIterator>, - C: DasColumn, + I: IntoIterator>, >( &self, block_root: Hash256, kzg_verified_data_columns: I, - data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); - let Some(epoch) = kzg_verified_data_columns.peek().map(|verified_blob| { - verified_blob - .as_data_column() - .slot() - .epoch(T::EthSpec::slots_per_epoch()) - }) else { + let Some(epoch) = kzg_verified_data_columns + .peek() + .map(|verified_blob| verified_blob.as_data_column().epoch()) + else { // No columns are processed. This can occur if all received columns were filtered out // before this point, e.g. due to a CGC change that caused extra columns to be downloaded // // before the new CGC took effect. @@ -616,13 +551,11 @@ impl DataAvailabilityCheckerInner { return Ok(Availability::MissingComponents(block_root)); }; - let (pending_components, merged_data) = + let pending_components = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_data_columns(kzg_verified_data_columns) })?; - data_publish_fn(merged_data); - let num_expected_columns = self .custody_context .num_of_data_columns_to_sample(epoch, &self.spec); @@ -645,7 +578,7 @@ impl DataAvailabilityCheckerInner { fn check_availability_and_cache_components( &self, block_root: Hash256, - pending_components: ComponentsLock<'_, T>, + pending_components: MappedRwLockReadGuard<'_, PendingComponents>, num_expected_columns_opt: Option, ) -> Result, AvailabilityCheckError> { if let Some(available_block) = pending_components.make_available( @@ -677,23 +610,23 @@ impl DataAvailabilityCheckerInner { /// /// Once the update is complete, the write lock is downgraded and a read guard with a /// reference of the updated `PendingComponents` is returned. - fn update_or_insert_pending_components( + fn update_or_insert_pending_components( &self, block_root: Hash256, epoch: Epoch, update_fn: F, - ) -> Result<(ComponentsLock<'_, T>, R), AvailabilityCheckError> + ) -> Result>, AvailabilityCheckError> where - F: FnOnce(&mut PendingComponents) -> R, + F: FnOnce(&mut PendingComponents) -> Result<(), AvailabilityCheckError>, { let mut write_lock = self.critical.write(); - let ret = { + { let pending_components = write_lock.get_or_insert_mut(block_root, || { PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) }); - update_fn(pending_components) - }; + update_fn(pending_components)? + } RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { cache.peek(&block_root) @@ -701,7 +634,6 @@ impl DataAvailabilityCheckerInner { .map_err(|_| { AvailabilityCheckError::Unexpected("pending components should exist".to_string()) }) - .map(|guard| (guard, ret)) } /// Check whether data column reconstruction should be attempted. @@ -771,9 +703,10 @@ impl DataAvailabilityCheckerInner { source: BlockImportSource, ) -> Result<(), AvailabilityCheckError> { let epoch = block.epoch(); - let (pending_components, ()) = + let pending_components = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.insert_pre_execution_block(block, source); + Ok(()) })?; let num_expected_columns_opt = self.get_num_expected_columns(epoch); @@ -803,7 +736,6 @@ impl DataAvailabilityCheckerInner { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, - data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = executed_block.as_block().epoch(); let block_root = executed_block.import_data.block_root; @@ -813,13 +745,12 @@ impl DataAvailabilityCheckerInner { .state_cache .register_pending_executed_block(executed_block); - let (pending_components, merged_data) = + let pending_components = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { - pending_components.merge_block(diet_executed_block) + pending_components.merge_block(diet_executed_block); + Ok(()) })?; - data_publish_fn(merged_data); - let num_expected_columns_opt = self.get_num_expected_columns(epoch); pending_components.span.in_scope(|| { @@ -888,9 +819,6 @@ impl DataAvailabilityCheckerInner { } } -type ComponentsLock<'a, T> = - MappedRwLockReadGuard<'a, PendingComponents<::EthSpec>>; - #[cfg(test)] mod test { use super::*; @@ -1131,7 +1059,7 @@ mod test { ); assert!(cache.critical.read().is_empty(), "cache should be empty"); let availability = cache - .put_executed_block(pending_block, |_| ()) + .put_executed_block(pending_block) .expect("should put block"); if blobs_expected == 0 { assert!( @@ -1198,7 +1126,7 @@ mod test { ); } let availability = cache - .put_executed_block(pending_block, |_| ()) + .put_executed_block(pending_block) .expect("should put block"); assert!( matches!(availability, Availability::Available(_)), @@ -1260,7 +1188,7 @@ mod test { // put the block in the cache let availability = cache - .put_executed_block(pending_block, |_| ()) + .put_executed_block(pending_block) .expect("should put block"); // grab the diet block from the cache for later testing @@ -1454,7 +1382,7 @@ mod pending_components_tests { setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); @@ -1469,7 +1397,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); cache.merge_blobs(blobs); assert_cache_consistent(cache, max_len); @@ -1485,7 +1413,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); assert_empty_blob_cache(cache); } @@ -1498,7 +1426,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); @@ -1514,7 +1442,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); assert_cache_consistent(cache, max_len); @@ -1530,7 +1458,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); - let _ = cache.merge_block(block_commitments); + cache.merge_block(block_commitments); assert_cache_consistent(cache, max_len); } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 9b6b7f7f069..e9f033be6ad 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -1,7 +1,9 @@ use crate::block_verification::{ BlockSlashInfo, get_validator_pubkey_cache, process_block_slash_info, }; -use crate::kzg_utils::{reconstruct_data_columns, validate_data_columns}; +use crate::kzg_utils::{ + reconstruct_data_columns, validate_full_data_columns, validate_partial_data_columns, +}; use crate::observed_data_sidecars::{ObservationStrategy, Observe}; use crate::{BeaconChain, BeaconChainError, BeaconChainTypes, metrics}; use educe::Educe; @@ -15,7 +17,6 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; -use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; use types::{ @@ -188,94 +189,39 @@ impl From for GossipDataColumnError { /// A wrapper around a `DataColumnSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. -#[derive(Debug)] -pub struct GossipVerifiedDataColumn< - T: BeaconChainTypes, - C: DasColumn, - O: ObservationStrategy = Observe, -> { +#[derive(Debug, Clone)] +pub struct GossipVerifiedDataColumn { block_root: Hash256, - data_column: KzgVerifiedDataColumn, + data_column: KzgVerifiedDataColumn, _phantom: PhantomData, } -impl, O: ObservationStrategy> Clone - for GossipVerifiedDataColumn -{ - fn clone(&self) -> Self { - Self { - block_root: self.block_root, - data_column: self.data_column.clone(), - _phantom: PhantomData, - } - } -} - -impl, O: ObservationStrategy> - GossipVerifiedDataColumn -{ - /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc) -> Self { - Self { - block_root: column_sidecar.block_root(), - data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), - _phantom: Default::default(), - } - } - - pub fn as_data_column(&self) -> &C { - self.data_column.as_data_column() - } - - /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc { - self.data_column.clone_data_column() - } - - pub fn block_root(&self) -> Hash256 { - self.block_root - } - - pub fn slot(&self) -> Slot { - self.data_column.data.slot() - } - - pub fn index(&self) -> ColumnIndex { - self.data_column.index() - } - - pub fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { - self.data_column.data.signed_block_header() - } - - pub fn into_inner(self) -> KzgVerifiedDataColumn { - self.data_column - } -} - -impl - GossipVerifiedDataColumn, O> -{ +impl GossipVerifiedDataColumn { pub fn new( column_sidecar: Arc>, subnet_id: DataColumnSubnetId, chain: &BeaconChain, ) -> Result { - let header = column_sidecar.signed_block_header().cloned(); + let header = column_sidecar.signed_block_header.clone(); // We only process slashing info if the gossip verification failed // since we do not process the data column any further in that case. validate_data_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { - if let Some(header) = header { - process_block_slash_info::<_, GossipDataColumnError>( - chain, - BlockSlashInfo::from_early_error_data_column(header, e), - ) - } else { - e - } + process_block_slash_info::<_, GossipDataColumnError>( + chain, + BlockSlashInfo::from_early_error_data_column(header.clone(), e), + ) }) } + /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. + pub fn __new_for_testing(column_sidecar: Arc>) -> Self { + Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), + _phantom: Default::default(), + } + } + /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY. /// When publishing a block constructed locally, the EL will have already verified the cell proofs. /// When publishing a block constructed externally, there will be no columns here. @@ -294,7 +240,7 @@ impl if chain .data_availability_checker - .determine_missing_cells(&column_sidecar.block_root(), column_sidecar.as_ref()) + .determine_missing_cells_full(&column_sidecar.block_root(), column_sidecar.as_ref()) .map(|cells| cells.is_empty()) .unwrap_or(false) { @@ -311,74 +257,179 @@ impl _phantom: Default::default(), }) } + + pub fn as_data_column(&self) -> &DataColumnSidecar { + self.data_column.as_data_column() + } + + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc> { + self.data_column.clone_data_column() + } + + pub fn block_root(&self) -> Hash256 { + self.block_root + } + + pub fn slot(&self) -> Slot { + self.data_column.as_data_column().slot() + } + + pub fn index(&self) -> ColumnIndex { + self.data_column.index() + } + + pub fn signed_block_header(&self) -> &SignedBeaconBlockHeader { + &self.data_column.as_data_column().signed_block_header + } + + pub fn into_inner(self) -> KzgVerifiedDataColumn { + self.data_column + } } -impl - GossipVerifiedDataColumn, O> -{ - pub fn new_partial( +/// Alias for backwards compatibility +pub type GossipVerifiedFullDataColumn = GossipVerifiedDataColumn; + +/// A wrapper around a `VerifiablePartialDataColumn` that indicates it has been approved for +/// re-gossiping on the p2p network. +#[derive(Debug, Clone)] +pub struct GossipVerifiedPartialDataColumn { + block_root: Hash256, + data_column: KzgVerifiedPartialDataColumn, + _phantom: PhantomData, +} + +impl GossipVerifiedPartialDataColumn { + pub fn new( column_sidecar: Arc>, chain: &BeaconChain, ) -> Result { validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) } -} -pub type GossipVerifiedFullDataColumn = - GossipVerifiedDataColumn::EthSpec>>; + /// Create a `GossipVerifiedPartialDataColumn` for testing ONLY. + pub fn __new_for_testing(column_sidecar: Arc>) -> Self { + Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedPartialDataColumn::__new_for_testing(column_sidecar), + _phantom: Default::default(), + } + } + + pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { + self.data_column.as_data_column() + } + + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc> { + self.data_column.clone_data_column() + } + + pub fn block_root(&self) -> Hash256 { + self.block_root + } + + pub fn slot(&self) -> Slot { + self.data_column.as_data_column().slot() + } + + pub fn index(&self) -> ColumnIndex { + self.data_column.index() + } + + pub fn into_inner(self) -> KzgVerifiedPartialDataColumn { + self.data_column + } +} /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] -pub struct KzgVerifiedDataColumn> { - data: Arc, - _phantom: PhantomData, +pub struct KzgVerifiedDataColumn { + data: Arc>, } -impl> KzgVerifiedDataColumn { - pub fn new(data_column: Arc, kzg: &Kzg) -> Result, KzgError)> { +impl KzgVerifiedDataColumn { + pub fn new( + data_column: Arc>, + kzg: &Kzg, + ) -> Result, KzgError)> { verify_kzg_for_data_column(data_column, kzg) } /// Mark a data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc) -> Self { - Self { - data: data_column, - _phantom: PhantomData, - } + pub fn from_execution_verified(data_column: Arc>) -> Self { + Self { data: data_column } } /// Create a `KzgVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc) -> Self { - Self { - data: data_column, - _phantom: PhantomData, - } + pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { + Self { data: data_column } } pub fn from_batch_with_scoring( - data_columns: Vec>, + data_columns: Vec>>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { verify_kzg_for_data_column_list(data_columns.iter(), kzg)?; Ok(data_columns .into_iter() - .map(|column| Self { - data: column, - _phantom: PhantomData, - }) + .map(|column| Self { data: column }) .collect()) } - pub fn to_data_column(self) -> Arc { + pub fn to_data_column(self) -> Arc> { + self.data + } + pub fn as_data_column(&self) -> &DataColumnSidecar { + &self.data + } + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc> { + self.data.clone() + } + + pub fn index(&self) -> ColumnIndex { + self.data.index + } +} + +/// Wrapper over a `VerifiablePartialDataColumn` for which we have completed kzg verification. +#[derive(Debug, Educe, Clone)] +#[educe(PartialEq, Eq)] +pub struct KzgVerifiedPartialDataColumn { + data: Arc>, +} + +impl KzgVerifiedPartialDataColumn { + pub fn new( + data_column: Arc>, + kzg: &Kzg, + ) -> Result, KzgError)> { + verify_kzg_for_partial_data_column(data_column, kzg) + } + + /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. + pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { + Self { data: data_column } + } + + /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed + /// from EL blobs. + pub fn from_execution_verified(data_column: Arc>) -> Self { + Self { data: data_column } + } + + pub fn to_data_column(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &C { + pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { &self.data } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc { + pub fn clone_data_column(&self) -> Arc> { self.data.clone() } @@ -420,73 +471,56 @@ impl CustodyDataColumn { } } -/// Data column that we must custody and has completed kzg verification +/// Data column that we must custody and has completed kzg verification. +/// Wraps a full `DataColumnSidecar`. #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] -pub struct KzgVerifiedCustodyDataColumn> { - data: Arc, - _phantom: PhantomData, +pub struct KzgVerifiedCustodyDataColumn { + data: Arc>, } -impl> KzgVerifiedCustodyDataColumn { +impl KzgVerifiedCustodyDataColumn { + /// Verify a column already marked as custody column + pub fn new( + data_column: CustodyDataColumn, + kzg: &Kzg, + ) -> Result, KzgError)> { + verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; + Ok(Self { + data: data_column.into_inner(), + }) + } + /// Mark a column as custody column. Caller must ensure that our current custody requirements /// include this column - pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { + pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { Self { data: kzg_verified.to_data_column(), - _phantom: PhantomData, } } - pub fn into_inner(self) -> Arc { + pub fn into_inner(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &C { + pub fn as_data_column(&self) -> &DataColumnSidecar { &self.data } - pub fn clone_arc(&self) -> Arc { + + pub fn clone_arc(&self) -> Arc> { self.data.clone() } - pub fn index(&self) -> ColumnIndex { - self.data.index() - } - // TODO(dknopik): below two functions are baaaad. They clone in far too many cases - - pub fn try_as_full( - &self, - block: Option<&SignedBeaconBlock>, - ) -> Option>> { - self.data - .as_full(block) - .map(|full| KzgVerifiedCustodyDataColumn { - data: Arc::new(full.into_owned()), - _phantom: PhantomData, - }) + pub fn index(&self) -> ColumnIndex { + self.data.index } - pub fn into_partial(self) -> KzgVerifiedCustodyDataColumn> { + pub fn into_partial(self) -> KzgVerifiedCustodyPartialDataColumn { let column = Arc::try_unwrap(self.data).unwrap_or_else(|column| (*column).clone()); - KzgVerifiedCustodyDataColumn { + KzgVerifiedCustodyPartialDataColumn { data: Arc::new(column.into_partial()), - _phantom: PhantomData, } } -} - -impl KzgVerifiedCustodyDataColumn> { - /// Verify a column already marked as custody column - pub fn new( - data_column: CustodyDataColumn, - kzg: &Kzg, - ) -> Result, KzgError)> { - verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; - Ok(Self { - data: data_column.data, - _phantom: PhantomData, - }) - } pub fn reconstruct_columns( kzg: &Kzg, @@ -505,16 +539,58 @@ impl KzgVerifiedCustodyDataColumn> { Ok(all_data_columns .into_iter() .map(|data| { - KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { - data, - _phantom: PhantomData, - }) + KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { data }) }) .collect::>()) } } -impl KzgVerifiedCustodyDataColumn> { +/// Partial data column that we must custody and has completed kzg verification. +/// Wraps a `VerifiablePartialDataColumn`. +#[derive(Debug, Educe, Clone)] +#[educe(PartialEq, Eq)] +pub struct KzgVerifiedCustodyPartialDataColumn { + data: Arc>, +} + +impl KzgVerifiedCustodyPartialDataColumn { + /// Mark a partial column as custody column. Caller must ensure that our current custody requirements + /// include this column + pub fn from_asserted_custody(kzg_verified: KzgVerifiedPartialDataColumn) -> Self { + Self { + data: kzg_verified.to_data_column(), + } + } + + pub fn into_inner(self) -> Arc> { + self.data + } + + pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { + &self.data + } + + pub fn clone_arc(&self) -> Arc> { + self.data.clone() + } + + pub fn index(&self) -> ColumnIndex { + self.data.index() + } + + // TODO(dknopik): This clones in far too many cases + // https://github.com/sigp/lighthouse/issues/XXXX + pub fn try_as_full( + &self, + block: Option<&SignedBeaconBlock>, + ) -> Option> { + self.data + .as_full(block) + .map(|full| KzgVerifiedCustodyDataColumn { + data: Arc::new(full.into_owned()), + }) + } + pub fn merge(&mut self, other: &Self) -> bool { let to_be_added = other .data @@ -540,7 +616,6 @@ impl KzgVerifiedCustodyDataColumn> kzg_commitments: self.data.kzg_commitments.clone(), slot: self.data.slot, }), - _phantom: PhantomData, }; true @@ -551,16 +626,26 @@ impl KzgVerifiedCustodyDataColumn> /// /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] -pub fn verify_kzg_for_data_column>( - data_column: Arc, +pub fn verify_kzg_for_data_column( + data_column: Arc>, kzg: &Kzg, -) -> Result, (Option, KzgError)> { +) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); - validate_data_columns(kzg, iter::once(&data_column))?; - Ok(KzgVerifiedDataColumn { - data: data_column, - _phantom: PhantomData, - }) + validate_full_data_columns(kzg, iter::once(&data_column))?; + Ok(KzgVerifiedDataColumn { data: data_column }) +} + +/// Complete kzg verification for a `VerifiablePartialDataColumn`. +/// +/// Returns an error if the kzg verification check fails. +#[instrument(skip_all, level = "debug")] +pub fn verify_kzg_for_partial_data_column( + data_column: Arc>, + kzg: &Kzg, +) -> Result, (Option, KzgError)> { + let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); + validate_partial_data_columns(kzg, iter::once(&data_column))?; + Ok(KzgVerifiedPartialDataColumn { data: data_column }) } /// Complete kzg verification for a list of `DataColumnSidecar`s. @@ -568,17 +653,15 @@ pub fn verify_kzg_for_data_column>( /// /// Note: This function should be preferred over calling `verify_kzg_for_data_column` /// in a loop since this function kzg verifies a list of data columns more efficiently. -pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I, A, C>( - data_column_iter: I, - kzg: &'a Kzg, +pub fn verify_kzg_for_data_column_list<'a, E: EthSpec>( + data_column_iter: impl Iterator>>, + kzg: &Kzg, ) -> Result<(), (Option, KzgError)> where - I: Iterator + Clone, - A: AsRef + 'a, - C: DasColumn + 'a, + E: 'a, { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES); - validate_data_columns(kzg, data_column_iter)?; + validate_full_data_columns(kzg, data_column_iter)?; Ok(()) } @@ -587,7 +670,7 @@ pub fn validate_data_column_sidecar_for_gossip>, subnet: DataColumnSubnetId, chain: &BeaconChain, -) -> Result, O>, GossipDataColumnError> { +) -> Result, GossipDataColumnError> { let column_slot = data_column.slot(); verify_data_column_sidecar(&data_column, &chain.spec)?; verify_index_matches_subnet(&data_column, subnet, &chain.spec)?; @@ -603,7 +686,7 @@ pub fn validate_data_column_sidecar_for_gossip( data_column: Arc>, chain: &BeaconChain, -) -> Result< - GossipVerifiedDataColumn, O>, - GossipDataColumnError, -> { +) -> Result, GossipDataColumnError> { // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply + // https://github.com/sigp/lighthouse/issues/XXXX let column_slot = data_column.slot(); let block_root = data_column.block_root(); verify_sidecar_not_from_future_slot(chain, column_slot)?; @@ -661,7 +742,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip< let filtered_column = if let Some(missing_cells) = chain .data_availability_checker - .determine_missing_cells(&block_root, data_column.as_ref()) + .determine_missing_cells_partial(&block_root, data_column.as_ref()) { if missing_cells.is_empty() { return Err(GossipDataColumnError::PriorKnownUnpublished); @@ -680,10 +761,10 @@ pub fn validate_partial_data_column_sidecar_for_gossip< // gossip accepted block let kzg = &chain.kzg; - let kzg_verified_data_column = verify_kzg_for_data_column(filtered_column, kzg) + let kzg_verified_data_column = verify_kzg_for_partial_data_column(filtered_column, kzg) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; - Ok(GossipVerifiedDataColumn { + Ok(GossipVerifiedPartialDataColumn { block_root, data_column: kzg_verified_data_column, _phantom: PhantomData, @@ -1006,7 +1087,7 @@ mod test { harness.advance_slot(); let verify_fn = |column_sidecar: DataColumnSidecar| { - GossipVerifiedDataColumn::<_, _>::new_for_block_publishing( + GossipVerifiedDataColumn::<_>::new_for_block_publishing( column_sidecar.into(), &harness.chain, ) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index e95cc9dfe13..3357cfaa493 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -35,6 +35,12 @@ impl FetchBlobsBeaconAdapter { &self.chain.task_executor } + pub(crate) fn partial_assembler( + &self, + ) -> &Arc> { + self.chain.data_availability_checker.partial_assembler() + } + pub(crate) async fn get_blobs_v1( &self, versioned_hashes: Vec, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 66e82f548b7..8a78e3abd13 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -14,7 +14,9 @@ mod tests; use crate::blob_verification::{GossipBlobError, KzgVerifiedBlob}; use crate::block_verification_types::AsBlock; -use crate::data_column_verification::{KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn}; +use crate::data_column_verification::{ + KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, +}; #[cfg_attr(test, double)] use crate::fetch_blobs::fetch_blobs_beacon_adapter::FetchBlobsBeaconAdapter; use crate::kzg_utils::blobs_to_partial_data_columns; @@ -34,7 +36,6 @@ use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_h use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; use types::blob_sidecar::BlobSidecarError; -use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ @@ -48,10 +49,10 @@ use types::{ #[derive(Debug)] pub enum EngineGetBlobsOutput { Blobs(Vec>), - /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. - CustodyColumns( - Vec>>, - ), + /// Complete columns ready to be imported into the `DataAvailabilityChecker`. + CompleteColumns(Vec>>), + /// Incomplete partial columns that need assembly via gossip + PartialColumns(Vec>>), } #[derive(Debug)] @@ -317,20 +318,46 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); } - // Up until this point we have not observed the data columns in the gossip cache, which allows - // them to arrive independently while this function is running. In publish_fn we will observe - // them and then publish any columns that had not already been observed. - publish_fn(EngineGetBlobsOutput::CustodyColumns( - custody_columns_to_import.clone(), - )); + // Initialize the partial assembler with the columns from the engine + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(FetchEngineBlobError::BeaconStateError)? + .clone(); - let availability_processing_status = chain_adapter - .process_engine_blobs( - block.slot(), - block_root, - EngineGetBlobsOutput::CustodyColumns(custody_columns_to_import), - ) - .await?; + let init_result = chain_adapter.partial_assembler().init_with_engine_blobs( + block_root, + block.slot(), + kzg_commitments, + custody_columns_to_import, + ); + + // Publish complete columns and incomplete partials + if !init_result.complete_columns.is_empty() { + publish_fn(EngineGetBlobsOutput::CompleteColumns( + init_result.complete_columns.clone(), + )); + } + if !init_result.incomplete_partials.is_empty() { + publish_fn(EngineGetBlobsOutput::PartialColumns( + init_result.incomplete_partials.clone(), + )); + } + + // Process complete columns through DA checker + let availability_processing_status = if !init_result.complete_columns.is_empty() { + chain_adapter + .process_engine_blobs( + block.slot(), + block_root, + EngineGetBlobsOutput::CompleteColumns(init_result.complete_columns), + ) + .await? + } else { + // No complete columns yet, still missing components + AvailabilityProcessingStatus::MissingComponents(block.slot(), block_root) + }; Ok(Some(availability_processing_status)) } @@ -342,10 +369,7 @@ async fn compute_custody_columns_to_import( block: Arc>>, blobs_and_proofs: Vec>, custody_columns_indices: &[ColumnIndex], -) -> Result< - Vec>>, - FetchEngineBlobError, -> { +) -> Result>, FetchEngineBlobError> { let kzg = chain_adapter.kzg().clone(); let spec = chain_adapter.spec().clone(); let chain_adapter_cloned = chain_adapter.clone(); @@ -383,8 +407,8 @@ async fn compute_custody_columns_to_import( .into_iter() .filter(|col| custody_columns_indices.contains(&col.index())) .map(|col| { - KzgVerifiedCustodyDataColumn::from_asserted_custody( - KzgVerifiedDataColumn::from_execution_verified(col), + KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( + KzgVerifiedPartialDataColumn::from_execution_verified(col), ) }) .collect::>() diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 059f4e07138..490cd56b7f2 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -224,7 +224,7 @@ mod get_blobs_v2 { assert!( matches!( published_columns, - EngineGetBlobsOutput::CustodyColumns(columns) if columns.len() == custody_columns.len() + EngineGetBlobsOutput::CompleteColumns(columns) if columns.len() == custody_columns.len() ), "should publish custody columns" ); diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index e4ae778c6c8..84a47c8b14d 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,7 +7,6 @@ use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; use types::beacon_block_body::KzgCommitments; -use types::das_column::DasColumn; use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError}; use types::partial_data_column_sidecar::{ CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, @@ -49,34 +48,95 @@ pub fn validate_blob( kzg.verify_blob_kzg_proof(&kzg_blob, kzg_commitment, kzg_proof) } -/// Validate a batch of `DataColumnSidecar`. -pub fn validate_data_columns<'a, E: EthSpec, I, A, C>( +/// Validate a batch of full `DataColumnSidecar`s. +/// +/// Full columns have all cells present, so we iterate over all cells directly. +pub fn validate_full_data_columns<'a, E: EthSpec>( kzg: &Kzg, - data_column_iter: I, -) -> Result<(), (Option, KzgError)> -where - I: Iterator + Clone, - A: AsRef + 'a, - C: DasColumn + 'a, -{ + data_column_iter: impl Iterator>>, +) -> Result<(), (Option, KzgError)> { let mut cells = Vec::new(); let mut proofs = Vec::new(); let mut column_indices = Vec::new(); let mut commitments = Vec::new(); for data_column in data_column_iter { - let data_column = data_column.as_ref(); - let col_index = data_column.index(); + let col_index = data_column.index; - if data_column.column().is_empty() { + if data_column.column.is_empty() { return Err((Some(col_index), KzgError::KzgVerificationFailed)); } - for data in data_column.iter().flatten() { - cells.push(ssz_cell_to_crypto_cell::(data.cell).map_err(|e| (Some(col_index), e))?); + // Full columns have all cells present - iterate over column, commitments, and proofs + for ((cell, commitment), proof) in data_column + .column + .iter() + .zip(data_column.kzg_commitments.iter()) + .zip(data_column.kzg_proofs.iter()) + { + cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); column_indices.push(col_index); - proofs.push(Bytes48::from(*data.proof)); - commitments.push(Bytes48::from(*data.commitment)); + proofs.push(Bytes48::from(*proof)); + commitments.push(Bytes48::from(*commitment)); + } + + let expected_len = column_indices.len(); + + // We make this check at each iteration so that the error is attributable to a specific column + if cells.len() != expected_len + || proofs.len() != expected_len + || commitments.len() != expected_len + { + return Err(( + Some(col_index), + KzgError::InconsistentArrayLength("Invalid data column".to_string()), + )); + } + } + + kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments) +} + +/// Validate a batch of partial `VerifiablePartialDataColumn`s. +/// +/// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. +pub fn validate_partial_data_columns<'a, E: EthSpec>( + kzg: &Kzg, + data_column_iter: impl Iterator>>, +) -> Result<(), (Option, KzgError)> { + let mut cells = Vec::new(); + let mut proofs = Vec::new(); + let mut column_indices = Vec::new(); + let mut commitments = Vec::new(); + + for data_column in data_column_iter { + let col_index = data_column.column.index; + let sidecar = &data_column.column.sidecar; + + if sidecar.column.is_empty() { + return Err((Some(col_index), KzgError::KzgVerificationFailed)); + } + + // Partial columns have a bitmap indicating present cells + // We iterate over the bitmap and only process present cells + let mut present_iterator = sidecar.column.iter().zip(sidecar.kzg_proofs.iter()); + for (present, commitment) in sidecar + .cells_present_bitmap + .iter() + .zip(data_column.kzg_commitments.iter()) + { + if present { + let (cell, proof) = present_iterator.next().ok_or(( + Some(col_index), + KzgError::InconsistentArrayLength( + "Partial column has fewer cells than bitmap indicates".to_string(), + ), + ))?; + cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); + column_indices.push(col_index); + proofs.push(Bytes48::from(*proof)); + commitments.push(Bytes48::from(*commitment)); + } } let expected_len = column_indices.len(); @@ -576,7 +636,7 @@ pub fn reconstruct_data_columns( mod test { use crate::kzg_utils::{ blobs_to_data_column_sidecars, reconstruct_blobs, reconstruct_data_columns, - validate_data_columns, + validate_full_data_columns, }; use bls::Signature; use eth2::types::BlobsBundle; @@ -615,7 +675,7 @@ mod test { blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) .unwrap(); - let result = validate_data_columns(kzg, column_sidecars.iter()); + let result = validate_full_data_columns(kzg, column_sidecars.iter()); assert!(result.is_ok()); } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index f92030a6714..b08ea76ba0b 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -42,6 +42,7 @@ pub mod observed_block_producers; pub mod observed_data_sidecars; pub mod observed_operations; mod observed_slashable; +pub mod partial_data_column_assembler; pub mod persisted_beacon_chain; pub mod persisted_custody; mod persisted_fork_choice; diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs new file mode 100644 index 00000000000..1f7445a9184 --- /dev/null +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -0,0 +1,234 @@ +use crate::data_column_verification::KzgVerifiedCustodyPartialDataColumn; +use lru::LruCache; +use parking_lot::RwLock; +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::Arc; +use types::beacon_block_body::KzgCommitments; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; +use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256, Slot}; + +/// Assembles partial data columns into complete columns +pub struct PartialDataColumnAssembler { + /// Cache of assemblies keyed by block root + assemblies: RwLock>>, +} + +/// Tracks partial columns being assembled for a single block +struct PartialAssembly { + #[allow(dead_code)] + block_root: Hash256, + slot: Slot, + kzg_commitments: KzgCommitments, + /// Map of column_index -> partial column being assembled + columns: HashMap>>, +} + +/// Result of merging a partial column +pub enum PartialMergeResult { + /// Merge was successful but column is still incomplete + Incomplete { + updated_partial: Arc>, + }, + /// Merge completed the column + Completed { + full_column: Arc>, + /// The updated partial for publishing (same as full but as partial type) + updated_partial: Arc>, + }, +} + +impl PartialDataColumnAssembler { + pub fn new(capacity: NonZeroUsize) -> Self { + Self { + assemblies: RwLock::new(LruCache::new(capacity)), + } + } + + /// Initialize assembler with columns from engine getBlobsV3 response. + /// Returns immediately complete columns and partials that need gossip propagation. + pub fn init_with_engine_blobs( + &self, + block_root: Hash256, + slot: Slot, + kzg_commitments: KzgCommitments, + custody_columns: Vec>, + ) -> InitResult { + let mut result = InitResult { + complete_columns: vec![], + incomplete_partials: vec![], + }; + + let mut assembly = PartialAssembly { + block_root, + slot, + kzg_commitments: kzg_commitments.clone(), + columns: HashMap::new(), + }; + + for custody_col in custody_columns { + let partial = custody_col.into_inner(); + let column_index = partial.column.index; + + // Check if this partial is already complete + if partial.column.sidecar.is_complete() { + // Convert to full column + if let Some(full) = Self::partial_to_full(&partial, &kzg_commitments) { + result.complete_columns.push(full); + continue; + } + } + + // Store incomplete partial + result.incomplete_partials.push(partial.clone()); + assembly.columns.insert(column_index, partial); + } + + // Only cache if we have incomplete columns + if !assembly.columns.is_empty() { + self.assemblies.write().put(block_root, assembly); + } + + result + } + + /// Merge a received partial column into the assembly. + /// Returns the merge result indicating if column is now complete. + pub fn merge_partial( + &self, + block_root: Hash256, + partial: Arc>, + ) -> Option> { + let mut assemblies = self.assemblies.write(); + let assembly = assemblies.get_mut(&block_root)?; + + let column_index = partial.column.index; + + let merged = if let Some(existing) = assembly.columns.get(&column_index) { + // Merge with existing partial + let merged_sidecar = existing.column.sidecar.merge(&partial.column.sidecar)?; + Arc::new(VerifiablePartialDataColumn { + column: Arc::new( + types::partial_data_column_sidecar::DanglingPartialDataColumn { + block_root: existing.column.block_root, + index: existing.column.index, + sidecar: merged_sidecar, + }, + ), + kzg_commitments: existing.kzg_commitments.clone(), + slot: existing.slot, + }) + } else { + // First time seeing this column index for this block + partial.clone() + }; + + // Check if merged column is now complete + if merged.column.sidecar.is_complete() { + // Remove from assembly since it's complete + assembly.columns.remove(&column_index); + + // Convert to full column + if let Some(full_column) = Self::partial_to_full(&merged, &assembly.kzg_commitments) { + return Some(PartialMergeResult::Completed { + full_column, + updated_partial: merged, + }); + } + } + + // Update assembly with merged partial + assembly.columns.insert(column_index, merged.clone()); + + Some(PartialMergeResult::Incomplete { + updated_partial: merged, + }) + } + + /// Get the current partial for a specific column if it exists in assembly + pub fn get_partial( + &self, + block_root: &Hash256, + column_index: ColumnIndex, + ) -> Option>> { + self.assemblies + .read() + .peek(block_root)? + .columns + .get(&column_index) + .cloned() + } + + /// Get commitments for a block if we have an active assembly + pub fn get_commitments(&self, block_root: &Hash256) -> Option> { + self.assemblies + .read() + .peek(block_root) + .map(|assembly| assembly.kzg_commitments.clone()) + } + + /// Check if we have an assembly for this block + pub fn has_assembly(&self, block_root: &Hash256) -> bool { + self.assemblies.read().contains(block_root) + } + + /// Remove assembly for a block (called when block is finalized or evicted) + pub fn remove_assembly(&self, block_root: &Hash256) { + self.assemblies.write().pop(block_root); + } + + /// Maintenance: remove assemblies older than cutoff epoch + pub fn do_maintenance(&self, cutoff_epoch: types::Epoch) { + let mut assemblies = self.assemblies.write(); + let mut to_remove = vec![]; + + for (root, assembly) in assemblies.iter() { + if assembly.slot.epoch(E::slots_per_epoch()) < cutoff_epoch { + to_remove.push(*root); + } + } + + for root in to_remove { + assemblies.pop(&root); + } + } + + /// Convert a complete partial to a full DataColumnSidecar + fn partial_to_full( + partial: &VerifiablePartialDataColumn, + kzg_commitments: &KzgCommitments, + ) -> Option>> { + // Use the as_full method - requires a block for signed header + // For now we'll construct it manually since we have all the data + if !partial.column.sidecar.is_complete() { + return None; + } + + let expected = partial.column.sidecar.cells_present_bitmap.len(); + if partial.column.sidecar.column.len() != expected + || partial.column.sidecar.kzg_proofs.len() != expected + || kzg_commitments.len() != expected + { + return None; + } + + // We need the signed block header and inclusion proof which requires the full block + // For the assembler, we'll return None here and require callers to use the + // VerifiablePartialDataColumn::as_full method with the actual block + // This is handled in the merge flow where we have access to the block + None + } + + /// Get cache size for metrics + pub fn cache_size(&self) -> usize { + self.assemblies.read().len() + } +} + +/// Result of initializing with engine blobs +pub struct InitResult { + /// Columns that were already complete + pub complete_columns: Vec>>, + /// Columns that need further assembly via gossip + pub incomplete_partials: Vec>>, +} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4fd873877f0..6d17d6d85c5 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2390,7 +2390,6 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await? .try_into() @@ -2415,7 +2414,6 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await? .try_into() @@ -3223,7 +3221,7 @@ where if !verified_columns.is_empty() { self.chain - .process_gossip_data_columns(verified_columns, || Ok(()), |_| ()) + .process_gossip_data_columns(verified_columns, || Ok(())) .await .unwrap(); } diff --git a/beacon_node/beacon_chain/tests/blob_verification.rs b/beacon_node/beacon_chain/tests/blob_verification.rs index 57332f735be..d1a0d87adf1 100644 --- a/beacon_node/beacon_chain/tests/blob_verification.rs +++ b/beacon_node/beacon_chain/tests/blob_verification.rs @@ -87,7 +87,6 @@ async fn rpc_blobs_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index bb8ccf09a49..2644b74b28e 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -507,7 +507,6 @@ async fn assert_invalid_signature( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await; assert!( @@ -579,7 +578,6 @@ async fn invalid_signature_gossip_block() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await; assert!( @@ -1023,7 +1021,6 @@ async fn block_gossip_verification() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .expect("should import valid gossip verified block"); @@ -1333,7 +1330,7 @@ async fn verify_and_process_gossip_data_sidecars( harness .chain - .process_gossip_data_columns(gossip_verified, || Ok(()), |_| ()) + .process_gossip_data_columns(gossip_verified, || Ok(())) .await .expect("should import valid gossip verified columns"); } @@ -1383,7 +1380,6 @@ async fn verify_block_for_gossip_slashing_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); @@ -1420,7 +1416,6 @@ async fn verify_block_for_gossip_doppelganger_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); @@ -1585,7 +1580,6 @@ async fn add_base_block_to_altair_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .expect_err("should error when processing base block"), @@ -1723,7 +1717,6 @@ async fn add_altair_block_to_base_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .expect_err("should error when processing altair block"), diff --git a/beacon_node/beacon_chain/tests/column_verification.rs b/beacon_node/beacon_chain/tests/column_verification.rs index 415089d394c..be9b3b2fa12 100644 --- a/beacon_node/beacon_chain/tests/column_verification.rs +++ b/beacon_node/beacon_chain/tests/column_verification.rs @@ -91,7 +91,6 @@ async fn rpc_columns_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index 97a471d1e51..86bdb03dafd 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -89,7 +89,7 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let _ = harness .chain - .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(()), |_| ()) + .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(())) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 81da193574c..5bd43835e33 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -695,7 +695,6 @@ async fn invalidates_all_descendants() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap() @@ -798,7 +797,6 @@ async fn switches_heads() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap() @@ -1065,7 +1063,6 @@ async fn invalid_parent() { assert!(matches!( rig.harness.chain.process_block(rpc_block.block_root(), rpc_block, NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ).await, Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root }) if invalid_root == parent_root @@ -1396,7 +1393,6 @@ async fn recover_from_invalid_head_by_importing_blocks() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index a6d76a6569a..ba0621ae720 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -3109,7 +3109,6 @@ async fn weak_subjectivity_sync_test( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); @@ -3713,7 +3712,6 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap_err(); @@ -3729,7 +3727,6 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index 33aebcaaf1f..17d9c5f697f 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -713,7 +713,6 @@ async fn run_skip_slot_test(skip_slots: u64) { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await .unwrap(); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index ca838e2feef..0a0fd361be6 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,10 +3,7 @@ use std::future::Future; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; -use beacon_chain::data_availability_checker::MergedData; -use beacon_chain::data_column_verification::{ - GossipVerifiedDataColumn, GossipVerifiedFullDataColumn, -}; +use beacon_chain::data_column_verification::GossipVerifiedDataColumn; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, @@ -33,12 +30,10 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tracing::{Span, debug, debug_span, error, field, info, instrument, warn}; use tree_hash::TreeHash; -use types::das_column::DasColumn; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, - DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, - FullPayload, FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, - SignedBlindedBeaconBlock, + DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, + FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, SignedBlindedBeaconBlock, }; use warp::{Rejection, Reply, reply::Response}; @@ -204,43 +199,6 @@ pub async fn publish_block>( Ok(()) }; - let sender_clone = network_tx.clone(); - let spec = chain.spec.clone(); - let data_publish_fn = move |merged_data: MergedData| { - debug!( - partial = merged_data.updated_partials.len(), - full = merged_data.completed_columns.len(), - "Sending merged data after block publish (should not happen?)" - ); - let messages: Vec<_> = merged_data - .updated_partials - .into_iter() - .map(|partial| { - PubsubMessage::PartialDataColumnSidecar(Box::new(( - DataColumnSubnetId::from_column_index(partial.index(), &spec), - partial.column.clone(), - None, - ))) - }) - .chain(merged_data.completed_columns.into_iter().flat_map(|full| { - let subnet = DataColumnSubnetId::from_column_index(full.index, &spec); - [ - PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - (*full).clone().into_partial().column.clone(), - None, - ))), - PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), - ] - })) - .collect(); - if !messages.is_empty() - && let Err(err) = crate::utils::publish_pubsub_messages(&sender_clone, messages) - { - warn!(?err, "Publishing data after block publish") - } - }; - // Wait for blobs/columns to get gossip verified before proceeding further as we need them for import. let (gossip_verified_blobs, gossip_verified_columns) = build_sidecar_task_handle.await?; @@ -276,30 +234,34 @@ pub async fn publish_block>( tokio::time::sleep(delay).await; } } + // Publish columns to network first (eager push for builder/producer case) publish_column_sidecars(network_tx, &gossip_verified_columns, &chain).map_err(|_| { warp_utils::reject::custom_server_error("unable to publish data column sidecars".into()) })?; + + // Add full columns directly to DA checker (not via gossip verification) let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let sampling_columns_indices = chain.sampling_columns_for_epoch(epoch); - let sampling_columns = gossip_verified_columns + let sampling_columns: Vec<_> = gossip_verified_columns .into_iter() .filter(|data_column| sampling_columns_indices.contains(&data_column.index())) - .collect::>(); + .map(|gv| gv.into_inner().to_data_column()) + .collect(); if !sampling_columns.is_empty() { - // Importing the columns could trigger block import and network publication in the case - // where the block was already seen on gossip. - if let Err(e) = - Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn, |_| ())) - .await + // Add columns directly to DA checker (block producer case) + let block_root = block.tree_hash_root(); + if let Err(e) = chain + .data_availability_checker + .put_full_data_columns(block_root, sampling_columns) { - let msg = format!("Invalid data column: {e}"); + let msg = format!("Failed to add data columns to DA checker: {e:?}"); return if let BroadcastValidation::Gossip = validation_level { Err(warp_utils::reject::broadcast_without_import(msg)) } else { error!( reason = &msg, - "Invalid data column during block publication" + "Data column addition failed during block publication" ); Err(warp_utils::reject::custom_bad_request(msg)) }; @@ -315,7 +277,6 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, - data_publish_fn, )) .await; post_block_import_logging_and_response( @@ -365,7 +326,6 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, - data_publish_fn, )) .await; post_block_import_logging_and_response( @@ -392,7 +352,7 @@ pub async fn publish_block>( type BuildDataSidecarTaskResult = Result< ( Vec>>, - Vec::EthSpec>>>, + Vec>, ), Rejection, >; @@ -450,7 +410,7 @@ fn build_data_columns( block: &SignedBeaconBlock>, blobs: BlobsList, kzg_cell_proofs: KzgProofs, -) -> Result>, Rejection> { +) -> Result>, Rejection> { let slot = block.slot(); let data_column_sidecars = build_blob_data_column_sidecars(chain, block, blobs, kzg_cell_proofs).map_err(|e| { @@ -544,7 +504,7 @@ fn publish_blob_sidecars( fn publish_column_sidecars( sender_clone: &UnboundedSender>, - data_column_sidecars: &[GossipVerifiedDataColumn>], + data_column_sidecars: &[GossipVerifiedDataColumn], chain: &BeaconChain, ) -> Result<(), BlockError> { let malicious_withhold_count = chain.config.malicious_withhold_count; @@ -568,9 +528,17 @@ fn publish_column_sidecars( .into_iter() .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); - let column = (*data_col).clone().into_partial().column; + // For block producers, eagerly send all column data via partial messages + // Gossipsub will handle per-peer diffing + let partial = (*data_col).clone().into_partial(); + // Pass bitmap of cells to eagerly send - for block producer, send all cells + let eager_data = Some(partial.column.sidecar.cells_present_bitmap.clone()); [ - PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, column, None))), + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + partial.column, + eager_data, + ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] }) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 8ebe499b2f0..868500e7314 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -6,8 +6,10 @@ use crate::{ }; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::AsBlock; -use beacon_chain::data_availability_checker::MergedData; -use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; +use beacon_chain::data_column_verification::{ + GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumn, +}; +use beacon_chain::partial_data_column_assembler::PartialMergeResult; use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, @@ -55,7 +57,6 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use types::das_column::DasColumn; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus @@ -930,7 +931,7 @@ impl NetworkBeaconProcessor { ); } - self.process_gossip_verified_data_column( + self.process_gossip_verified_partial_data_column( peer_id, gossip_verified_data_column, seen_duration, @@ -1279,10 +1280,12 @@ impl NetworkBeaconProcessor { } } - async fn process_gossip_verified_data_column>( + /// Process a gossip-verified full data column (not partial). + /// Partials are handled by process_gossip_verified_partial_data_column. + async fn process_gossip_verified_data_column( self: &Arc, peer_id: PeerId, - verified_data_column: GossipVerifiedDataColumn, + verified_data_column: GossipVerifiedDataColumn, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { @@ -1291,47 +1294,9 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); - let cloned_self = self.clone(); - let data_publish_fn = move |merged_data: MergedData| { - debug!( - partial = merged_data.updated_partials.len(), - full = merged_data.completed_columns.len(), - "Sending merged data" - ); - let messages: Vec<_> = merged_data - .updated_partials - .into_iter() - .map(|partial| { - PubsubMessage::PartialDataColumnSidecar(Box::new(( - DataColumnSubnetId::from_column_index( - partial.index(), - &cloned_self.chain.spec, - ), - partial.column.clone(), - None, - ))) - }) - .chain(merged_data.completed_columns.into_iter().flat_map(|full| { - let subnet = - DataColumnSubnetId::from_column_index(full.index, &self.chain.spec); - [ - PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - (*full).clone().into_partial().column.clone(), - None, - ))), - PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), - ] - })) - .collect(); - if !messages.is_empty() { - cloned_self.send_network_message(NetworkMessage::Publish { messages }) - } - }; - let result = self .chain - .process_gossip_data_columns(vec![verified_data_column], || Ok(()), data_publish_fn) + .process_gossip_data_columns(vec![verified_data_column], || Ok(())) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); @@ -1430,6 +1395,104 @@ impl NetworkBeaconProcessor { } } + /// Process a gossip-verified partial data column by merging it in the assembler + async fn process_gossip_verified_partial_data_column( + self: &Arc, + _peer_id: PeerId, + verified_partial: GossipVerifiedPartialDataColumn, + _seen_duration: Duration, + ) { + let processing_start_time = Instant::now(); + let block_root = verified_partial.block_root(); + let _data_column_slot = verified_partial.slot(); + let data_column_index = verified_partial.index(); + + let result = self + .chain + .process_gossip_partial_data_column(verified_partial) + .await; + + match &result { + Ok(merge_result) => { + match merge_result { + PartialMergeResult::Completed { + full_column, + updated_partial, + } => { + debug!( + %block_root, + index = data_column_index, + "Partial data column completed to full column" + ); + + // Publish both the updated partial and the full column + let subnet = DataColumnSubnetId::from_column_index( + full_column.index, + &self.chain.spec, + ); + self.send_network_message(NetworkMessage::Publish { + messages: vec![ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + updated_partial.column.clone(), + None, + ))), + PubsubMessage::DataColumnSidecar(Box::new(( + subnet, + full_column.clone(), + ))), + ], + }); + + // Check if block is now fully available + // This is handled in process_gossip_partial_data_column + } + PartialMergeResult::Incomplete { updated_partial } => { + trace!( + %block_root, + index = data_column_index, + "Partial data column merged but column still incomplete" + ); + + // Publish the updated partial + let subnet = DataColumnSubnetId::from_column_index( + updated_partial.column.index, + &self.chain.spec, + ); + self.send_network_message(NetworkMessage::Publish { + messages: vec![PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + updated_partial.column.clone(), + None, + )))], + }); + } + } + + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, + processing_start_time.elapsed().as_millis() as i64, + ); + } + Err(e) => { + warn!( + error = ?e, + %block_root, + index = data_column_index, + "Error processing partial data column" + ); + } + } + + // Notify sync if block was imported + if result.is_ok() { + self.send_sync_message(SyncMessage::GossipBlockProcessResult { + block_root, + imported: false, // Will be true if completed in the chain method + }); + } + } + /// Process the beacon block received from the gossip network and: /// /// - If it passes gossip propagation criteria, tell the network thread to forward it. @@ -1810,7 +1873,6 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Gossip, || Ok(()), - |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "block"); diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 5ac99d3e612..a6fa7761f2c 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -790,7 +790,6 @@ impl NetworkBeaconProcessor { let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let custody_columns = self.chain.sampling_columns_for_epoch(epoch); let self_cloned = self.clone(); - let block_cloned = block.clone(); let publish_fn = move |blobs_or_data_column| { if publish_blobs { match blobs_or_data_column { @@ -800,30 +799,22 @@ impl NetworkBeaconProcessor { block_root, ); } - EngineGetBlobsOutput::CustodyColumns(columns) => { - // Gradually publish any full columns - self_cloned.publish_data_columns_gradually( - columns - .iter() - .flat_map(|c| { - c.clone_arc() - .as_full(Some(&block_cloned)) - .map(|block| Arc::new(block.into_owned())) - }) - .collect(), - block_root, - ); - // "Publish" all columns as partial without eager send + EngineGetBlobsOutput::CompleteColumns(columns) => { + // Gradually publish full columns + self_cloned.publish_data_columns_gradually(columns, block_root); + } + EngineGetBlobsOutput::PartialColumns(partials) => { + // Publish partial columns without eager send self_cloned.send_network_message(NetworkMessage::Publish { - messages: columns + messages: partials .into_iter() - .map(|c| { + .map(|partial| { PubsubMessage::PartialDataColumnSidecar(Box::new(( DataColumnSubnetId::from_column_index( - c.index(), + partial.column.index, &self_cloned.chain.spec, ), - c.into_partial().into_inner().column.clone(), + partial.column.clone(), None, ))) }) @@ -1092,7 +1083,6 @@ impl NetworkBeaconProcessor { } } -use types::das_column::DasColumn; #[cfg(test)] use { beacon_chain::builder::Witness, beacon_processor::BeaconProcessorChannels, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index fa11d4ea484..6581b46dd3c 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -178,7 +178,6 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Rpc, "block"); diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 1a3e62f74be..ed04fe7bb97 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -1128,12 +1128,7 @@ async fn accept_processed_gossip_data_columns_without_import() { let block_root = rig.next_block.canonical_root(); rig.chain .data_availability_checker - .put_gossip_verified_data_columns( - block_root, - rig.next_block.slot(), - verified_data_columns, - |_| (), - ) + .put_gossip_verified_data_columns(block_root, rig.next_block.slot(), verified_data_columns) .expect("should put data columns into availability cache"); // WHEN an already processed but unobserved data column is received via gossip diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 737574490de..ef52f896785 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1079,7 +1079,7 @@ impl TestRig { .harness .chain .data_availability_checker - .put_executed_block(executed_block, |_| ()) + .put_executed_block(executed_block) .unwrap() { Availability::Available(_) => panic!("block removed from da_checker, available"), diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index 8b9c876907c..cb728a90c1b 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -431,7 +431,6 @@ impl TestRig { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), - |_| (), ) .await .unwrap() diff --git a/consensus/types/src/data/das_column.rs b/consensus/types/src/data/das_column.rs deleted file mode 100644 index 9445ee62558..00000000000 --- a/consensus/types/src/data/das_column.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! A PeerDAS data column. May or may not contain all cells. This is not necessarily implementable -//! for a beacon spec container, more as more data might be required, such as the partial message -//! group id alongside a partial message. - -use crate::beacon_block_body::KzgCommitments; -use crate::data::partial_data_column_sidecar::VerifiablePartialDataColumn; -use crate::data_column_sidecar::{Cell, DataColumn}; -use crate::{ - ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, - Slot, -}; -use kzg::{KzgCommitment, KzgProof}; -use ssz_types::VariableList; -use std::borrow::Cow; - -// TODO(dknopik): Name good? -// TODO(dknopik): Maybe move to unified cell storage? -// TODO(dknopik): Move generic parameter to associated type? -pub trait DasColumn: Clone { - fn slot(&self) -> Slot; - fn index(&self) -> ColumnIndex; - fn cell_count_total(&self) -> usize; - fn cells_present(&self) -> impl Iterator; - fn column(&self) -> &DataColumn; - fn kzg_proofs(&self) -> &VariableList; - fn kzg_commitments(&self) -> &KzgCommitments; - fn block_root(&self) -> Hash256; - fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader>; - fn into_partial(self) -> VerifiablePartialDataColumn; - - /// Convert this column into a full data column (e.g. for gossip). Note that this is potentially - /// expensive. - fn as_full( - &self, - header: Option<&SignedBeaconBlock>, - ) -> Option>>; - - fn iter(&self) -> impl Iterator>>; - - fn compare>(&self, rhs: &C) -> ColumnComparison { - if self.slot() != rhs.slot() - || self.index() != rhs.index() - || self.block_root() != rhs.block_root() - { - return ColumnComparison::DifferentColumns; - } - - if self.cell_count_total() != rhs.cell_count_total() { - return ColumnComparison::DataConflict; - } - - let mut missing_in_rhs = vec![]; - let mut missing_in_lhs = vec![]; - for (index, (lhs, rhs)) in self.iter().zip(rhs.iter()).enumerate() { - match (lhs, rhs) { - (None, None) => {} - (Some(_), None) => missing_in_rhs.push(index), - (None, Some(_)) => missing_in_lhs.push(index), - (Some(lhs), Some(rhs)) => { - if lhs != rhs { - return ColumnComparison::DataConflict; - } - } - } - } - if missing_in_rhs.is_empty() && missing_in_lhs.is_empty() { - return ColumnComparison::Equal; - } - - ColumnComparison::MissingCells { - missing_in_lhs, - missing_in_rhs, - } - } -} - -#[derive(Debug)] -pub enum ColumnComparison { - DifferentColumns, - DataConflict, - MissingCells { - missing_in_lhs: Vec, - missing_in_rhs: Vec, - }, - Equal, -} - -impl PartialEq> for VerifiablePartialDataColumn { - fn eq(&self, other: &DataColumnSidecar) -> bool { - // Slight optimisation: Can only be the same if `self` is fully present - self.column.sidecar.is_complete() - && self.slot() == other.slot() - && self.index() == other.index() - && self.block_root() == other.block_root() - && self.kzg_commitments() == other.kzg_commitments() - && self.column() == other.column() - && self.kzg_proofs() == other.kzg_proofs() - } -} - -#[derive(Clone, Copy, PartialEq, Debug)] -pub struct CellWithMetadata<'a, E: EthSpec> { - pub cell: &'a Cell, - pub proof: &'a KzgProof, - pub commitment: &'a KzgCommitment, -} diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 2d194fb7e8c..c35e4b34953 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -1,6 +1,3 @@ -use crate::SignedBeaconBlock; -use crate::data::das_column::CellWithMetadata; -use crate::data::das_column::DasColumn; use crate::data::partial_data_column_sidecar::CellBitmap; use crate::data::partial_data_column_sidecar::DanglingPartialDataColumn; use crate::data::partial_data_column_sidecar::PartialDataColumnSidecar; @@ -18,7 +15,6 @@ use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; -use std::borrow::Cow; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -138,6 +134,32 @@ impl DataColumnSidecar { .as_ssz_bytes() .len() } + + /// Convert this full data column into a verifiable partial data column. + pub fn into_partial(self) -> VerifiablePartialDataColumn { + let cell_count = self.column.len(); + let mut bitmap = + CellBitmap::::with_capacity(cell_count).expect("our column has the same bound"); + for idx in 0..cell_count { + bitmap + .set(idx, true) + .expect("The correct size is initialized right above"); + } + + VerifiablePartialDataColumn { + slot: self.slot(), + column: Arc::new(DanglingPartialDataColumn { + block_root: self.block_root(), + index: self.index, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: self.column, + kzg_proofs: self.kzg_proofs, + }, + }), + kzg_commitments: self.kzg_commitments, + } + } } #[derive(Debug)] @@ -178,86 +200,3 @@ impl From for DataColumnSidecarError { Self::SszError(e) } } - -impl DasColumn for DataColumnSidecar { - fn slot(&self) -> Slot { - self.slot() - } - - fn index(&self) -> ColumnIndex { - self.index - } - - fn cell_count_total(&self) -> usize { - self.column.len() - } - - fn cells_present(&self) -> impl Iterator { - 0..self.cell_count_total() - } - - fn column(&self) -> &DataColumn { - &self.column - } - - fn kzg_proofs(&self) -> &VariableList { - &self.kzg_proofs - } - - fn kzg_commitments(&self) -> &KzgCommitments { - &self.kzg_commitments - } - - fn block_root(&self) -> Hash256 { - self.block_root() - } - - fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { - Some(&self.signed_block_header) - } - - fn into_partial(self) -> VerifiablePartialDataColumn { - let mut bitmap = CellBitmap::::with_capacity(self.cell_count_total()) - .expect("our column has the same bound"); - for idx in 0..self.cell_count_total() { - bitmap - .set(idx, true) - .expect("The correct size is initialized right above"); - } - - VerifiablePartialDataColumn { - slot: self.slot(), - column: Arc::new(DanglingPartialDataColumn { - block_root: self.block_root(), - index: self.index(), - sidecar: PartialDataColumnSidecar { - cells_present_bitmap: bitmap, - column: self.column, - kzg_proofs: self.kzg_proofs, - }, - }), - kzg_commitments: self.kzg_commitments, - } - } - - fn as_full( - &self, - _block: Option<&SignedBeaconBlock>, - ) -> Option>> { - Some(Cow::Borrowed(self)) - } - - fn iter(&self) -> impl Iterator>> { - self.column - .iter() - .zip(self.kzg_commitments.iter()) - .zip(self.kzg_proofs.iter()) - .map(|((cell, commitment), proof)| { - Some(CellWithMetadata { - cell, - commitment, - proof, - }) - }) - } -} diff --git a/consensus/types/src/data/mod.rs b/consensus/types/src/data/mod.rs index f3e20b2cc0d..63f8cb8e269 100644 --- a/consensus/types/src/data/mod.rs +++ b/consensus/types/src/data/mod.rs @@ -1,5 +1,4 @@ mod blob_sidecar; -pub mod das_column; mod data_column_custody_group; mod data_column_sidecar; mod data_column_subnet_id; diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index c6b08fac8cf..57e2ca30e58 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,9 +1,8 @@ use crate::beacon_block_body::KzgCommitments; -use crate::data::das_column::{CellWithMetadata, DasColumn}; use crate::data_column_sidecar::{Cell, DataColumn}; use crate::test_utils::TestRandom; use crate::{AbstractExecPayload, ColumnIndex, DataColumnSidecar}; -use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot}; +use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, Slot}; use context_deserialize::context_deserialize; use educe::Educe; use kzg::KzgProof; @@ -14,7 +13,6 @@ use ssz_types::VariableList; use std::borrow::Cow; use std::sync::Arc; use test_random_derive::TestRandom; -use tree_hash::TreeHash; use tree_hash_derive::TreeHash; pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; @@ -259,55 +257,22 @@ impl VerifiablePartialDataColumn { slot: self.slot, }) } -} -impl DasColumn for VerifiablePartialDataColumn { - fn slot(&self) -> Slot { + pub fn slot(&self) -> Slot { self.slot } - fn index(&self) -> ColumnIndex { + pub fn index(&self) -> ColumnIndex { self.column.index } - fn cell_count_total(&self) -> usize { - self.column.sidecar.cells_present_bitmap.len() - } - - fn cells_present(&self) -> impl Iterator { - self.column - .sidecar - .cells_present_bitmap - .iter() - .enumerate() - .filter_map(|(idx, bit)| bit.then_some(idx)) - } - - fn column(&self) -> &DataColumn { - &self.column.sidecar.column - } - - fn kzg_proofs(&self) -> &VariableList { - &self.column.sidecar.kzg_proofs + pub fn block_root(&self) -> Hash256 { + self.column.block_root } - fn kzg_commitments(&self) -> &KzgCommitments { - &self.kzg_commitments - } - - fn block_root(&self) -> Hash256 { - self.column.block_root.tree_hash_root() - } - - fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { - None - } - - fn into_partial(self) -> VerifiablePartialDataColumn { - self - } - - fn as_full( + /// Convert this partial column into a full `DataColumnSidecar` if all cells are present. + /// Requires the signed beacon block to populate the header and inclusion proof. + pub fn as_full( &self, block: Option<&SignedBeaconBlock>, ) -> Option>> { @@ -339,25 +304,4 @@ impl DasColumn for VerifiablePartialDataColumn { signed_block_header, })) } - - fn iter(&self) -> impl Iterator>> { - let sidecar = &self.column.sidecar; - let mut present_iterator = sidecar.column.iter().zip(sidecar.kzg_proofs.iter()); - sidecar - .cells_present_bitmap - .iter() - .zip(self.kzg_commitments.iter()) - .map(move |(present, commitment)| { - if present { - let (cell, proof) = present_iterator.next()?; - Some(CellWithMetadata { - cell, - proof, - commitment, - }) - } else { - None - } - }) - } } diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 8006f0215e3..8e9d438a243 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -530,12 +530,11 @@ impl Tester { }) .collect(); - let result = - self.block_on_dangerous(self.harness.chain.process_gossip_data_columns( - gossip_verified_data_columns, - || Ok(()), - |_| (), - ))?; + let result = self.block_on_dangerous( + self.harness + .chain + .process_gossip_data_columns(gossip_verified_data_columns, || Ok(())), + )?; if valid { assert!(result.is_ok()); } @@ -549,7 +548,6 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = data_column_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); @@ -640,7 +638,6 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), - |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = blob_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); From b68a9670cbfe838403699dfeeeebacb5772980ca Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 14 Jan 2026 12:31:46 +0100 Subject: [PATCH 046/146] add getBlobsV3 metrics --- .../beacon_chain/src/fetch_blobs/mod.rs | 20 +++++++++-- beacon_node/beacon_chain/src/metrics.rs | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 33f095802cc..3c2fcf0be64 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -249,12 +249,28 @@ async fn fetch_and_process_blobs_v2_or_v3( let get_blobs_v3 = chain_adapter.supports_get_blobs_v3().await?; let response = if get_blobs_v3 { debug!(num_expected_blobs, "Fetching available blobs from the EL"); - chain_adapter + // Track request count and duration for standardized metrics + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_REQUESTS_TOTAL); + let _timer = + metrics::start_timer(&metrics::BEACON_ENGINE_GET_BLOBS_V3_REQUEST_DURATION_SECONDS); + + let response = chain_adapter .get_blobs_v3(versioned_hashes) .await .inspect_err(|_| { inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); - })? + })?; + + if response + .as_ref() + .is_some_and(|vec| vec.iter().all(|blob| blob.is_some())) + { + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_COMPLETE_RESPONSES_TOTAL); + } else { + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_PARTIAL_RESPONSES_TOTAL); + } + + response } else { debug!(num_expected_blobs, "Fetching all blobs from the EL"); diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 55f6ae04e08..39f364bcb67 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1737,6 +1737,40 @@ pub static BEACON_ENGINE_GET_BLOBS_V2_REQUEST_DURATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_engine_getBlobsV3_requests_total", + "Total number of engine_getBlobsV3 requests made to the execution layer", + ) + }); + +pub static BEACON_ENGINE_GET_BLOBS_V3_COMPLETE_RESPONSES_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_engine_getBlobsV3_complete_responses_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with all blobs", + ) + }); + +pub static BEACON_ENGINE_GET_BLOBS_V3_PARTIAL_RESPONSES_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_engine_getBlobsV3_partial_responses_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with at least one blob missing", + ) + }); + +pub static BEACON_ENGINE_GET_BLOBS_V3_REQUEST_DURATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_histogram( + "beacon_engine_getBlobsV3_request_duration_seconds", + "Duration of engine_getBlobsV3 requests to the execution layer in seconds", + ) + }); + /* * Light server message verification */ From 42ed853ee6482ce23d3a6fe632726f0650062618 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 15 Jan 2026 11:59:31 +0100 Subject: [PATCH 047/146] update libp2p --- Cargo.lock | 62 +++++++++---------- .../lighthouse_network/src/service/mod.rs | 42 ++++++------- .../lighthouse_network/src/types/partial.rs | 23 +++---- 3 files changed, 64 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffea856f4a2..ba1734ca4a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,7 +685,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -696,7 +696,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3056,7 +3056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4676,7 +4676,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4943,7 +4943,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "bytes", "either", @@ -4975,7 +4975,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4985,7 +4985,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4995,7 +4995,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "either", "fnv", @@ -5019,7 +5019,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "async-trait", "futures", @@ -5034,7 +5034,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5064,7 +5064,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "asynchronous-codec", "either", @@ -5104,7 +5104,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "hickory-proto", @@ -5122,7 +5122,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "libp2p-core", @@ -5137,7 +5137,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "asynchronous-codec", "bytes", @@ -5155,7 +5155,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "asynchronous-codec", "bytes", @@ -5177,7 +5177,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "asynchronous-codec", "bytes", @@ -5192,7 +5192,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "futures-timer", @@ -5213,7 +5213,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "either", "fnv", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "heck", "quote", @@ -5244,7 +5244,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "futures-timer", @@ -5259,7 +5259,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "futures-rustls", @@ -5277,7 +5277,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "futures-timer", @@ -5291,7 +5291,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "either", "futures", @@ -5987,7 +5987,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "bytes", "futures", @@ -6254,7 +6254,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7100,7 +7100,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "asynchronous-codec", "bytes", @@ -7657,7 +7657,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7670,7 +7670,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7776,7 +7776,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5673256a216017679fb5bd6505cea079575527ad" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" dependencies = [ "futures", "pin-project", @@ -8735,7 +8735,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10072,7 +10072,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 682231a2b78..67e12e3a052 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -809,18 +809,18 @@ impl Network { .insert(topic.clone()); let config = &self.network_globals.config; - let disable_request = - config.disable_partial_messages_request || config.disable_partial_messages_support; - let disable_support = config.disable_partial_messages_support; - let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe( - &topic, - partial && !disable_request, - partial && !disable_support, - ) { + let subscribe_result = if partial && !config.disable_partial_messages_support { + let request_partials = !config.disable_partial_messages_request; + self.gossipsub_mut() + .subscribe_partial(&topic, request_partials) + } else { + self.gossipsub_mut().subscribe(&topic) + }; + + match subscribe_result { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false @@ -1315,16 +1315,16 @@ impl Network { } } Event::Partial { - topic_id, - propagation_source, + topic_hash, + peer_id, group_id, message, metadata, } => { - let topic = GossipTopic::decode(topic_id.as_str()) + let topic = GossipTopic::decode(topic_hash.as_str()) .inspect_err(|error| { debug!( - topic = ?topic_id, + topic = ?topic_hash, error, "Could not decode gossipsub partial message topic" ); @@ -1342,19 +1342,19 @@ impl Network { metadata_string.push('0'); } } - debug!(metadata = metadata_string, %topic, %propagation_source, "Got metadata") + debug!(metadata = metadata_string, %topic, %peer_id, "Got metadata") } else { - warn!(?metadata, %topic, %propagation_source, "Got weird metadata"); + warn!(?metadata, %topic, %peer_id, "Got weird metadata"); } } else { - debug!(%propagation_source, "Got no metadata") + debug!(%peer_id, "Got no metadata") } if let Some(message) = message { - match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { + match PubsubMessage::decode_partial(&topic_hash, &group_id, &message) { Err(error) => { debug!( - topic = ?topic_id, + topic = ?topic_hash, error, "Could not decode gossipsub partial message" ); @@ -1369,14 +1369,14 @@ impl Network { Ok(message) => { debug!( %message, - %propagation_source, + %peer_id, "Decoded partial message" ); // Notify the network return Some(NetworkEvent::PubsubMessage { id: MessageId::new(&[]), // TODO(dknopik): waht to send - source: propagation_source, - topic: topic_id, + source: peer_id, + topic: topic_hash, message, }); } diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index cf5e79a3461..4ab0adb6306 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,5 +1,5 @@ -use gossipsub::partial::{Metadata, PublishAction}; -use gossipsub::{Partial, PartialMessageError}; +use crate::PeerId; +use gossipsub::{Metadata, Partial, PartialAction, PartialError}; use ssz::{Decode, Encode}; use std::fmt::Debug; use std::sync::Arc; @@ -55,11 +55,11 @@ impl Metadata for CellBitmapMetadata { &self.encoded } - fn update(&mut self, data: &[u8]) -> Result { - let data = CellBitmap::::from_ssz_bytes(data) - .map_err(|_| PartialMessageError::InvalidFormat)?; + fn update(&mut self, data: &[u8]) -> Result { + let data = + CellBitmap::::from_ssz_bytes(data).map_err(|_| PartialError::InvalidFormat)?; if data.len() != self.bitmap.len() { - return Err(PartialMessageError::OutOfRange); + return Err(PartialError::OutOfRange); } let new_bitmap = self.bitmap.union(&data); if self.bitmap == new_bitmap { @@ -92,12 +92,13 @@ impl Partial for PartialDataColumnSidecarMessage { .as_ssz_bytes() } - fn partial_message_bytes_from_metadata( + fn partial_action_from_metadata( &self, + _peer_id: PeerId, metadata: Option<&[u8]>, - ) -> Result { + ) -> Result { match metadata { - None => Ok(PublishAction { + None => Ok(PartialAction { need: false, send: self.send_eager.clone().map(|eager| { ( @@ -108,7 +109,7 @@ impl Partial for PartialDataColumnSidecarMessage { }), Some(metadata) => { let peer_has = CellBitmap::::from_ssz_bytes(metadata) - .map_err(|_| PartialMessageError::InvalidFormat)?; + .map_err(|_| PartialError::InvalidFormat)?; let need = !peer_has.is_subset(&self.partial_column.sidecar.cells_present_bitmap); let send = self @@ -124,7 +125,7 @@ impl Partial for PartialDataColumnSidecarMessage { ) }); - Ok(PublishAction { need, send }) + Ok(PartialAction { need, send }) } } } From 1f79e90b725712dcf44d9faf263786204a6c199c Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 14 Jan 2026 14:30:58 +0100 Subject: [PATCH 048/146] add cell usefulness metric definitions --- beacon_node/beacon_chain/src/metrics.rs | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 39f364bcb67..058f8b7b827 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1771,6 +1771,49 @@ pub static BEACON_ENGINE_GET_BLOBS_V3_REQUEST_DURATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_partial_message_useful_cells_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with at least one blob missing", + &["index"], + ) + }); + +pub static BEACON_PARTIAL_MESSAGE_CELLS_RECEIVED_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_partial_message_cells_received_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with at least one blob missing", + &["index"], + ) + }); + +pub static BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_useful_full_columns_received_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with at least one blob missing", + &["index"], + ) + }); + +pub static BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_partial_message_column_completions_total", + "Total number of successful engine_getBlobsV3 responses from the execution layer \ + with at least one blob missing", + &["index"], + ) + }); + /* * Light server message verification */ From 24faa31cff73b672429fb5b7ad6e42654d895f7f Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 15:34:42 +0100 Subject: [PATCH 049/146] update libp2p --- Cargo.lock | 56 +++++++++---------- .../lighthouse_network/src/types/partial.rs | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 141fe50ebba..c202fba9504 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3056,7 +3056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4676,7 +4676,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4943,7 +4943,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "bytes", "either", @@ -4974,7 +4974,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4984,7 +4984,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4994,7 +4994,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "either", "fnv", @@ -5018,7 +5018,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "async-trait", "futures", @@ -5033,7 +5033,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5063,7 +5063,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "asynchronous-codec", "either", @@ -5103,7 +5103,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "hickory-proto", @@ -5121,7 +5121,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "libp2p-core", @@ -5137,7 +5137,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "asynchronous-codec", "bytes", @@ -5155,7 +5155,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "asynchronous-codec", "bytes", @@ -5177,7 +5177,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "futures-timer", @@ -5198,7 +5198,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "heck", "quote", @@ -5229,7 +5229,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "futures-timer", @@ -5244,7 +5244,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "futures-rustls", @@ -5262,7 +5262,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "futures-timer", @@ -5276,7 +5276,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "either", "futures", @@ -5972,7 +5972,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "bytes", "futures", @@ -6239,7 +6239,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7085,7 +7085,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "asynchronous-codec", "bytes", @@ -7147,7 +7147,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7655,7 +7655,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7761,7 +7761,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#922e5c24ecd029254476f1eb7a740c8705cd49aa" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" dependencies = [ "futures", "pin-project", @@ -8720,7 +8720,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10057,7 +10057,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 4be466bbbdf..03659d64aff 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,5 +1,5 @@ use crate::PeerId; -use libp2p::gossipsub::{Metadata, Partial, PartialAction, PartialError}; +use libp2p::gossipsub::partial_messages::{Metadata, Partial, PartialAction, PartialError}; use ssz::{Decode, Encode}; use std::fmt::Debug; use std::sync::Arc; From 3f55ade440a8430f4e5f886eaf3d976c2519bac3 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 15 Jan 2026 13:13:54 +0100 Subject: [PATCH 050/146] add PartialDataColumnHeader --- beacon_node/beacon_chain/src/kzg_utils.rs | 1 + .../types/src/data/data_column_sidecar.rs | 6 +++ .../src/data/partial_data_column_sidecar.rs | 42 +++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index a809a7b8b3e..a2d533ce096 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -485,6 +485,7 @@ pub(crate) fn build_partial_data_columns( .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, + header: VariableList::empty(), }, }), kzg_commitments: kzg_commitments.clone(), diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index c35e4b34953..0f5fa4f00ec 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -19,6 +19,7 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use crate::partial_data_column_sidecar::PartialDataColumnHeader; use crate::{ block::{BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlockHeader}, core::{Epoch, EthSpec, Hash256, Slot}, @@ -155,6 +156,11 @@ impl DataColumnSidecar { cells_present_bitmap: bitmap, column: self.column, kzg_proofs: self.kzg_proofs, + header: VariableList::repeat_full(PartialDataColumnHeader { + kzg_commitments: self.kzg_commitments.clone(), + signed_block_header: self.signed_block_header, + kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof, + }), }, }), kzg_commitments: self.kzg_commitments, diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index baac256206c..dedf69812ea 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,19 +1,18 @@ use crate::test_utils::TestRandom; use crate::{ - AbstractExecPayload, Cell, ColumnIndex, DataColumn, DataColumnSidecar, KzgCommitments, + AbstractExecPayload, Cell, ColumnIndex, DataColumn, DataColumnSidecar, EthSpec, Hash256, + KzgCommitments, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; -use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, Slot}; -use context_deserialize::context_deserialize; use educe::Educe; use kzg::KzgProof; -use serde::{Deserialize, Serialize}; use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; +use ssz_types::{FixedVector, VariableList}; use std::borrow::Cow; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +use typenum::U1; pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; @@ -22,15 +21,13 @@ pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Educe)] -#[serde(bound = "E: EthSpec")] +#[derive(Debug, Clone, Encode, Decode, TreeHash, TestRandom, Educe)] #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] -#[context_deserialize(ForkName)] pub struct PartialDataColumnSidecar { pub cells_present_bitmap: CellBitmap, - #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] pub column: DataColumn, pub kzg_proofs: VariableList, + pub header: VariableList, U1>, } impl PartialDataColumnSidecar { @@ -40,6 +37,7 @@ impl PartialDataColumnSidecar { cells_present_bitmap: BitList::with_capacity(1).unwrap(), column: VariableList::new(vec![Cell::::default()]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), + header: VariableList::new(vec![]).unwrap(), } .as_ssz_bytes() .len() @@ -51,6 +49,7 @@ impl PartialDataColumnSidecar { cells_present_bitmap: BitList::with_capacity(block_blobs).unwrap(), column: VariableList::new(vec![Cell::::default(); present_blobs]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty(); present_blobs]).unwrap(), + header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push } .as_ssz_bytes() .len() @@ -61,6 +60,7 @@ impl PartialDataColumnSidecar { cells_present_bitmap: BitList::with_capacity(max_blobs_per_block).unwrap(), column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), + header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push } .as_ssz_bytes() .len() @@ -121,6 +121,7 @@ impl PartialDataColumnSidecar { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, + header: self.header.clone(), }) } @@ -172,6 +173,11 @@ impl PartialDataColumnSidecar { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, + header: if !self.header.is_empty() { + self.header.clone() + } else { + other.header.clone() + }, }) } } @@ -194,10 +200,28 @@ impl From> for PartialDataColumnSidecar { cells_present_bitmap, column: value.column, kzg_proofs: value.kzg_proofs, + header: VariableList::repeat_full(PartialDataColumnHeader { + kzg_commitments: value.kzg_commitments, + signed_block_header: value.signed_block_header, + kzg_commitments_inclusion_proof: value.kzg_commitments_inclusion_proof, + }), } } } +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive(Debug, Clone, Encode, Decode, TreeHash, TestRandom, Educe)] +#[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +pub struct PartialDataColumnHeader { + pub kzg_commitments: KzgCommitments, + pub signed_block_header: SignedBeaconBlockHeader, + pub kzg_commitments_inclusion_proof: FixedVector, +} + // TODO(dknopik): Name? #[derive(Debug, Clone, PartialEq)] pub struct DanglingPartialDataColumn { From 1885e2509bac054114855e780786e0c861c6d8d7 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 15 Jan 2026 21:49:45 +0100 Subject: [PATCH 051/146] remove eager sending of cells --- beacon_node/http_api/src/publish_blocks.rs | 8 +---- .../lighthouse_network/src/types/partial.rs | 36 ++----------------- .../lighthouse_network/src/types/pubsub.rs | 15 ++------ .../gossip_methods.rs | 2 -- .../src/network_beacon_processor/mod.rs | 1 - beacon_node/network/src/router.rs | 2 +- 6 files changed, 7 insertions(+), 57 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 0a0fd361be6..590e286dc4a 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -531,14 +531,8 @@ fn publish_column_sidecars( // For block producers, eagerly send all column data via partial messages // Gossipsub will handle per-peer diffing let partial = (*data_col).clone().into_partial(); - // Pass bitmap of cells to eagerly send - for block producer, send all cells - let eager_data = Some(partial.column.sidecar.cells_present_bitmap.clone()); [ - PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - partial.column, - eager_data, - ))), + PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, partial.column))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] }) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 03659d64aff..c307ce3bf7e 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -9,41 +9,14 @@ use types::data::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataCo #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, - send_eager: Option>, } impl PartialDataColumnSidecarMessage { pub fn new(partial_column: Arc>) -> Self { - PartialDataColumnSidecarMessage { - partial_column, - send_eager: None, - } - } - - pub fn eagerly_send(&mut self, cells: &CellBitmap) { - let Some(eager) = self - .partial_column - .sidecar - .clone_filter(|idx| cells.get(idx).unwrap_or(false)) - else { - return; - }; - - self.send_eager = Some(SendEager { - data: eager.as_ssz_bytes(), - metadata: eager.cells_present_bitmap.into(), - }) + PartialDataColumnSidecarMessage { partial_column } } } -#[derive(Debug, Clone, PartialEq)] -struct SendEager { - /// The encoded message to send eagerly, i.e. when we have no metadata for that peer. - data: Vec, - /// The metadata to associate with a peer after sending it the eager message. - metadata: CellBitmapMetadata, -} - #[derive(Debug, Clone, PartialEq, Eq)] struct CellBitmapMetadata { bitmap: CellBitmap, @@ -100,12 +73,7 @@ impl Partial for PartialDataColumnSidecarMessage { match metadata { None => Ok(PartialAction { need: false, - send: self.send_eager.clone().map(|eager| { - ( - eager.data, - Box::new(eager.metadata) as Box, - ) - }), + send: None, }), Some(metadata) => { let peer_has = CellBitmap::::from_ssz_bytes(metadata) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 81004240223..16abf581cdb 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -10,7 +10,7 @@ use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::data::partial_data_column_sidecar::{ - CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, + DanglingPartialDataColumn, PartialDataColumnSidecar, }; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, @@ -24,11 +24,7 @@ use types::{ SyncCommitteeMessage, SyncSubnetId, }; -type PartialDataColumnSidecarTuple = ( - DataColumnSubnetId, - Arc>, - Option>, -); +type PartialDataColumnSidecarTuple = (DataColumnSubnetId, Arc>); #[derive(Debug, Clone, PartialEq)] pub enum PubsubMessage { @@ -428,7 +424,6 @@ impl PubsubMessage { Ok(Self::PartialDataColumnSidecar(Box::new(( *id, Arc::new(data_column), - None, )))) } other => Err(format!("Partial message unsupported for topic: {other}")), @@ -448,11 +443,7 @@ impl PubsubMessage { PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::PartialDataColumnSidecar(data) => { let sidecar = &data.1; - let eager_cells = &data.2; - let mut message = PartialDataColumnSidecarMessage::new(sidecar.clone()); - if let Some(eager_cells) = eager_cells { - message.eagerly_send(eager_cells); - } + let message = PartialDataColumnSidecarMessage::new(sidecar.clone()); return EncodedPubsubMessage::PartialDataColumnSidecarMessage(message); } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 5f14d256263..1134455ab46 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1435,7 +1435,6 @@ impl NetworkBeaconProcessor { PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, updated_partial.column.clone(), - None, ))), PubsubMessage::DataColumnSidecar(Box::new(( subnet, @@ -1463,7 +1462,6 @@ impl NetworkBeaconProcessor { messages: vec![PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, updated_partial.column.clone(), - None, )))], }); } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 4af7597951a..93f76f0b580 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -815,7 +815,6 @@ impl NetworkBeaconProcessor { &self_cloned.chain.spec, ), partial.column.clone(), - None, ))) }) .collect(), diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 2659a2e8283..289186c182e 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -385,7 +385,7 @@ impl Router { ) } PubsubMessage::PartialDataColumnSidecar(data) => { - let (_, column_sidecar, _) = *data; + let (_, column_sidecar) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor .send_gossip_partial_data_column_sidecar( From 75a69a2bb8d3fdb3f5addd50c6aa37c00098e3b2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 15 Jan 2026 21:34:50 +0100 Subject: [PATCH 052/146] remove VerifiablePartialDataColumn --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +- .../src/data_availability_checker.rs | 19 ++-- .../src/data_column_verification.rs | 104 ++++++------------ .../beacon_chain/src/fetch_blobs/mod.rs | 6 +- beacon_node/beacon_chain/src/kzg_utils.rs | 64 ++++++----- .../src/partial_data_column_assembler.rs | 48 ++++---- beacon_node/http_api/src/publish_blocks.rs | 2 +- beacon_node/network/src/metrics.rs | 8 -- .../gossip_methods.rs | 84 ++++++-------- .../src/network_beacon_processor/mod.rs | 5 +- .../types/src/data/data_column_sidecar.rs | 33 +++--- .../src/data/partial_data_column_sidecar.rs | 103 ++--------------- crypto/kzg/src/lib.rs | 10 +- 13 files changed, 166 insertions(+), 324 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8062f74ef10..f52db3fcb6c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -136,7 +136,7 @@ use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace use tree_hash::TreeHash; use types::data::{ColumnIndex, FixedBlobSidecarList}; use types::execution::BlockProductionVersion; -use types::partial_data_column_sidecar::VerifiablePartialDataColumn; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -2200,7 +2200,7 @@ impl BeaconChain { #[instrument(skip_all, level = "trace")] pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, - data_column_sidecar: Arc>, + data_column_sidecar: Arc>, ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index eedc56e64a3..9b831b5fc15 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -42,7 +42,7 @@ use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; use types::DataColumnSidecar; use types::new_non_zero_usize; -use types::partial_data_column_sidecar::VerifiablePartialDataColumn; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -229,9 +229,9 @@ impl DataAvailabilityChecker { pub fn determine_missing_cells_partial( &self, block_root: &Hash256, - data_column: &VerifiablePartialDataColumn, + data_column: &DanglingPartialDataColumn, ) -> Option> { - let column_index = data_column.column.index; + let column_index = data_column.index; // Check DA checker cache first - if we have a full column cached, nothing is missing if self @@ -253,7 +253,6 @@ impl DataAvailabilityChecker { // No cached data, return all present cells as "missing" (new data we want) let incoming_cells: Vec = data_column - .column .sidecar .cells_present_bitmap .iter() @@ -970,9 +969,9 @@ impl MaybeAvailableBlock { /// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. fn compare_full_to_partial( incoming_cell_count: usize, - cached_partial: &VerifiablePartialDataColumn, + cached_partial: &DanglingPartialDataColumn, ) -> Option> { - let cached_bitmap = &cached_partial.column.sidecar.cells_present_bitmap; + let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; // Check that cell counts match if cached_bitmap.len() != incoming_cell_count { @@ -999,11 +998,11 @@ fn compare_full_to_partial( /// Compare an incoming partial column against a cached partial. /// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. fn compare_partial_to_partial( - incoming: &VerifiablePartialDataColumn, - cached_partial: &VerifiablePartialDataColumn, + incoming: &DanglingPartialDataColumn, + cached_partial: &DanglingPartialDataColumn, ) -> Option> { - let incoming_bitmap = &incoming.column.sidecar.cells_present_bitmap; - let cached_bitmap = &cached_partial.column.sidecar.cells_present_bitmap; + let incoming_bitmap = &incoming.sidecar.cells_present_bitmap; + let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; // Check that cell counts match if incoming_bitmap.len() != cached_bitmap.len() { diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 8d2ce85942a..7f4d324c50a 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -18,10 +18,10 @@ use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; use types::data::ColumnIndex; -use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, - SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip data column. @@ -302,27 +302,27 @@ pub struct GossipVerifiedPartialDataColumn GossipVerifiedPartialDataColumn { pub fn new( - column_sidecar: Arc>, + column_sidecar: Arc>, chain: &BeaconChain, ) -> Result { validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) } /// Create a `GossipVerifiedPartialDataColumn` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { + pub fn __new_for_testing(column_sidecar: Arc>) -> Self { Self { - block_root: column_sidecar.block_root(), + block_root: column_sidecar.block_root, data_column: KzgVerifiedPartialDataColumn::__new_for_testing(column_sidecar), _phantom: Default::default(), } } - pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { + pub fn as_data_column(&self) -> &DanglingPartialDataColumn { self.data_column.as_data_column() } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc> { self.data_column.clone_data_column() } @@ -331,7 +331,15 @@ impl GossipVerifiedPartialDataColum } pub fn slot(&self) -> Slot { - self.data_column.as_data_column().slot() + self.data_column + .as_data_column() + .sidecar + .header + .first() + .expect("This was verified, so we have the header") + .signed_block_header + .message + .slot } pub fn index(&self) -> ColumnIndex { @@ -400,41 +408,41 @@ impl KzgVerifiedDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedPartialDataColumn { - data: Arc>, + data: Arc>, } impl KzgVerifiedPartialDataColumn { pub fn new( - data_column: Arc>, + data_column: Arc>, kzg: &Kzg, ) -> Result, KzgError)> { verify_kzg_for_partial_data_column(data_column, kzg) } /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { + pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { Self { data: data_column } } /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc>) -> Self { + pub fn from_execution_verified(data_column: Arc>) -> Self { Self { data: data_column } } - pub fn to_data_column(self) -> Arc> { + pub fn to_data_column(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { + pub fn as_data_column(&self) -> &DanglingPartialDataColumn { &self.data } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc> { self.data.clone() } pub fn index(&self) -> ColumnIndex { - self.data.index() + self.data.index } } @@ -550,7 +558,7 @@ impl KzgVerifiedCustodyDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedCustodyPartialDataColumn { - data: Arc>, + data: Arc>, } impl KzgVerifiedCustodyPartialDataColumn { @@ -562,63 +570,20 @@ impl KzgVerifiedCustodyPartialDataColumn { } } - pub fn into_inner(self) -> Arc> { + pub fn into_inner(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &VerifiablePartialDataColumn { + pub fn as_data_column(&self) -> &DanglingPartialDataColumn { &self.data } - pub fn clone_arc(&self) -> Arc> { + pub fn clone_arc(&self) -> Arc> { self.data.clone() } pub fn index(&self) -> ColumnIndex { - self.data.index() - } - - // TODO(dknopik): This clones in far too many cases - // https://github.com/sigp/lighthouse/issues/XXXX - pub fn try_as_full( - &self, - block: Option<&SignedBeaconBlock>, - ) -> Option> { - self.data - .as_full(block) - .map(|full| KzgVerifiedCustodyDataColumn { - data: Arc::new(full.into_owned()), - }) - } - - pub fn merge(&mut self, other: &Self) -> bool { - let to_be_added = other - .data - .column - .sidecar - .cells_present_bitmap - .difference(&self.data.column.sidecar.cells_present_bitmap); - if to_be_added.is_zero() { - return false; - }; - - let Some(merged) = self.data.column.sidecar.merge(&other.data.column.sidecar) else { - return false; - }; - - *self = Self { - data: Arc::new(VerifiablePartialDataColumn { - column: Arc::new(DanglingPartialDataColumn { - block_root: self.data.column.block_root, - index: self.data.column.index, - sidecar: merged, - }), - kzg_commitments: self.data.kzg_commitments.clone(), - slot: self.data.slot, - }), - }; - - true + self.data.index } } @@ -640,7 +605,7 @@ pub fn verify_kzg_for_data_column( /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( - data_column: Arc>, + data_column: Arc>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); @@ -727,15 +692,10 @@ pub fn validate_partial_data_column_sidecar_for_gossip< T: BeaconChainTypes, O: ObservationStrategy, >( - data_column: Arc>, + data_column: Arc>, chain: &BeaconChain, ) -> Result, GossipDataColumnError> { - // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply - // https://github.com/sigp/lighthouse/issues/XXXX - let column_slot = data_column.slot(); - let block_root = data_column.block_root(); - verify_sidecar_not_from_future_slot(chain, column_slot)?; - verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; + let block_root = data_column.block_root; let filtered_column = if let Some(missing_cells) = chain .data_availability_checker diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 7b41982c477..651a62a5ffa 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -36,7 +36,7 @@ use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_h use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; use types::data::{BlobSidecarError, DataColumnSidecarError}; -use types::partial_data_column_sidecar::VerifiablePartialDataColumn; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::{ BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, @@ -51,7 +51,7 @@ pub enum EngineGetBlobsOutput { /// Complete columns ready to be imported into the `DataAvailabilityChecker`. CompleteColumns(Vec>>), /// Incomplete partial columns that need assembly via gossip - PartialColumns(Vec>>), + PartialColumns(Vec>>), } #[derive(Debug)] @@ -430,7 +430,7 @@ async fn compute_custody_columns_to_import( .map(|data_columns| { data_columns .into_iter() - .filter(|col| custody_columns_indices.contains(&col.index())) + .filter(|col| custody_columns_indices.contains(&col.index)) .map(|col| { KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( KzgVerifiedPartialDataColumn::from_execution_verified(col), diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index a2d533ce096..f376f599902 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -9,7 +9,7 @@ use tracing::instrument; use types::data::{Cell, DataColumn, DataColumnSidecarError}; use types::kzg_ext::KzgCommitments; use types::partial_data_column_sidecar::{ - CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, + CellBitmap, DanglingPartialDataColumn, PartialDataColumnHeader, }; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, @@ -102,7 +102,7 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. pub fn validate_partial_data_columns<'a, E: EthSpec>( kzg: &Kzg, - data_column_iter: impl Iterator>>, + data_column_iter: impl Iterator>>, ) -> Result<(), (Option, KzgError)> { let mut cells = Vec::new(); let mut proofs = Vec::new(); @@ -110,8 +110,8 @@ pub fn validate_partial_data_columns<'a, E: EthSpec>( let mut commitments = Vec::new(); for data_column in data_column_iter { - let col_index = data_column.column.index; - let sidecar = &data_column.column.sidecar; + let col_index = data_column.index; + let sidecar = &data_column.sidecar; if sidecar.column.is_empty() { return Err((Some(col_index), KzgError::KzgVerificationFailed)); @@ -120,11 +120,15 @@ pub fn validate_partial_data_columns<'a, E: EthSpec>( // Partial columns have a bitmap indicating present cells // We iterate over the bitmap and only process present cells let mut present_iterator = sidecar.column.iter().zip(sidecar.kzg_proofs.iter()); - for (present, commitment) in sidecar - .cells_present_bitmap - .iter() - .zip(data_column.kzg_commitments.iter()) - { + for (present, commitment) in sidecar.cells_present_bitmap.iter().zip( + data_column + .sidecar + .header + .first() + .ok_or((Some(col_index), KzgError::PartialWithoutHeader))? + .kzg_commitments + .iter(), + ) { if present { let (cell, proof) = present_iterator.next().ok_or(( Some(col_index), @@ -285,7 +289,7 @@ pub fn blobs_to_partial_data_columns( block: &SignedBeaconBlock, kzg: &Kzg, spec: &ChainSpec, -) -> Result>>, DataColumnSidecarError> { +) -> Result>>, DataColumnSidecarError> { if blobs_and_proofs.is_empty() { return Ok(vec![]); } @@ -295,6 +299,7 @@ pub fn blobs_to_partial_data_columns( .body() .blob_kzg_commitments() .map_err(|_err| DataColumnSidecarError::PreDeneb)?; + let kzg_commitments_inclusion_proof = block.message().body().kzg_commitments_merkle_proof()?; let signed_block_header = block.signed_block_header(); let blob_cells_and_proofs_vec = blobs_and_proofs @@ -323,6 +328,7 @@ pub fn blobs_to_partial_data_columns( build_partial_data_columns( kzg_commitments.clone(), + kzg_commitments_inclusion_proof, signed_block_header, blob_cells_and_proofs_vec, spec, @@ -415,10 +421,11 @@ pub(crate) fn build_data_column_sidecars( pub(crate) fn build_partial_data_columns( kzg_commitments: KzgCommitments, + kzg_commitments_inclusion_proof: FixedVector, signed_block_header: SignedBeaconBlockHeader, blob_cells_and_proofs_vec: Vec>, spec: &ChainSpec, -) -> Result>>, String> { +) -> Result>>, String> { let number_of_columns = E::number_of_columns(); let max_blobs_per_block = spec .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) @@ -470,27 +477,28 @@ pub(crate) fn build_partial_data_columns( } } - let sidecars: Result>>, String> = columns + let sidecars: Result>>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() .map(|(index, (col, proofs))| { - Ok(Arc::new(VerifiablePartialDataColumn { - column: Arc::new(DanglingPartialDataColumn { - block_root: signed_block_header.message.canonical_root(), - index: index as u64, - sidecar: PartialDataColumnSidecar { - cells_present_bitmap: bitmap.clone(), - column: DataColumn::::try_from(col) - .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - kzg_proofs: VariableList::try_from(proofs) - .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - header: VariableList::empty(), - }, - }), - kzg_commitments: kzg_commitments.clone(), - slot: signed_block_header.message.slot, - })) + let column = DanglingPartialDataColumn { + block_root: signed_block_header.message.canonical_root(), + index: index as u64, + sidecar: types::partial_data_column_sidecar::PartialDataColumnSidecar { + cells_present_bitmap: bitmap.clone(), + column: DataColumn::::try_from(col) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, + kzg_proofs: VariableList::try_from(proofs) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, + header: VariableList::repeat_full(PartialDataColumnHeader { + kzg_commitments: kzg_commitments.clone(), + signed_block_header: signed_block_header.clone(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }), + }, + }; + Ok(Arc::new(column)) }) .collect(); diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index ef22d8f0403..362c8043eec 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -4,7 +4,7 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; -use types::partial_data_column_sidecar::VerifiablePartialDataColumn; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256, KzgCommitments, Slot}; /// Assembles partial data columns into complete columns @@ -20,20 +20,20 @@ struct PartialAssembly { slot: Slot, kzg_commitments: KzgCommitments, /// Map of column_index -> partial column being assembled - columns: HashMap>>, + columns: HashMap>>, } /// Result of merging a partial column pub enum PartialMergeResult { /// Merge was successful but column is still incomplete Incomplete { - updated_partial: Arc>, + updated_partial: Arc>, }, /// Merge completed the column Completed { full_column: Arc>, /// The updated partial for publishing (same as full but as partial type) - updated_partial: Arc>, + updated_partial: Arc>, }, } @@ -67,10 +67,10 @@ impl PartialDataColumnAssembler { for custody_col in custody_columns { let partial = custody_col.into_inner(); - let column_index = partial.column.index; + let column_index = partial.index; // Check if this partial is already complete - if partial.column.sidecar.is_complete() { + if partial.sidecar.is_complete() { // Convert to full column if let Some(full) = Self::partial_to_full(&partial, &kzg_commitments) { result.complete_columns.push(full); @@ -96,26 +96,20 @@ impl PartialDataColumnAssembler { pub fn merge_partial( &self, block_root: Hash256, - partial: Arc>, + partial: Arc>, ) -> Option> { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_mut(&block_root)?; - let column_index = partial.column.index; + let column_index = partial.index; let merged = if let Some(existing) = assembly.columns.get(&column_index) { // Merge with existing partial - let merged_sidecar = existing.column.sidecar.merge(&partial.column.sidecar)?; - Arc::new(VerifiablePartialDataColumn { - column: Arc::new( - types::partial_data_column_sidecar::DanglingPartialDataColumn { - block_root: existing.column.block_root, - index: existing.column.index, - sidecar: merged_sidecar, - }, - ), - kzg_commitments: existing.kzg_commitments.clone(), - slot: existing.slot, + let merged_sidecar = existing.sidecar.merge(&partial.sidecar)?; + Arc::new(DanglingPartialDataColumn { + block_root: existing.block_root, + index: existing.index, + sidecar: merged_sidecar, }) } else { // First time seeing this column index for this block @@ -123,7 +117,7 @@ impl PartialDataColumnAssembler { }; // Check if merged column is now complete - if merged.column.sidecar.is_complete() { + if merged.sidecar.is_complete() { // Remove from assembly since it's complete assembly.columns.remove(&column_index); @@ -149,7 +143,7 @@ impl PartialDataColumnAssembler { &self, block_root: &Hash256, column_index: ColumnIndex, - ) -> Option>> { + ) -> Option>> { self.assemblies .read() .peek(block_root)? @@ -194,18 +188,18 @@ impl PartialDataColumnAssembler { /// Convert a complete partial to a full DataColumnSidecar fn partial_to_full( - partial: &VerifiablePartialDataColumn, + partial: &DanglingPartialDataColumn, kzg_commitments: &KzgCommitments, ) -> Option>> { // Use the as_full method - requires a block for signed header // For now we'll construct it manually since we have all the data - if !partial.column.sidecar.is_complete() { + if !partial.sidecar.is_complete() { return None; } - let expected = partial.column.sidecar.cells_present_bitmap.len(); - if partial.column.sidecar.column.len() != expected - || partial.column.sidecar.kzg_proofs.len() != expected + let expected = partial.sidecar.cells_present_bitmap.len(); + if partial.sidecar.column.len() != expected + || partial.sidecar.kzg_proofs.len() != expected || kzg_commitments.len() != expected { return None; @@ -229,5 +223,5 @@ pub struct InitResult { /// Columns that were already complete pub complete_columns: Vec>>, /// Columns that need further assembly via gossip - pub incomplete_partials: Vec>>, + pub incomplete_partials: Vec>>, } diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 590e286dc4a..87c4103f24f 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -532,7 +532,7 @@ fn publish_column_sidecars( // Gossipsub will handle per-peer diffing let partial = (*data_col).clone().into_partial(); [ - PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, partial.column))), + PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, Arc::new(partial)))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] }) diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index efaae60267f..7a05bfe560e 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -151,14 +151,6 @@ pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: L "Total number of gossip partial data column sidecar verified for propagation.", ) }); -pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL: LazyLock< - Result, -> = LazyLock::new(|| { - try_create_int_counter( - "beacon_processor_gossip_partial_data_column_cached_total", - "Total number of gossip partial data column sidecars received before their block.", - ) -}); // Gossip Exits. pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 1134455ab46..ff555134124 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -12,8 +12,8 @@ use beacon_chain::data_column_verification::{ use beacon_chain::partial_data_column_assembler::PartialMergeResult; use beacon_chain::store::Error; use beacon_chain::{ - AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, - BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, + GossipVerifiedBlock, NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -49,7 +49,7 @@ use types::{ Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, }; -use beacon_processor::work_reprocessing_queue::{QueuedColumnReconstruction, QueuedPartialColumn}; +use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; use beacon_processor::{ DuplicateCache, GossipAggregatePackage, GossipAttestationBatch, work_reprocessing_queue::{ @@ -57,7 +57,7 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -786,46 +786,29 @@ impl NetworkBeaconProcessor { peer_id: PeerId, column_sidecar: Arc>, seen_duration: Duration, - allow_reprocess: bool, ) { - let partial_column = match self - .chain - .data_availability_checker - .get_cached_block(&column_sidecar.block_root) - { - Some( - BlockProcessStatus::ExecutionValidated(block) - | BlockProcessStatus::NotValidated(block, _), - ) => Some(VerifiablePartialDataColumn::from_dangling_and_block( - column_sidecar.clone(), - &block, - )), - None | Some(BlockProcessStatus::Unknown) => None, - }; - - match partial_column { - Some(Ok(partial_column)) => { - debug!("Received partial while having block"); - self.process_gossip_verifiable_partial_data_column_sidecar( - peer_id, - Arc::new(partial_column), - seen_duration, - ) - .await; - } - Some(Err(err)) => { - warn!(?err, "Error creating verifiable partial data column"); - } - None => self.handle_dangling_partial_data_column_sidecar_missing_block( - peer_id, - column_sidecar, - seen_duration, - allow_reprocess, - ), - } + // TODO(dknopik): add check and cache for header + + // TODO(dknopik): make this more elegant with above todo + let slot = column_sidecar + .sidecar + .header + .first() + .unwrap() + .signed_block_header + .message + .slot; + + self.process_gossip_verifiable_partial_data_column_sidecar( + peer_id, + column_sidecar, + slot, + seen_duration, + ) + .await; } - fn handle_dangling_partial_data_column_sidecar_missing_block( + /*fn handle_dangling_partial_data_column_sidecar_missing_block( self: &Arc, peer_id: PeerId, column_sidecar: Arc>, @@ -876,24 +859,24 @@ impl NetworkBeaconProcessor { { error!("Failed to send attestation for re-processing") } - } + }*/ #[instrument( name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, parent = None, level = "debug", skip_all, - fields(slot = %column_sidecar.slot(), block_root = ?column_sidecar.block_root(), index = column_sidecar.index()), + fields(slot = %slot, block_root = ?column_sidecar.block_root, index = column_sidecar.index), )] pub async fn process_gossip_verifiable_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: Arc>, + slot: Slot, seen_duration: Duration, ) { - let slot = column_sidecar.slot(); - let block_root = column_sidecar.block_root(); - let index = column_sidecar.index(); + let block_root = column_sidecar.block_root; + let index = column_sidecar.index; let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. metrics::observe_duration( @@ -1404,7 +1387,6 @@ impl NetworkBeaconProcessor { ) { let processing_start_time = Instant::now(); let block_root = verified_partial.block_root(); - let _data_column_slot = verified_partial.slot(); let data_column_index = verified_partial.index(); let result = self @@ -1434,7 +1416,7 @@ impl NetworkBeaconProcessor { messages: vec![ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - updated_partial.column.clone(), + updated_partial.clone(), ))), PubsubMessage::DataColumnSidecar(Box::new(( subnet, @@ -1455,13 +1437,13 @@ impl NetworkBeaconProcessor { // Publish the updated partial let subnet = DataColumnSubnetId::from_column_index( - updated_partial.column.index, + updated_partial.index, &self.chain.spec, ); self.send_network_message(NetworkMessage::Publish { messages: vec![PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - updated_partial.column.clone(), + updated_partial.clone(), )))], }); } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 93f76f0b580..d71cbf87250 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -264,7 +264,6 @@ impl NetworkBeaconProcessor { peer_id, column_sidecar, seen_timestamp, - true, ) .await }; @@ -811,10 +810,10 @@ impl NetworkBeaconProcessor { .map(|partial| { PubsubMessage::PartialDataColumnSidecar(Box::new(( DataColumnSubnetId::from_column_index( - partial.column.index, + partial.index, &self_cloned.chain.spec, ), - partial.column.clone(), + partial.clone(), ))) }) .collect(), diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 0f5fa4f00ec..b57b18b0fa0 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -1,7 +1,6 @@ use crate::data::partial_data_column_sidecar::CellBitmap; use crate::data::partial_data_column_sidecar::DanglingPartialDataColumn; use crate::data::partial_data_column_sidecar::PartialDataColumnSidecar; -use crate::data::partial_data_column_sidecar::VerifiablePartialDataColumn; use std::sync::Arc; use bls::Signature; @@ -137,7 +136,7 @@ impl DataColumnSidecar { } /// Convert this full data column into a verifiable partial data column. - pub fn into_partial(self) -> VerifiablePartialDataColumn { + pub fn into_partial(self) -> DanglingPartialDataColumn { let cell_count = self.column.len(); let mut bitmap = CellBitmap::::with_capacity(cell_count).expect("our column has the same bound"); @@ -147,23 +146,19 @@ impl DataColumnSidecar { .expect("The correct size is initialized right above"); } - VerifiablePartialDataColumn { - slot: self.slot(), - column: Arc::new(DanglingPartialDataColumn { - block_root: self.block_root(), - index: self.index, - sidecar: PartialDataColumnSidecar { - cells_present_bitmap: bitmap, - column: self.column, - kzg_proofs: self.kzg_proofs, - header: VariableList::repeat_full(PartialDataColumnHeader { - kzg_commitments: self.kzg_commitments.clone(), - signed_block_header: self.signed_block_header, - kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof, - }), - }, - }), - kzg_commitments: self.kzg_commitments, + DanglingPartialDataColumn { + block_root: self.block_root(), + index: self.index, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: self.column, + kzg_proofs: self.kzg_proofs, + header: VariableList::repeat_full(PartialDataColumnHeader { + kzg_commitments: self.kzg_commitments.clone(), + signed_block_header: self.signed_block_header, + kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof, + }), + }, } } } diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index dedf69812ea..45c65b915cf 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,15 +1,13 @@ use crate::test_utils::TestRandom; use crate::{ - AbstractExecPayload, Cell, ColumnIndex, DataColumn, DataColumnSidecar, EthSpec, Hash256, - KzgCommitments, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + Cell, ColumnIndex, DataColumn, DataColumnSidecar, EthSpec, Hash256, KzgCommitments, + SignedBeaconBlockHeader, }; use educe::Educe; use kzg::KzgProof; use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; -use std::borrow::Cow; -use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::U1; @@ -230,102 +228,15 @@ pub struct DanglingPartialDataColumn { pub sidecar: PartialDataColumnSidecar, } -#[derive(Debug, Clone, PartialEq)] -pub struct VerifiablePartialDataColumn { - pub column: Arc>, - pub kzg_commitments: KzgCommitments, - pub slot: Slot, -} - -// TODO(dknopik): Is there an existing approriate error type? -#[derive(Debug)] -pub enum PartialDataColumnMatchingError { - MismatchingBlock, - InvalidForkBlock, -} - -impl VerifiablePartialDataColumn { - pub fn from_dangling_and_block>( - column: Arc>, - block: &SignedBeaconBlock, - ) -> Result { - if column.block_root != block.canonical_root() { - return Err(PartialDataColumnMatchingError::MismatchingBlock); - } - - let kzg_commitments = block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| PartialDataColumnMatchingError::InvalidForkBlock)? - .clone(); - - Ok(VerifiablePartialDataColumn { - column, - kzg_commitments, - slot: block.slot(), - }) - } - +impl DanglingPartialDataColumn { pub fn clone_filter(&self, filter: F) -> Option where F: Fn(usize) -> bool, { - Some(VerifiablePartialDataColumn { - column: Arc::new(DanglingPartialDataColumn { - sidecar: self.column.sidecar.clone_filter(filter)?, - block_root: self.column.block_root, - index: self.column.index, - }), - kzg_commitments: self.kzg_commitments.clone(), - slot: self.slot, + Some(DanglingPartialDataColumn { + sidecar: self.sidecar.clone_filter(filter)?, + block_root: self.block_root, + index: self.index, }) } - - pub fn slot(&self) -> Slot { - self.slot - } - - pub fn index(&self) -> ColumnIndex { - self.column.index - } - - pub fn block_root(&self) -> Hash256 { - self.column.block_root - } - - /// Convert this partial column into a full `DataColumnSidecar` if all cells are present. - /// Requires the signed beacon block to populate the header and inclusion proof. - pub fn as_full( - &self, - block: Option<&SignedBeaconBlock>, - ) -> Option>> { - // we definitely require the block - let block = block?; - - // we need to have all columns - if !self.column.sidecar.is_complete() { - return None; - } - - // we need to have the correct amount of everything - let expected = self.column.sidecar.cells_present_bitmap.len(); - if self.column.sidecar.column.len() != expected - || self.column.sidecar.kzg_proofs.len() != expected - || self.kzg_commitments.len() != expected - { - return None; - } - - let (signed_block_header, kzg_commitments_inclusion_proof) = - block.signed_block_header_and_kzg_commitments_proof().ok()?; - Some(Cow::Owned(DataColumnSidecar { - kzg_commitments_inclusion_proof, - index: self.column.index, - column: self.column.sidecar.column.clone(), - kzg_commitments: self.kzg_commitments.clone(), - kzg_proofs: self.column.sidecar.kzg_proofs.clone(), - signed_block_header, - })) - } } diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 0fe95b77230..05b016f596a 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -13,17 +13,17 @@ pub use crate::{ }; pub use c_kzg::{ - Blob, Bytes32, Bytes48, KzgSettings, BYTES_PER_BLOB, BYTES_PER_COMMITMENT, - BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, FIELD_ELEMENTS_PER_BLOB, + BYTES_PER_BLOB, BYTES_PER_COMMITMENT, BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, Blob, Bytes32, + Bytes48, FIELD_ELEMENTS_PER_BLOB, KzgSettings, }; use crate::trusted_setup::load_trusted_setup; use rayon::prelude::*; pub use rust_eth_kzg::{ - constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, + constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, }; -use tracing::{instrument, Span}; +use tracing::{Span, instrument}; /// Disables the fixed-base multi-scalar multiplication optimization for computing /// cell KZG proofs, because `rust-eth-kzg` already handles the precomputation. @@ -55,6 +55,8 @@ pub enum Error { ReconstructFailed(String), /// Kzg was not initialized with PeerDAS enabled. DASContextUninitialized, + /// A partial column was provided without necessary data + PartialWithoutHeader, } impl From for Error { From b0485deb1127383c3d5bf0ca789232e1f76157a7 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 16 Jan 2026 10:27:36 +0100 Subject: [PATCH 053/146] Use separate network flow for partial messages --- beacon_node/http_api/src/publish_blocks.rs | 46 +++++--- beacon_node/lighthouse_network/src/lib.rs | 4 +- .../lighthouse_network/src/service/mod.rs | 92 +++++++++++----- .../lighthouse_network/src/types/mod.rs | 3 +- .../lighthouse_network/src/types/pubsub.rs | 104 +++++------------- .../gossip_methods.rs | 28 ++--- .../src/network_beacon_processor/mod.rs | 15 +-- beacon_node/network/src/router.rs | 19 ++-- beacon_node/network/src/service.rs | 22 +++- 9 files changed, 165 insertions(+), 168 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 87c4103f24f..22fdc806720 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -524,21 +524,37 @@ fn publish_column_sidecars( .collect::>(); debug!(indices = ?dropped_indices, "Dropping data columns from publishing"); } - let pubsub_messages = data_column_sidecars - .into_iter() - .flat_map(|data_col| { - let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); - // For block producers, eagerly send all column data via partial messages - // Gossipsub will handle per-peer diffing - let partial = (*data_col).clone().into_partial(); - [ - PubsubMessage::PartialDataColumnSidecar(Box::new((subnet, Arc::new(partial)))), - PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), - ] - }) - .collect::>(); - crate::utils::publish_pubsub_messages(sender_clone, pubsub_messages) - .map_err(|_| BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish))) + let mut full_messages = Vec::new(); + let mut partial_columns = Vec::new(); + + for data_col in data_column_sidecars { + partial_columns.push(Arc::new((*data_col).clone().into_partial())); + + let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); + full_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( + subnet, data_col, + )))); + } + + // Publish full messages + if !full_messages.is_empty() { + crate::utils::publish_pubsub_messages(sender_clone, full_messages).map_err(|_| { + BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish)) + })?; + } + + // Publish partial messages + if !partial_columns.is_empty() { + crate::utils::publish_network_message( + sender_clone, + NetworkMessage::PublishPartial { + columns: partial_columns, + }, + ) + .map_err(|_| BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish)))?; + } + + Ok(()) } async fn post_block_import_logging_and_response( diff --git a/beacon_node/lighthouse_network/src/lib.rs b/beacon_node/lighthouse_network/src/lib.rs index 863a7a4a430..cc3b3a1e2e6 100644 --- a/beacon_node/lighthouse_network/src/lib.rs +++ b/beacon_node/lighthouse_network/src/lib.rs @@ -98,8 +98,8 @@ impl std::fmt::Display for ClearDialError<'_> { } pub use crate::types::{ - Enr, EnrSyncCommitteeBitfield, GossipTopic, NetworkGlobals, PubsubMessage, Subnet, - SubnetDiscovery, + Enr, EnrSyncCommitteeBitfield, GossipTopic, NetworkGlobals, PartialDataColumnSidecarMessage, + PubsubMessage, Subnet, SubnetDiscovery, decode_partial, }; pub use prometheus_client; diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index da3b8a19b75..69ad5b97741 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -15,11 +15,11 @@ use crate::rpc::{ RequestType, ResponseTermination, RpcResponse, RpcSuccessResponse, }; use crate::types::{ - EncodedPubsubMessage, GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, - SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, + GossipEncoding, GossipKind, GossipTopic, PartialDataColumnSidecarMessage, SnappyTransform, + Subnet, SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, subnet_from_topic_hash, }; -use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; +use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, decode_partial, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; @@ -42,10 +42,11 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; -use types::data::partial_data_column_sidecar::CellBitmap; +use types::data::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; use types::{ChainSpec, ForkName}; use types::{ - EnrForkId, EthSpec, ForkContext, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, + DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, Slot, SubnetId, + consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, }; use utils::{Context as ServiceContext, build_transport, strip_peer_id}; @@ -102,6 +103,13 @@ pub enum NetworkEvent { /// The message itself. message: PubsubMessage, }, + /// A partial data column sidecar received via gossipsub partial protocol. + PartialDataColumnSidecar { + /// The peer from which we received this message. + source: PeerId, + /// The partial column data. + column: Arc>, + }, /// Inform the network to send a Status to this peer. StatusPeer(PeerId), NewListenAddr(Multiaddr), @@ -851,19 +859,10 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let message_data = message.encode(); + let bytes = message.encode(); let gossipsub = self.gossipsub_mut(); let publish_topic: Topic = topic.clone().into(); - let (cache, result) = match message_data { - EncodedPubsubMessage::Full(bytes) => ( - Some(bytes.clone()), - gossipsub.publish(publish_topic, bytes).map(|_| ()), - ), - EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { - (None, gossipsub.publish_partial(publish_topic, partial)) - } - }; - if let Err(e) = result { + if let Err(e) = gossipsub.publish(publish_topic, bytes.clone()) { match e { PublishError::Duplicate => { debug!( @@ -900,9 +899,7 @@ impl Network { } } - if let PublishError::NoPeersSubscribedToTopic = e - && let Some(bytes) = cache - { + if let PublishError::NoPeersSubscribedToTopic = e { self.gossip_cache.insert(topic, bytes); } } @@ -910,6 +907,50 @@ impl Network { } } + /// Publishes partial data column sidecars to the gossipsub network. + pub fn publish_partial(&mut self, columns: Vec>>) { + for column in columns { + let subnet = + DataColumnSubnetId::from_column_index(column.index, &self.fork_context.spec); + let topic = GossipTopic::new( + GossipKind::DataColumnSidecar(subnet), + GossipEncoding::default(), + self.enr_fork_id.fork_digest, + ); + let partial_message = PartialDataColumnSidecarMessage::new(column); + let publish_topic: Topic = topic.clone().into(); + + if let Err(e) = self + .gossipsub_mut() + .publish_partial(publish_topic, partial_message) + { + match e { + PublishError::Duplicate => { + debug!( + kind = %topic.kind(), + "Attempted to publish duplicate partial message" + ); + } + ref e => { + warn!( + error = ?e, + kind = %topic.kind(), + "Could not publish partial message" + ); + } + } + + // add to metrics + if let Some(v) = metrics::get_int_gauge( + &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC, + &[&format!("{:?}", topic.kind())], + ) { + v.inc() + }; + } + } + } + /// Informs the gossipsub about the result of a message validation. /// If the message is valid it will get propagated by gossipsub. pub fn report_message_validation_result( @@ -1351,7 +1392,7 @@ impl Network { } if let Some(message) = message { - match PubsubMessage::decode_partial(&topic_hash, &group_id, &message) { + match decode_partial::(&topic_hash, &group_id, &message) { Err(error) => { debug!( topic = ?topic_hash, @@ -1366,18 +1407,17 @@ impl Network { // MessageAcceptance::Reject, //); } - Ok(message) => { + Ok(column) => { debug!( - %message, + block_root = %column.block_root, + index = column.index, %peer_id, "Decoded partial message" ); // Notify the network - return Some(NetworkEvent::PubsubMessage { - id: MessageId::new(&[]), // TODO(dknopik): waht to send + return Some(NetworkEvent::PartialDataColumnSidecar { source: peer_id, - topic: topic_hash, - message, + column, }); } } diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 85e4e3a28df..aede15d4a0f 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -14,7 +14,8 @@ pub type Enr = discv5::enr::Enr; pub use eth2::lighthouse::sync_state::{BackFillState, CustodyBackFillState, SyncState}; pub use globals::NetworkGlobals; -pub use pubsub::{EncodedPubsubMessage, PubsubMessage, SnappyTransform}; +pub use partial::PartialDataColumnSidecarMessage; +pub use pubsub::{PubsubMessage, SnappyTransform, decode_partial}; pub use subnet::{Subnet, SubnetDiscovery}; pub use topics::{ GossipEncoding, GossipKind, GossipTopic, TopicConfig, all_topics_at_fork, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 16abf581cdb..ece24f52476 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,10 +1,7 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::types::partial::PartialDataColumnSidecarMessage; +use crate::TopicHash; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; -use crate::{Gossipsub, TopicHash}; -use gossipsub::{IdentTopic, PublishError}; -use libp2p::gossipsub; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; @@ -24,8 +21,6 @@ use types::{ SyncCommitteeMessage, SyncSubnetId, }; -type PartialDataColumnSidecarTuple = (DataColumnSubnetId, Arc>); - #[derive(Debug, Clone, PartialEq)] pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. @@ -34,9 +29,6 @@ pub enum PubsubMessage { BlobSidecar(Box<(u64, Arc>)>), /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), - /// Gossipsub message providing notification of a [`PartialDataColumnSidecar`] along with the subnet id where it was received. - /// TODO(dknopik) - it is time for this to move into its own enum. - PartialDataColumnSidecar(Box>), // TODO(dknopik): review `Arc` situation /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a `SingleAttestation` with its subnet id. @@ -146,9 +138,6 @@ impl PubsubMessage { PubsubMessage::DataColumnSidecar(column_sidecar_data) => { GossipKind::DataColumnSidecar(column_sidecar_data.0) } - PubsubMessage::PartialDataColumnSidecar(partial_column_sidecar_data) => { - GossipKind::DataColumnSidecar(partial_column_sidecar_data.0) - } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) @@ -406,46 +395,16 @@ impl PubsubMessage { } } - pub fn decode_partial(topic: &TopicHash, group: &[u8], data: &[u8]) -> Result { - match GossipTopic::decode(topic.as_str()) { - Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)), - Ok(gossip_topic) => match gossip_topic.kind() { - GossipKind::DataColumnSidecar(id) => { - let block_root = Hash256::from_ssz_bytes(group) - .map_err(|e| format!("Error decoding group: {:?}", e))?; - let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) - .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; - let data_column = DanglingPartialDataColumn { - block_root, - // Partial messages are spec'd under the assumption that there is one column per subnet. - index: **id, - sidecar, - }; - Ok(Self::PartialDataColumnSidecar(Box::new(( - *id, - Arc::new(data_column), - )))) - } - other => Err(format!("Partial message unsupported for topic: {other}")), - }, - } - } - /// Encodes a `PubsubMessage`. - pub fn encode(&self) -> EncodedPubsubMessage { + pub fn encode(&self) -> Vec { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the // messages for us. - let bytes = match &self { + match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), - PubsubMessage::PartialDataColumnSidecar(data) => { - let sidecar = &data.1; - let message = PartialDataColumnSidecarMessage::new(sidecar.clone()); - return EncodedPubsubMessage::PartialDataColumnSidecarMessage(message); - } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -456,12 +415,34 @@ impl PubsubMessage { PubsubMessage::BlsToExecutionChange(data) => data.as_ssz_bytes(), PubsubMessage::LightClientFinalityUpdate(data) => data.as_ssz_bytes(), PubsubMessage::LightClientOptimisticUpdate(data) => data.as_ssz_bytes(), - }; - EncodedPubsubMessage::Full(bytes) + } } +} - pub fn is_partial(&self) -> bool { - matches!(self, PubsubMessage::PartialDataColumnSidecar(_)) +/// Decodes incoming partial data column sidecar from gossipsub partial protocol. +pub fn decode_partial( + topic: &TopicHash, + group: &[u8], + data: &[u8], +) -> Result>, String> { + match GossipTopic::decode(topic.as_str()) { + Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)), + Ok(gossip_topic) => match gossip_topic.kind() { + GossipKind::DataColumnSidecar(id) => { + let block_root = Hash256::from_ssz_bytes(group) + .map_err(|e| format!("Error decoding group: {:?}", e))?; + let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) + .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; + let data_column = DanglingPartialDataColumn { + block_root, + // Partial messages are spec'd under the assumption that there is one column per subnet. + index: **id, + sidecar, + }; + Ok(Arc::new(data_column)) + } + other => Err(format!("Partial message unsupported for topic: {other}")), + }, } } @@ -486,13 +467,6 @@ impl std::fmt::Display for PubsubMessage { data.1.slot(), data.1.index, ), - PubsubMessage::PartialDataColumnSidecar(data) => write!( - f, - "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", - data.1.block_root, - data.1.index, - hex::encode(data.1.sidecar.cells_present_bitmap.as_slice()), - ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {:?}, aggregator_index: {}", @@ -530,23 +504,3 @@ impl std::fmt::Display for PubsubMessage { } } } - -pub enum EncodedPubsubMessage { - Full(Vec), - PartialDataColumnSidecarMessage(PartialDataColumnSidecarMessage), -} - -impl EncodedPubsubMessage { - pub fn do_publish( - self, - gossipsub: &mut Gossipsub, - topic: IdentTopic, - ) -> Result<(), PublishError> { - match self { - EncodedPubsubMessage::Full(bytes) => gossipsub.publish(topic, bytes).map(|_| ()), - EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { - gossipsub.publish_partial(topic, partial) - } - } - } -} diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index ff555134124..406772fb979 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1412,17 +1412,14 @@ impl NetworkBeaconProcessor { full_column.index, &self.chain.spec, ); + self.send_network_message(NetworkMessage::PublishPartial { + columns: vec![updated_partial.clone()], + }); self.send_network_message(NetworkMessage::Publish { - messages: vec![ - PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - updated_partial.clone(), - ))), - PubsubMessage::DataColumnSidecar(Box::new(( - subnet, - full_column.clone(), - ))), - ], + messages: vec![PubsubMessage::DataColumnSidecar(Box::new(( + subnet, + full_column.clone(), + )))], }); // Check if block is now fully available @@ -1436,15 +1433,8 @@ impl NetworkBeaconProcessor { ); // Publish the updated partial - let subnet = DataColumnSubnetId::from_column_index( - updated_partial.index, - &self.chain.spec, - ); - self.send_network_message(NetworkMessage::Publish { - messages: vec![PubsubMessage::PartialDataColumnSidecar(Box::new(( - subnet, - updated_partial.clone(), - )))], + self.send_network_message(NetworkMessage::PublishPartial { + columns: vec![updated_partial.clone()], }); } } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index d71cbf87250..292d4d5c7dd 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -804,19 +804,8 @@ impl NetworkBeaconProcessor { } EngineGetBlobsOutput::PartialColumns(partials) => { // Publish partial columns without eager send - self_cloned.send_network_message(NetworkMessage::Publish { - messages: partials - .into_iter() - .map(|partial| { - PubsubMessage::PartialDataColumnSidecar(Box::new(( - DataColumnSubnetId::from_column_index( - partial.index, - &self_cloned.chain.spec, - ), - partial.clone(), - ))) - }) - .collect(), + self_cloned.send_network_message(NetworkMessage::PublishPartial { + columns: partials, }) } }; diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 289186c182e..b58ef38826d 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -24,6 +24,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, trace, warn}; +use types::data::partial_data_column_sidecar::DanglingPartialDataColumn; use types::{BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, SignedBeaconBlock}; /// Handles messages from the network and routes them to the appropriate service to be handled. @@ -69,6 +70,8 @@ pub enum RouterMessage { /// message, the message itself and a bool which indicates if the message should be processed /// by the beacon chain after successful verification. PubsubMessage(MessageId, PeerId, PubsubMessage, bool), + /// A partial data column sidecar has been received via gossipsub partial protocol. + PartialDataColumnSidecar(PeerId, Arc>), /// The peer manager has requested we re-status a peer. StatusPeer(PeerId), /// The peer has an updated custody group count from METADATA. @@ -180,6 +183,11 @@ impl Router { RouterMessage::PubsubMessage(id, peer_id, gossip, should_process) => { self.handle_gossip(id, peer_id, gossip, should_process); } + RouterMessage::PartialDataColumnSidecar(peer_id, column) => self + .handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_partial_data_column_sidecar(peer_id, column, timestamp_now()), + ), } } @@ -384,17 +392,6 @@ impl Router { ), ) } - PubsubMessage::PartialDataColumnSidecar(data) => { - let (_, column_sidecar) = *data; - self.handle_beacon_processor_send_result( - self.network_beacon_processor - .send_gossip_partial_data_column_sidecar( - peer_id, - column_sidecar, - timestamp_now(), - ), - ) - } PubsubMessage::VoluntaryExit(exit) => { debug!(%peer_id, "Received a voluntary exit"); self.handle_beacon_processor_send_result( diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index b69f2d00227..b5016dd5661 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -38,6 +38,7 @@ use tokio::sync::mpsc; use tokio::time::Sleep; use tracing::{debug, error, info, trace, warn}; use typenum::Unsigned; +use types::data::partial_data_column_sidecar::DanglingPartialDataColumn; use types::{ EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, ValidatorSubscription, @@ -83,6 +84,10 @@ pub enum NetworkMessage { }, /// Publish a list of messages to the gossipsub protocol. Publish { messages: Vec> }, + /// Publish partial data column sidecars via the partial gossipsub protocol. + PublishPartial { + columns: Vec>>, + }, /// Validates a received gossipsub message. This will propagate the message on the network. ValidationResult { /// The peer that sent us the message. We don't send back to this peer. @@ -560,6 +565,9 @@ impl NetworkService { } } } + NetworkEvent::PartialDataColumnSidecar { source, column } => { + self.send_to_router(RouterMessage::PartialDataColumnSidecar(source, column)); + } NetworkEvent::NewListenAddr(multiaddr) => { self.network_globals .listen_multiaddrs @@ -643,12 +651,7 @@ impl NetworkService { NetworkMessage::Publish { messages } => { let mut topic_kinds = Vec::new(); for message in &messages { - let message_kind = if message.is_partial() { - "partial" - } else { - "full" - }; - let kind = (message.kind(), message_kind); + let kind = message.kind(); if !topic_kinds.contains(&kind) { topic_kinds.push(kind); } @@ -660,6 +663,13 @@ impl NetworkService { ); self.libp2p.publish(messages); } + NetworkMessage::PublishPartial { columns } => { + debug!( + count = columns.len(), + "Sending partial data column sidecars" + ); + self.libp2p.publish_partial(columns); + } NetworkMessage::ReportPeer { peer_id, action, From f5050732dc2fbf42f6eff2833e59f11a512c5e2d Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 15 Jan 2026 21:39:05 +0100 Subject: [PATCH 054/146] add component for tracking sent headers --- beacon_node/lighthouse_network/src/lib.rs | 4 +- .../lighthouse_network/src/service/mod.rs | 17 +++- .../service/partial_column_header_tracker.rs | 28 ++++++ .../lighthouse_network/src/types/mod.rs | 3 +- .../lighthouse_network/src/types/partial.rs | 96 +++++++++++++++---- .../lighthouse_network/src/types/pubsub.rs | 2 +- 6 files changed, 124 insertions(+), 26 deletions(-) create mode 100644 beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs diff --git a/beacon_node/lighthouse_network/src/lib.rs b/beacon_node/lighthouse_network/src/lib.rs index cc3b3a1e2e6..fdb6ff095e5 100644 --- a/beacon_node/lighthouse_network/src/lib.rs +++ b/beacon_node/lighthouse_network/src/lib.rs @@ -98,8 +98,8 @@ impl std::fmt::Display for ClearDialError<'_> { } pub use crate::types::{ - Enr, EnrSyncCommitteeBitfield, GossipTopic, NetworkGlobals, PartialDataColumnSidecarMessage, - PubsubMessage, Subnet, SubnetDiscovery, decode_partial, + Enr, EnrSyncCommitteeBitfield, GossipTopic, NetworkGlobals, PubsubMessage, Subnet, + SubnetDiscovery, decode_partial, }; pub use prometheus_client; diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 69ad5b97741..1ecde1771de 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -14,9 +14,10 @@ use crate::rpc::{ GoodbyeReason, HandlerErr, InboundRequestId, Protocol, RPC, RPCError, RPCMessage, RPCReceived, RequestType, ResponseTermination, RpcResponse, RpcSuccessResponse, }; +use crate::service::partial_column_header_tracker::PartialColumnHeaderTracker; use crate::types::{ - GossipEncoding, GossipKind, GossipTopic, PartialDataColumnSidecarMessage, SnappyTransform, - Subnet, SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, + GossipEncoding, GossipKind, GossipTopic, OutgoingPartialColumn, SnappyTransform, Subnet, + SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, subnet_from_topic_hash, }; use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, decode_partial, metrics}; @@ -53,7 +54,9 @@ use utils::{Context as ServiceContext, build_transport, strip_peer_id}; pub mod api_types; mod gossip_cache; pub mod gossipsub_scoring_parameters; +mod partial_column_header_tracker; pub mod utils; + /// The number of peers we target per subnet for discovery queries. pub const TARGET_SUBNET_PEERS: usize = 3; @@ -173,6 +176,7 @@ pub struct Network { /// The interval for updating gossipsub scores update_gossipsub_scores: tokio::time::Interval, gossip_cache: GossipCache, + partial_column_header_tracker: PartialColumnHeaderTracker, /// This node's PeerId. pub local_peer_id: PeerId, } @@ -518,6 +522,7 @@ impl Network { score_settings, update_gossipsub_scores, gossip_cache, + partial_column_header_tracker: PartialColumnHeaderTracker::new(), local_peer_id, }; @@ -917,7 +922,13 @@ impl Network { GossipEncoding::default(), self.enr_fork_id.fork_digest, ); - let partial_message = PartialDataColumnSidecarMessage::new(column); + let header_sent_set = self + .partial_column_header_tracker + .get_for_block(column.block_root); + let Ok(partial_message) = OutgoingPartialColumn::new(column, header_sent_set) else { + error!("Unable to publish partial column without header"); + continue; + }; let publish_topic: Topic = topic.clone().into(); if let Err(e) = self diff --git a/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs b/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs new file mode 100644 index 00000000000..720bbf7668c --- /dev/null +++ b/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs @@ -0,0 +1,28 @@ +use crate::types::HeaderSentSet; +use lru::LruCache; +use parking_lot::Mutex; +use std::collections::HashSet; +use std::num::NonZeroUsize; +use std::sync::Arc; +use types::Hash256; + +const MAX_BLOCKS: NonZeroUsize = NonZeroUsize::new(4).unwrap(); + +pub struct PartialColumnHeaderTracker { + blocks: LruCache, +} + +impl PartialColumnHeaderTracker { + pub fn new() -> Self { + PartialColumnHeaderTracker { + blocks: LruCache::new(MAX_BLOCKS), + } + } + + pub fn get_for_block(&mut self, hash: Hash256) -> HeaderSentSet { + Arc::clone( + self.blocks + .get_or_insert(hash, || Arc::new(Mutex::new(HashSet::new()))), + ) + } +} diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index aede15d4a0f..d0173e5b9ac 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -14,7 +14,8 @@ pub type Enr = discv5::enr::Enr; pub use eth2::lighthouse::sync_state::{BackFillState, CustodyBackFillState, SyncState}; pub use globals::NetworkGlobals; -pub use partial::PartialDataColumnSidecarMessage; +pub use partial::HeaderSentSet; +pub use partial::OutgoingPartialColumn; pub use pubsub::{PubsubMessage, SnappyTransform, decode_partial}; pub use subnet::{Subnet, SubnetDiscovery}; pub use topics::{ diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index c307ce3bf7e..afdb608b151 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,59 +1,106 @@ use crate::PeerId; use libp2p::gossipsub::partial_messages::{Metadata, Partial, PartialAction, PartialError}; +use parking_lot::Mutex; use ssz::{Decode, Encode}; +use ssz_types::VariableList; +use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; use types::data::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; +use types::partial_data_column_sidecar::PartialDataColumnSidecar; -#[derive(Debug, Clone, PartialEq)] -pub struct PartialDataColumnSidecarMessage { +pub type HeaderSentSet = Arc>>; + +pub struct NoHeaderInColumnError; + +#[derive(Debug, Clone)] +pub struct OutgoingPartialColumn { pub partial_column: Arc>, + pub header_message: Vec, + pub header_sent_set: HeaderSentSet, } -impl PartialDataColumnSidecarMessage { - pub fn new(partial_column: Arc>) -> Self { - PartialDataColumnSidecarMessage { partial_column } +impl OutgoingPartialColumn { + pub fn new( + partial_column: Arc>, + header_sent_set: HeaderSentSet, + ) -> Result { + let Some(header) = partial_column.sidecar.header.first().cloned() else { + return Err(NoHeaderInColumnError); + }; + + let header_message = PartialDataColumnSidecar { + cells_present_bitmap: CellBitmap::::with_capacity( + partial_column.sidecar.cells_present_bitmap.len(), + ) + .expect("Taking length from bitmap with same bound"), + column: VariableList::empty(), + kzg_proofs: VariableList::empty(), + header: VariableList::repeat_full(header), + } + .as_ssz_bytes(); + + Ok(OutgoingPartialColumn { + partial_column, + header_message, + header_sent_set, + }) } } #[derive(Debug, Clone, PartialEq, Eq)] -struct CellBitmapMetadata { - bitmap: CellBitmap, - encoded: Vec, +enum CellBitmapMetadata { + Unknown, + Known { + bitmap: CellBitmap, + encoded: Vec, + }, } impl Metadata for CellBitmapMetadata { fn as_slice(&self) -> &[u8] { - &self.encoded + match self { + CellBitmapMetadata::Unknown => &[], + CellBitmapMetadata::Known { encoded, .. } => encoded, + } } fn update(&mut self, data: &[u8]) -> Result { - let data = + let peer_bitmap = CellBitmap::::from_ssz_bytes(data).map_err(|_| PartialError::InvalidFormat)?; - if data.len() != self.bitmap.len() { + + let CellBitmapMetadata::Known { bitmap, encoded } = self else { + *self = CellBitmapMetadata::Known { + bitmap: peer_bitmap, + encoded: data.to_vec(), + }; + return Ok(true); + }; + + if peer_bitmap.len() != bitmap.len() { return Err(PartialError::OutOfRange); } - let new_bitmap = self.bitmap.union(&data); - if self.bitmap == new_bitmap { + let new_bitmap = bitmap.union(&peer_bitmap); + if *bitmap == new_bitmap { return Ok(false); } - self.bitmap = new_bitmap; - self.encoded = self.bitmap.as_ssz_bytes(); + *bitmap = new_bitmap; + *encoded = bitmap.as_ssz_bytes(); Ok(true) } } impl From> for CellBitmapMetadata { fn from(value: CellBitmap) -> Self { - Self { + Self::Known { encoded: value.as_ssz_bytes(), bitmap: value, } } } -impl Partial for PartialDataColumnSidecarMessage { +impl Partial for OutgoingPartialColumn { fn group_id(&self) -> Vec { self.partial_column.block_root.as_slice().to_vec() } @@ -67,11 +114,22 @@ impl Partial for PartialDataColumnSidecarMessage { fn partial_action_from_metadata( &self, - _peer_id: PeerId, + peer_id: PeerId, metadata: Option<&[u8]>, ) -> Result { match metadata { - None => Ok(PartialAction { + None => { + // send the header-only messsage to the peer if we have not yet + let send = self.header_sent_set.lock().insert(peer_id).then(|| { + ( + self.header_message.clone(), + Box::new(CellBitmapMetadata::::Unknown) as Box, + ) + }); + + Ok(PartialAction { need: false, send }) + } + Some([]) => Ok(PartialAction { need: false, send: None, }), diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index ece24f52476..d7ba32a0d7b 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,7 +1,7 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::TopicHash; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use gossipsub::TopicHash; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; From 74a23b5acf2ceefb193979c7871498a53d18ee2b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 16 Jan 2026 15:53:48 +0100 Subject: [PATCH 055/146] make early partial handling slightly less awkward --- .../lighthouse_network/src/service/mod.rs | 14 ++----- .../lighthouse_network/src/types/pubsub.rs | 37 +++++++++---------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 1ecde1771de..0a2d2feaf05 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1383,18 +1383,10 @@ impl Network { }) .ok()?; - // TODO(dknopik): this currently hardcodes the data column metadata format + // TODO(dknopik): Remove as soon as partial messages stable? if let Some(metadata) = metadata { if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { - let mut metadata_string = String::with_capacity(metadata.len()); - for bit in metadata.iter() { - if bit { - metadata_string.push('1'); - } else { - metadata_string.push('0'); - } - } - debug!(metadata = metadata_string, %topic, %peer_id, "Got metadata") + debug!(%metadata, %topic, %peer_id, "Got metadata") } else { warn!(?metadata, %topic, %peer_id, "Got weird metadata"); } @@ -1403,7 +1395,7 @@ impl Network { } if let Some(message) = message { - match decode_partial::(&topic_hash, &group_id, &message) { + match decode_partial::(&topic, &group_id, &message) { Err(error) => { debug!( topic = ?topic_hash, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index d7ba32a0d7b..aca425ceba4 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -420,29 +420,28 @@ impl PubsubMessage { } /// Decodes incoming partial data column sidecar from gossipsub partial protocol. +/// Note: Currenly, data columns are the only supported partial messages. In future this could +/// return an enum. pub fn decode_partial( - topic: &TopicHash, + topic: &GossipTopic, group: &[u8], data: &[u8], ) -> Result>, String> { - match GossipTopic::decode(topic.as_str()) { - Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)), - Ok(gossip_topic) => match gossip_topic.kind() { - GossipKind::DataColumnSidecar(id) => { - let block_root = Hash256::from_ssz_bytes(group) - .map_err(|e| format!("Error decoding group: {:?}", e))?; - let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) - .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; - let data_column = DanglingPartialDataColumn { - block_root, - // Partial messages are spec'd under the assumption that there is one column per subnet. - index: **id, - sidecar, - }; - Ok(Arc::new(data_column)) - } - other => Err(format!("Partial message unsupported for topic: {other}")), - }, + match topic.kind() { + GossipKind::DataColumnSidecar(id) => { + let block_root = Hash256::from_ssz_bytes(group) + .map_err(|e| format!("Error decoding group: {:?}", e))?; + let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) + .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; + let data_column = DanglingPartialDataColumn { + block_root, + // Partial messages are spec'd under the assumption that there is one column per subnet. + index: **id, + sidecar, + }; + Ok(Arc::new(data_column)) + } + other => Err(format!("Partial message unsupported for topic: {other}")), } } From 074b1373f8ea6186ba25f86c003c8dfe4909ddfd Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 20 Jan 2026 10:23:23 +0100 Subject: [PATCH 056/146] Arc the cells in a partial data column sidecar --- beacon_node/beacon_chain/src/kzg_utils.rs | 4 +- .../types/src/data/data_column_sidecar.rs | 14 ++++++- .../src/data/partial_data_column_sidecar.rs | 41 +++---------------- 3 files changed, 20 insertions(+), 39 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index f376f599902..3dceca4bbe2 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -472,7 +472,7 @@ pub(crate) fn build_partial_data_columns( .get_mut(col) .ok_or(format!("Missing data column proofs at index {col}"))?; - column.push(cell.clone()); + column.push(Arc::new(cell)); column_proofs.push(*proof); } } @@ -487,7 +487,7 @@ pub(crate) fn build_partial_data_columns( index: index as u64, sidecar: types::partial_data_column_sidecar::PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), - column: DataColumn::::try_from(col) + column: VariableList::try_from(col) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index b57b18b0fa0..8b7d21b15d8 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -146,12 +146,22 @@ impl DataColumnSidecar { .expect("The correct size is initialized right above"); } + let block_root = self.block_root(); + + let column = self + .column + .into_iter() + .map(Arc::new) + .collect::>() + .try_into() + .expect("our column has the same bound"); + DanglingPartialDataColumn { - block_root: self.block_root(), + block_root, index: self.index, sidecar: PartialDataColumnSidecar { cells_present_bitmap: bitmap, - column: self.column, + column, kzg_proofs: self.kzg_proofs, header: VariableList::repeat_full(PartialDataColumnHeader { kzg_commitments: self.kzg_commitments.clone(), diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 45c65b915cf..a1f7bb770eb 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,13 +1,11 @@ use crate::test_utils::TestRandom; -use crate::{ - Cell, ColumnIndex, DataColumn, DataColumnSidecar, EthSpec, Hash256, KzgCommitments, - SignedBeaconBlockHeader, -}; +use crate::{Cell, ColumnIndex, EthSpec, Hash256, KzgCommitments, SignedBeaconBlockHeader}; use educe::Educe; use kzg::KzgProof; use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; +use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use typenum::U1; @@ -23,7 +21,7 @@ pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnSidecar { pub cells_present_bitmap: CellBitmap, - pub column: DataColumn, + pub column: VariableList>, ::MaxBlobCommitmentsPerBlock>, pub kzg_proofs: VariableList, pub header: VariableList, U1>, } @@ -33,7 +31,7 @@ impl PartialDataColumnSidecar { // min size is one cell Self { cells_present_bitmap: BitList::with_capacity(1).unwrap(), - column: VariableList::new(vec![Cell::::default()]).unwrap(), + column: VariableList::new(vec![Arc::new(Cell::::default())]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), header: VariableList::new(vec![]).unwrap(), } @@ -45,7 +43,7 @@ impl PartialDataColumnSidecar { // min size is one cell Self { cells_present_bitmap: BitList::with_capacity(block_blobs).unwrap(), - column: VariableList::new(vec![Cell::::default(); present_blobs]).unwrap(), + column: VariableList::new(vec![Default::default(); present_blobs]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty(); present_blobs]).unwrap(), header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push } @@ -56,7 +54,7 @@ impl PartialDataColumnSidecar { pub fn max_size(max_blobs_per_block: usize) -> usize { Self { cells_present_bitmap: BitList::with_capacity(max_blobs_per_block).unwrap(), - column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), + column: VariableList::new(vec![Default::default(); max_blobs_per_block]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push } @@ -180,33 +178,6 @@ impl PartialDataColumnSidecar { } } -// TODO(dknopik): More specific error cases - e.g. internal inconsistency -pub struct MissingCellError; - -impl From> for PartialDataColumnSidecar { - fn from(value: DataColumnSidecar) -> Self { - // Create a bitmap with all cells marked as present - let mut cells_present_bitmap = BitList::with_capacity(value.column.len()) - .expect("Bitmap and cell list are both bounded by `MaxBlobCommitmentsPerBlock`"); - for idx in 0..value.column.len() { - cells_present_bitmap.set(idx, true).expect( - "Bitmap was created with column length, so we should be able to push a value", - ); - } - - Self { - cells_present_bitmap, - column: value.column, - kzg_proofs: value.kzg_proofs, - header: VariableList::repeat_full(PartialDataColumnHeader { - kzg_commitments: value.kzg_commitments, - signed_block_header: value.signed_block_header, - kzg_commitments_inclusion_proof: value.kzg_commitments_inclusion_proof, - }), - } - } -} - #[cfg_attr( feature = "arbitrary", derive(arbitrary::Arbitrary), From 76b4b5c6e62c0eef608a1276deeae65c070c5946 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 20 Jan 2026 10:23:23 +0100 Subject: [PATCH 057/146] Rename "DanglingPartialDataColumn" --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +-- .../src/data_availability_checker.rs | 10 +++--- .../src/data_column_verification.rs | 36 +++++++++---------- .../beacon_chain/src/fetch_blobs/mod.rs | 4 +-- beacon_node/beacon_chain/src/kzg_utils.rs | 14 ++++---- .../src/partial_data_column_assembler.rs | 18 +++++----- .../lighthouse_network/src/service/mod.rs | 6 ++-- .../lighthouse_network/src/types/partial.rs | 6 ++-- .../lighthouse_network/src/types/pubsub.rs | 8 ++--- .../gossip_methods.rs | 6 ++-- .../src/network_beacon_processor/mod.rs | 4 +-- beacon_node/network/src/router.rs | 4 +-- beacon_node/network/src/service.rs | 4 +-- .../types/src/data/data_column_sidecar.rs | 6 ++-- .../src/data/partial_data_column_sidecar.rs | 7 ++-- 15 files changed, 66 insertions(+), 71 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f52db3fcb6c..f3f070b4b1d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -136,7 +136,7 @@ use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace use tree_hash::TreeHash; use types::data::{ColumnIndex, FixedBlobSidecarList}; use types::execution::BlockProductionVersion; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -2200,7 +2200,7 @@ impl BeaconChain { #[instrument(skip_all, level = "trace")] pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, - data_column_sidecar: Arc>, + data_column_sidecar: Arc>, ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9b831b5fc15..ccea73d15d6 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -42,7 +42,7 @@ use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; use types::DataColumnSidecar; use types::new_non_zero_usize; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -229,7 +229,7 @@ impl DataAvailabilityChecker { pub fn determine_missing_cells_partial( &self, block_root: &Hash256, - data_column: &DanglingPartialDataColumn, + data_column: &PartialDataColumn, ) -> Option> { let column_index = data_column.index; @@ -969,7 +969,7 @@ impl MaybeAvailableBlock { /// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. fn compare_full_to_partial( incoming_cell_count: usize, - cached_partial: &DanglingPartialDataColumn, + cached_partial: &PartialDataColumn, ) -> Option> { let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; @@ -998,8 +998,8 @@ fn compare_full_to_partial( /// Compare an incoming partial column against a cached partial. /// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. fn compare_partial_to_partial( - incoming: &DanglingPartialDataColumn, - cached_partial: &DanglingPartialDataColumn, + incoming: &PartialDataColumn, + cached_partial: &PartialDataColumn, ) -> Option> { let incoming_bitmap = &incoming.sidecar.cells_present_bitmap; let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 7f4d324c50a..4ba7df26402 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -18,7 +18,7 @@ use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; use types::data::ColumnIndex; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, @@ -302,14 +302,14 @@ pub struct GossipVerifiedPartialDataColumn GossipVerifiedPartialDataColumn { pub fn new( - column_sidecar: Arc>, + column_sidecar: Arc>, chain: &BeaconChain, ) -> Result { validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) } /// Create a `GossipVerifiedPartialDataColumn` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { + pub fn __new_for_testing(column_sidecar: Arc>) -> Self { Self { block_root: column_sidecar.block_root, data_column: KzgVerifiedPartialDataColumn::__new_for_testing(column_sidecar), @@ -317,12 +317,12 @@ impl GossipVerifiedPartialDataColum } } - pub fn as_data_column(&self) -> &DanglingPartialDataColumn { + pub fn as_data_column(&self) -> &PartialDataColumn { self.data_column.as_data_column() } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc> { self.data_column.clone_data_column() } @@ -408,36 +408,36 @@ impl KzgVerifiedDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedPartialDataColumn { - data: Arc>, + data: Arc>, } impl KzgVerifiedPartialDataColumn { pub fn new( - data_column: Arc>, + data_column: Arc>, kzg: &Kzg, ) -> Result, KzgError)> { verify_kzg_for_partial_data_column(data_column, kzg) } /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { + pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { Self { data: data_column } } /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc>) -> Self { + pub fn from_execution_verified(data_column: Arc>) -> Self { Self { data: data_column } } - pub fn to_data_column(self) -> Arc> { + pub fn to_data_column(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &DanglingPartialDataColumn { + pub fn as_data_column(&self) -> &PartialDataColumn { &self.data } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc> { self.data.clone() } @@ -558,7 +558,7 @@ impl KzgVerifiedCustodyDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedCustodyPartialDataColumn { - data: Arc>, + data: Arc>, } impl KzgVerifiedCustodyPartialDataColumn { @@ -570,15 +570,15 @@ impl KzgVerifiedCustodyPartialDataColumn { } } - pub fn into_inner(self) -> Arc> { + pub fn into_inner(self) -> Arc> { self.data } - pub fn as_data_column(&self) -> &DanglingPartialDataColumn { + pub fn as_data_column(&self) -> &PartialDataColumn { &self.data } - pub fn clone_arc(&self) -> Arc> { + pub fn clone_arc(&self) -> Arc> { self.data.clone() } @@ -605,7 +605,7 @@ pub fn verify_kzg_for_data_column( /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( - data_column: Arc>, + data_column: Arc>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); @@ -692,7 +692,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip< T: BeaconChainTypes, O: ObservationStrategy, >( - data_column: Arc>, + data_column: Arc>, chain: &BeaconChain, ) -> Result, GossipDataColumnError> { let block_root = data_column.block_root; diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 651a62a5ffa..83de13ecee3 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -36,7 +36,7 @@ use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_h use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; use types::data::{BlobSidecarError, DataColumnSidecarError}; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; use types::{ BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, @@ -51,7 +51,7 @@ pub enum EngineGetBlobsOutput { /// Complete columns ready to be imported into the `DataAvailabilityChecker`. CompleteColumns(Vec>>), /// Incomplete partial columns that need assembly via gossip - PartialColumns(Vec>>), + PartialColumns(Vec>>), } #[derive(Debug)] diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 3dceca4bbe2..06ffd0aaac7 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -8,9 +8,7 @@ use std::sync::Arc; use tracing::instrument; use types::data::{Cell, DataColumn, DataColumnSidecarError}; use types::kzg_ext::KzgCommitments; -use types::partial_data_column_sidecar::{ - CellBitmap, DanglingPartialDataColumn, PartialDataColumnHeader, -}; +use types::partial_data_column_sidecar::{CellBitmap, PartialDataColumn, PartialDataColumnHeader}; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, @@ -102,7 +100,7 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. pub fn validate_partial_data_columns<'a, E: EthSpec>( kzg: &Kzg, - data_column_iter: impl Iterator>>, + data_column_iter: impl Iterator>>, ) -> Result<(), (Option, KzgError)> { let mut cells = Vec::new(); let mut proofs = Vec::new(); @@ -289,7 +287,7 @@ pub fn blobs_to_partial_data_columns( block: &SignedBeaconBlock, kzg: &Kzg, spec: &ChainSpec, -) -> Result>>, DataColumnSidecarError> { +) -> Result>>, DataColumnSidecarError> { if blobs_and_proofs.is_empty() { return Ok(vec![]); } @@ -425,7 +423,7 @@ pub(crate) fn build_partial_data_columns( signed_block_header: SignedBeaconBlockHeader, blob_cells_and_proofs_vec: Vec>, spec: &ChainSpec, -) -> Result>>, String> { +) -> Result>>, String> { let number_of_columns = E::number_of_columns(); let max_blobs_per_block = spec .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) @@ -477,12 +475,12 @@ pub(crate) fn build_partial_data_columns( } } - let sidecars: Result>>, String> = columns + let sidecars: Result>>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() .map(|(index, (col, proofs))| { - let column = DanglingPartialDataColumn { + let column = PartialDataColumn { block_root: signed_block_header.message.canonical_root(), index: index as u64, sidecar: types::partial_data_column_sidecar::PartialDataColumnSidecar { diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 362c8043eec..6cf77d943ed 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -4,7 +4,7 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256, KzgCommitments, Slot}; /// Assembles partial data columns into complete columns @@ -20,20 +20,20 @@ struct PartialAssembly { slot: Slot, kzg_commitments: KzgCommitments, /// Map of column_index -> partial column being assembled - columns: HashMap>>, + columns: HashMap>>, } /// Result of merging a partial column pub enum PartialMergeResult { /// Merge was successful but column is still incomplete Incomplete { - updated_partial: Arc>, + updated_partial: Arc>, }, /// Merge completed the column Completed { full_column: Arc>, /// The updated partial for publishing (same as full but as partial type) - updated_partial: Arc>, + updated_partial: Arc>, }, } @@ -96,7 +96,7 @@ impl PartialDataColumnAssembler { pub fn merge_partial( &self, block_root: Hash256, - partial: Arc>, + partial: Arc>, ) -> Option> { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_mut(&block_root)?; @@ -106,7 +106,7 @@ impl PartialDataColumnAssembler { let merged = if let Some(existing) = assembly.columns.get(&column_index) { // Merge with existing partial let merged_sidecar = existing.sidecar.merge(&partial.sidecar)?; - Arc::new(DanglingPartialDataColumn { + Arc::new(PartialDataColumn { block_root: existing.block_root, index: existing.index, sidecar: merged_sidecar, @@ -143,7 +143,7 @@ impl PartialDataColumnAssembler { &self, block_root: &Hash256, column_index: ColumnIndex, - ) -> Option>> { + ) -> Option>> { self.assemblies .read() .peek(block_root)? @@ -188,7 +188,7 @@ impl PartialDataColumnAssembler { /// Convert a complete partial to a full DataColumnSidecar fn partial_to_full( - partial: &DanglingPartialDataColumn, + partial: &PartialDataColumn, kzg_commitments: &KzgCommitments, ) -> Option>> { // Use the as_full method - requires a block for signed header @@ -223,5 +223,5 @@ pub struct InitResult { /// Columns that were already complete pub complete_columns: Vec>>, /// Columns that need further assembly via gossip - pub incomplete_partials: Vec>>, + pub incomplete_partials: Vec>>, } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 0a2d2feaf05..49118ee55e4 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -43,7 +43,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; -use types::data::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; +use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; use types::{ChainSpec, ForkName}; use types::{ DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, Slot, SubnetId, @@ -111,7 +111,7 @@ pub enum NetworkEvent { /// The peer from which we received this message. source: PeerId, /// The partial column data. - column: Arc>, + column: Arc>, }, /// Inform the network to send a Status to this peer. StatusPeer(PeerId), @@ -913,7 +913,7 @@ impl Network { } /// Publishes partial data column sidecars to the gossipsub network. - pub fn publish_partial(&mut self, columns: Vec>>) { + pub fn publish_partial(&mut self, columns: Vec>>) { for column in columns { let subnet = DataColumnSubnetId::from_column_index(column.index, &self.fork_context.spec); diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index afdb608b151..afc62125519 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; -use types::data::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; +use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; use types::partial_data_column_sidecar::PartialDataColumnSidecar; pub type HeaderSentSet = Arc>>; @@ -16,14 +16,14 @@ pub struct NoHeaderInColumnError; #[derive(Debug, Clone)] pub struct OutgoingPartialColumn { - pub partial_column: Arc>, + pub partial_column: Arc>, pub header_message: Vec, pub header_sent_set: HeaderSentSet, } impl OutgoingPartialColumn { pub fn new( - partial_column: Arc>, + partial_column: Arc>, header_sent_set: HeaderSentSet, ) -> Result { let Some(header) = partial_column.sidecar.header.first().cloned() else { diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index aca425ceba4..9b6d52d29bc 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -6,9 +6,7 @@ use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use types::data::partial_data_column_sidecar::{ - DanglingPartialDataColumn, PartialDataColumnSidecar, -}; +use types::data::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, @@ -426,14 +424,14 @@ pub fn decode_partial( topic: &GossipTopic, group: &[u8], data: &[u8], -) -> Result>, String> { +) -> Result>, String> { match topic.kind() { GossipKind::DataColumnSidecar(id) => { let block_root = Hash256::from_ssz_bytes(group) .map_err(|e| format!("Error decoding group: {:?}", e))?; let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; - let data_column = DanglingPartialDataColumn { + let data_column = PartialDataColumn { block_root, // Partial messages are spec'd under the assumption that there is one column per subnet. index: **id, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 406772fb979..d4394f890ad 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -57,7 +57,7 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -784,7 +784,7 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_dangling_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: Arc>, seen_duration: Duration, ) { // TODO(dknopik): add check and cache for header @@ -871,7 +871,7 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_verifiable_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: Arc>, slot: Slot, seen_duration: Duration, ) { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 292d4d5c7dd..dad54993c70 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -28,7 +28,7 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; use tracing::{debug, error, instrument, trace, warn}; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::partial_data_column_sidecar::PartialDataColumn; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -254,7 +254,7 @@ impl NetworkBeaconProcessor { pub fn send_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: Arc>, seen_timestamp: Duration, ) -> Result<(), Error> { let processor = self.clone(); diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index b58ef38826d..a05cb65adcb 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -24,7 +24,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, trace, warn}; -use types::data::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::data::partial_data_column_sidecar::PartialDataColumn; use types::{BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, SignedBeaconBlock}; /// Handles messages from the network and routes them to the appropriate service to be handled. @@ -71,7 +71,7 @@ pub enum RouterMessage { /// by the beacon chain after successful verification. PubsubMessage(MessageId, PeerId, PubsubMessage, bool), /// A partial data column sidecar has been received via gossipsub partial protocol. - PartialDataColumnSidecar(PeerId, Arc>), + PartialDataColumnSidecar(PeerId, Arc>), /// The peer manager has requested we re-status a peer. StatusPeer(PeerId), /// The peer has an updated custody group count from METADATA. diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index b5016dd5661..29715cfa340 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -38,7 +38,7 @@ use tokio::sync::mpsc; use tokio::time::Sleep; use tracing::{debug, error, info, trace, warn}; use typenum::Unsigned; -use types::data::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::data::partial_data_column_sidecar::PartialDataColumn; use types::{ EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, ValidatorSubscription, @@ -86,7 +86,7 @@ pub enum NetworkMessage { Publish { messages: Vec> }, /// Publish partial data column sidecars via the partial gossipsub protocol. PublishPartial { - columns: Vec>>, + columns: Vec>>, }, /// Validates a received gossipsub message. This will propagate the message on the network. ValidationResult { diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 8b7d21b15d8..b67a0ed4552 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -1,5 +1,5 @@ use crate::data::partial_data_column_sidecar::CellBitmap; -use crate::data::partial_data_column_sidecar::DanglingPartialDataColumn; +use crate::data::partial_data_column_sidecar::PartialDataColumn; use crate::data::partial_data_column_sidecar::PartialDataColumnSidecar; use std::sync::Arc; @@ -136,7 +136,7 @@ impl DataColumnSidecar { } /// Convert this full data column into a verifiable partial data column. - pub fn into_partial(self) -> DanglingPartialDataColumn { + pub fn into_partial(self) -> PartialDataColumn { let cell_count = self.column.len(); let mut bitmap = CellBitmap::::with_capacity(cell_count).expect("our column has the same bound"); @@ -156,7 +156,7 @@ impl DataColumnSidecar { .try_into() .expect("our column has the same bound"); - DanglingPartialDataColumn { + PartialDataColumn { block_root, index: self.index, sidecar: PartialDataColumnSidecar { diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index a1f7bb770eb..4a428fef757 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -191,20 +191,19 @@ pub struct PartialDataColumnHeader { pub kzg_commitments_inclusion_proof: FixedVector, } -// TODO(dknopik): Name? #[derive(Debug, Clone, PartialEq)] -pub struct DanglingPartialDataColumn { +pub struct PartialDataColumn { pub block_root: Hash256, pub index: ColumnIndex, pub sidecar: PartialDataColumnSidecar, } -impl DanglingPartialDataColumn { +impl PartialDataColumn { pub fn clone_filter(&self, filter: F) -> Option where F: Fn(usize) -> bool, { - Some(DanglingPartialDataColumn { + Some(PartialDataColumn { sidecar: self.sidecar.clone_filter(filter)?, block_root: self.block_root, index: self.index, From 07ff6db097b83983645e0e60443cf93c356ac918 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 20 Jan 2026 12:30:59 +0100 Subject: [PATCH 058/146] Remove GossipVerifiedPartialDataColumn --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 +- .../src/data_column_verification.rs | 97 +++++-------------- .../gossip_methods.rs | 4 +- 3 files changed, 31 insertions(+), 79 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f3f070b4b1d..d760c36e6a9 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -26,7 +26,8 @@ use crate::data_availability_checker::{ DataAvailabilityChecker, DataColumnReconstructionResult, }; use crate::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumn, + GossipDataColumnError, GossipVerifiedDataColumn, KzgVerifiedPartialDataColumn, + validate_partial_data_column_sidecar_for_gossip, }; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; @@ -2201,11 +2202,11 @@ impl BeaconChain { pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, data_column_sidecar: Arc>, - ) -> Result, GossipDataColumnError> { + ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); - GossipVerifiedPartialDataColumn::new(data_column_sidecar, self).inspect(|_| { + validate_partial_data_column_sidecar_for_gossip(data_column_sidecar, self).inspect(|_| { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); }) } @@ -3096,7 +3097,7 @@ impl BeaconChain { #[instrument(skip_all, level = "debug")] pub async fn process_gossip_partial_data_column( self: &Arc, - verified_partial: GossipVerifiedPartialDataColumn, + verified_partial: KzgVerifiedPartialDataColumn, ) -> Result, BlockError> { let block_root = verified_partial.block_root(); diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 4ba7df26402..34358544234 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -291,66 +291,6 @@ impl GossipVerifiedDataColumn /// Alias for backwards compatibility pub type GossipVerifiedFullDataColumn = GossipVerifiedDataColumn; -/// A wrapper around a `VerifiablePartialDataColumn` that indicates it has been approved for -/// re-gossiping on the p2p network. -#[derive(Debug, Clone)] -pub struct GossipVerifiedPartialDataColumn { - block_root: Hash256, - data_column: KzgVerifiedPartialDataColumn, - _phantom: PhantomData, -} - -impl GossipVerifiedPartialDataColumn { - pub fn new( - column_sidecar: Arc>, - chain: &BeaconChain, - ) -> Result { - validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) - } - - /// Create a `GossipVerifiedPartialDataColumn` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { - Self { - block_root: column_sidecar.block_root, - data_column: KzgVerifiedPartialDataColumn::__new_for_testing(column_sidecar), - _phantom: Default::default(), - } - } - - pub fn as_data_column(&self) -> &PartialDataColumn { - self.data_column.as_data_column() - } - - /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { - self.data_column.clone_data_column() - } - - pub fn block_root(&self) -> Hash256 { - self.block_root - } - - pub fn slot(&self) -> Slot { - self.data_column - .as_data_column() - .sidecar - .header - .first() - .expect("This was verified, so we have the header") - .signed_block_header - .message - .slot - } - - pub fn index(&self) -> ColumnIndex { - self.data_column.index() - } - - pub fn into_inner(self) -> KzgVerifiedPartialDataColumn { - self.data_column - } -} - /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] @@ -444,6 +384,26 @@ impl KzgVerifiedPartialDataColumn { pub fn index(&self) -> ColumnIndex { self.data.index } + + pub fn block_root(&self) -> Hash256 { + self.data.block_root + } + + /// Returns the slot of this partial data column. + /// + /// # Panics + /// Panics if the header is not present. The header is guaranteed to be present + /// for columns that have passed gossip validation. + pub fn slot(&self) -> Slot { + self.data + .sidecar + .header + .first() + .expect("Header must be present for gossip-validated partial columns") + .signed_block_header + .message + .slot + } } pub type CustodyDataColumnList = @@ -688,13 +648,10 @@ pub fn validate_data_column_sidecar_for_gossip( +pub fn validate_partial_data_column_sidecar_for_gossip( data_column: Arc>, chain: &BeaconChain, -) -> Result, GossipDataColumnError> { +) -> Result, GossipDataColumnError> { let block_root = data_column.block_root; let filtered_column = if let Some(missing_cells) = chain @@ -718,14 +675,8 @@ pub fn validate_partial_data_column_sidecar_for_gossip< // gossip accepted block let kzg = &chain.kzg; - let kzg_verified_data_column = verify_kzg_for_partial_data_column(filtered_column, kzg) - .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; - - Ok(GossipVerifiedPartialDataColumn { - block_root, - data_column: kzg_verified_data_column, - _phantom: PhantomData, - }) + verify_kzg_for_partial_data_column(filtered_column, kzg) + .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e)) } /// Verify if the data column sidecar is valid. diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index d4394f890ad..5298f5aa12b 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -7,7 +7,7 @@ use crate::{ use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumn, + GossipDataColumnError, GossipVerifiedDataColumn, KzgVerifiedPartialDataColumn, }; use beacon_chain::partial_data_column_assembler::PartialMergeResult; use beacon_chain::store::Error; @@ -1382,7 +1382,7 @@ impl NetworkBeaconProcessor { async fn process_gossip_verified_partial_data_column( self: &Arc, _peer_id: PeerId, - verified_partial: GossipVerifiedPartialDataColumn, + verified_partial: KzgVerifiedPartialDataColumn, _seen_duration: Duration, ) { let processing_start_time = Instant::now(); From 262aca8cb925acfa7d503d824acde157d1324c0e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 20 Jan 2026 12:25:39 +0100 Subject: [PATCH 059/146] Arc less --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +-- .../src/data_column_verification.rs | 37 ++++++++----------- beacon_node/beacon_chain/src/kzg_utils.rs | 10 ++--- .../src/partial_data_column_assembler.rs | 5 ++- .../lighthouse_network/src/service/mod.rs | 2 +- .../lighthouse_network/src/types/pubsub.rs | 4 +- .../gossip_methods.rs | 6 +-- .../src/network_beacon_processor/mod.rs | 2 +- beacon_node/network/src/router.rs | 2 +- 9 files changed, 34 insertions(+), 40 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d760c36e6a9..d53c72638fa 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2201,7 +2201,7 @@ impl BeaconChain { #[instrument(skip_all, level = "trace")] pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, - data_column_sidecar: Arc>, + data_column_sidecar: PartialDataColumn, ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = @@ -3113,11 +3113,11 @@ impl BeaconChain { } // Merge the partial into the assembler - let partial_arc = verified_partial.clone_data_column(); + let partial = verified_partial.clone_data_column(); let merge_result = self .data_availability_checker .partial_assembler() - .merge_partial(block_root, partial_arc) + .merge_partial(block_root, partial) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; // If we completed a column, process it through the DA checker diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 34358544234..655db00ff2a 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -348,36 +348,35 @@ impl KzgVerifiedDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedPartialDataColumn { - data: Arc>, + data: PartialDataColumn, } impl KzgVerifiedPartialDataColumn { pub fn new( - data_column: Arc>, + data_column: PartialDataColumn, kzg: &Kzg, ) -> Result, KzgError)> { verify_kzg_for_partial_data_column(data_column, kzg) } /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { + pub(crate) fn __new_for_testing(data_column: PartialDataColumn) -> Self { Self { data: data_column } } /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc>) -> Self { + pub fn from_execution_verified(data_column: PartialDataColumn) -> Self { Self { data: data_column } } - pub fn to_data_column(self) -> Arc> { + pub fn to_data_column(self) -> PartialDataColumn { self.data } pub fn as_data_column(&self) -> &PartialDataColumn { &self.data } - /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> PartialDataColumn { self.data.clone() } @@ -486,7 +485,7 @@ impl KzgVerifiedCustodyDataColumn { pub fn into_partial(self) -> KzgVerifiedCustodyPartialDataColumn { let column = Arc::try_unwrap(self.data).unwrap_or_else(|column| (*column).clone()); KzgVerifiedCustodyPartialDataColumn { - data: Arc::new(column.into_partial()), + data: column.into_partial(), } } @@ -518,7 +517,7 @@ impl KzgVerifiedCustodyDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedCustodyPartialDataColumn { - data: Arc>, + data: PartialDataColumn, } impl KzgVerifiedCustodyPartialDataColumn { @@ -530,7 +529,7 @@ impl KzgVerifiedCustodyPartialDataColumn { } } - pub fn into_inner(self) -> Arc> { + pub fn into_inner(self) -> PartialDataColumn { self.data } @@ -538,10 +537,6 @@ impl KzgVerifiedCustodyPartialDataColumn { &self.data } - pub fn clone_arc(&self) -> Arc> { - self.data.clone() - } - pub fn index(&self) -> ColumnIndex { self.data.index } @@ -565,7 +560,7 @@ pub fn verify_kzg_for_data_column( /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( - data_column: Arc>, + data_column: PartialDataColumn, kzg: &Kzg, ) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); @@ -649,24 +644,22 @@ pub fn validate_data_column_sidecar_for_gossip( - data_column: Arc>, + data_column: PartialDataColumn, chain: &BeaconChain, ) -> Result, GossipDataColumnError> { let block_root = data_column.block_root; let filtered_column = if let Some(missing_cells) = chain .data_availability_checker - .determine_missing_cells_partial(&block_root, data_column.as_ref()) + .determine_missing_cells_partial(&block_root, &data_column) { if missing_cells.is_empty() { return Err(GossipDataColumnError::PriorKnownUnpublished); } - Arc::new( - data_column - .clone_filter(|idx| missing_cells.contains(&idx)) - .ok_or_else(|| GossipDataColumnError::PriorKnownUnpublished)?, - ) + data_column + .clone_filter(|idx| missing_cells.contains(&idx)) + .ok_or_else(|| GossipDataColumnError::PriorKnownUnpublished)? } else { data_column }; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 06ffd0aaac7..df093d61429 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -100,7 +100,7 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. pub fn validate_partial_data_columns<'a, E: EthSpec>( kzg: &Kzg, - data_column_iter: impl Iterator>>, + data_column_iter: impl Iterator>, ) -> Result<(), (Option, KzgError)> { let mut cells = Vec::new(); let mut proofs = Vec::new(); @@ -287,7 +287,7 @@ pub fn blobs_to_partial_data_columns( block: &SignedBeaconBlock, kzg: &Kzg, spec: &ChainSpec, -) -> Result>>, DataColumnSidecarError> { +) -> Result>, DataColumnSidecarError> { if blobs_and_proofs.is_empty() { return Ok(vec![]); } @@ -423,7 +423,7 @@ pub(crate) fn build_partial_data_columns( signed_block_header: SignedBeaconBlockHeader, blob_cells_and_proofs_vec: Vec>, spec: &ChainSpec, -) -> Result>>, String> { +) -> Result>, String> { let number_of_columns = E::number_of_columns(); let max_blobs_per_block = spec .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) @@ -475,7 +475,7 @@ pub(crate) fn build_partial_data_columns( } } - let sidecars: Result>>, String> = columns + let sidecars: Result>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() @@ -496,7 +496,7 @@ pub(crate) fn build_partial_data_columns( }), }, }; - Ok(Arc::new(column)) + Ok(column) }) .collect(); diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 6cf77d943ed..2dc9f9292c8 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -79,6 +79,7 @@ impl PartialDataColumnAssembler { } // Store incomplete partial + let partial = Arc::new(partial); result.incomplete_partials.push(partial.clone()); assembly.columns.insert(column_index, partial); } @@ -96,7 +97,7 @@ impl PartialDataColumnAssembler { pub fn merge_partial( &self, block_root: Hash256, - partial: Arc>, + partial: PartialDataColumn, ) -> Option> { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_mut(&block_root)?; @@ -113,7 +114,7 @@ impl PartialDataColumnAssembler { }) } else { // First time seeing this column index for this block - partial.clone() + Arc::new(partial.clone()) }; // Check if merged column is now complete diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 49118ee55e4..57ba0571ea3 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -111,7 +111,7 @@ pub enum NetworkEvent { /// The peer from which we received this message. source: PeerId, /// The partial column data. - column: Arc>, + column: PartialDataColumn, }, /// Inform the network to send a Status to this peer. StatusPeer(PeerId), diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 9b6d52d29bc..472cefe5a9e 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -424,7 +424,7 @@ pub fn decode_partial( topic: &GossipTopic, group: &[u8], data: &[u8], -) -> Result>, String> { +) -> Result, String> { match topic.kind() { GossipKind::DataColumnSidecar(id) => { let block_root = Hash256::from_ssz_bytes(group) @@ -437,7 +437,7 @@ pub fn decode_partial( index: **id, sidecar, }; - Ok(Arc::new(data_column)) + Ok(data_column) } other => Err(format!("Partial message unsupported for topic: {other}")), } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 5298f5aa12b..d17addea513 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -784,7 +784,7 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_dangling_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: PartialDataColumn, seen_duration: Duration, ) { // TODO(dknopik): add check and cache for header @@ -871,7 +871,7 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_verifiable_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: PartialDataColumn, slot: Slot, seen_duration: Duration, ) { @@ -885,7 +885,7 @@ impl NetworkBeaconProcessor { ); match self .chain - .verify_partial_data_column_sidecar_for_gossip(column_sidecar.clone()) + .verify_partial_data_column_sidecar_for_gossip(column_sidecar) { Ok(gossip_verified_data_column) => { metrics::inc_counter( diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index dad54993c70..052a716c871 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -254,7 +254,7 @@ impl NetworkBeaconProcessor { pub fn send_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + column_sidecar: PartialDataColumn, seen_timestamp: Duration, ) -> Result<(), Error> { let processor = self.clone(); diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index a05cb65adcb..4012a58246b 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -71,7 +71,7 @@ pub enum RouterMessage { /// by the beacon chain after successful verification. PubsubMessage(MessageId, PeerId, PubsubMessage, bool), /// A partial data column sidecar has been received via gossipsub partial protocol. - PartialDataColumnSidecar(PeerId, Arc>), + PartialDataColumnSidecar(PeerId, PartialDataColumn), /// The peer manager has requested we re-status a peer. StatusPeer(PeerId), /// The peer has an updated custody group count from METADATA. From 1f59e205a411a695e38d70c0b18bbccd11d7e5eb Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 19 Jan 2026 13:57:39 +0100 Subject: [PATCH 060/146] Add validation and caching of partial header --- beacon_node/beacon_chain/src/beacon_chain.rs | 46 +- .../src/data_availability_checker.rs | 18 +- .../src/data_availability_checker/error.rs | 2 + .../src/data_column_verification.rs | 127 +++++- .../beacon_chain/src/fetch_blobs/mod.rs | 41 +- beacon_node/beacon_chain/src/metrics.rs | 29 ++ .../src/partial_data_column_assembler.rs | 211 ++++----- .../gossip_methods.rs | 410 ++++++++---------- .../src/network_beacon_processor/mod.rs | 6 +- .../src/data/partial_data_column_sidecar.rs | 62 ++- 10 files changed, 548 insertions(+), 404 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d53c72638fa..a5b57bc0a1b 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -26,8 +26,8 @@ use crate::data_availability_checker::{ DataAvailabilityChecker, DataColumnReconstructionResult, }; use crate::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, KzgVerifiedPartialDataColumn, - validate_partial_data_column_sidecar_for_gossip, + GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumnHeader, + KzgVerifiedPartialDataColumn, validate_partial_data_column_sidecar_for_gossip, }; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; @@ -137,7 +137,7 @@ use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace use tree_hash::TreeHash; use types::data::{ColumnIndex, FixedBlobSidecarList}; use types::execution::BlockProductionVersion; -use types::partial_data_column_sidecar::PartialDataColumn; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -340,6 +340,7 @@ struct PartialBeaconBlock { pub enum BlockProcessStatus { /// Block is not in any pre-import cache. Block may be in the data-base or in the fork-choice. Unknown, + /// Only the header /// Block is currently processing but not yet validated. NotValidated(Arc>, BlockImportSource), /// Block is fully valid, but not yet imported. It's cached in the da_checker while awaiting @@ -2198,6 +2199,35 @@ impl BeaconChain { }) } + pub fn verify_partial_data_column_header_for_gossip( + self: &Arc, + block_root: Hash256, + data_column_header: PartialDataColumnHeader, + ) -> Result, GossipDataColumnError> { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_REQUESTS); + let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + if let Some(cached_header) = self + .data_availability_checker + .partial_assembler() + .get_header(&block_root) + { + return if *cached_header == data_column_header { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_DUPES); + Ok(GossipVerifiedPartialDataColumnHeader::new_from_cached( + data_column_header, + )) + } else { + Err(GossipDataColumnError::PartialHeaderMismatches) + }; + } + + GossipVerifiedPartialDataColumnHeader::new(block_root, data_column_header, self).inspect( + |_| { + metrics::inc_counter(&metrics::DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); + }, + ) + } + #[instrument(skip_all, level = "trace")] pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, @@ -3117,18 +3147,14 @@ impl BeaconChain { let merge_result = self .data_availability_checker .partial_assembler() - .merge_partial(block_root, partial) + .merge_partials(block_root, vec![partial]) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; // If we completed a column, process it through the DA checker - if let crate::partial_data_column_assembler::PartialMergeResult::Completed { - ref full_column, - .. - } = merge_result - { + if !merge_result.full_columns.is_empty() { let availability = self .data_availability_checker - .put_full_data_columns(block_root, vec![full_column.clone()])?; + .put_full_data_columns(block_root, merge_result.full_columns.clone())?; // If we achieved full availability, process it if let Availability::Available(available_block) = availability { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index ccea73d15d6..4cddcbf4922 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -42,7 +42,7 @@ use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; use types::DataColumnSidecar; use types::new_non_zero_usize; -use types::partial_data_column_sidecar::PartialDataColumn; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -449,6 +449,10 @@ impl DataAvailabilityChecker { &self, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { + let block = executed_block.as_block(); + self.partial_assembler + .init(block.canonical_root(), block) + .map_err(Error::PartialAssemblerError)?; self.availability_cache.put_executed_block(executed_block) } @@ -460,10 +464,22 @@ impl DataAvailabilityChecker { block: Arc>, source: BlockImportSource, ) -> Result<(), Error> { + self.partial_assembler + .init(block_root, block.as_ref()) + .map_err(Error::PartialAssemblerError)?; self.availability_cache .put_pre_execution_block(block_root, block, source) } + /// Inserts a pre-execution block into the cache. + /// This does NOT override an existing executed block. + pub fn put_partial_data_column_header(&self, header: PartialDataColumnHeader) { + let _ = self + .partial_assembler + .init(header.signed_block_header.message.canonical_root(), header) + .map_err(|e| match e {}); + } + /// Removes a pre-execution block from the cache. /// This does NOT remove an existing executed block. pub fn remove_block_on_execution_error(&self, block_root: &Hash256) { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/error.rs b/beacon_node/beacon_chain/src/data_availability_checker/error.rs index c9efb7a4149..a5448ece383 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/error.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/error.rs @@ -21,6 +21,7 @@ pub enum Error { ParentStateMissing(Hash256), BlockReplayError(state_processing::BlockReplayError), RebuildingStateCaches(BeaconStateError), + PartialAssemblerError(BeaconStateError), SlotClockError, } @@ -44,6 +45,7 @@ impl Error { | Error::ParentStateMissing(_) | Error::BlockReplayError(_) | Error::RebuildingStateCaches(_) + | Error::PartialAssemblerError(_) | Error::SlotClockError => ErrorCategory::Internal, Error::InvalidBlobs { .. } | Error::InvalidColumn { .. } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 655db00ff2a..e9fbd4ca9a8 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -17,8 +17,9 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; +use tree_hash::TreeHash; use types::data::ColumnIndex; -use types::partial_data_column_sidecar::PartialDataColumn; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, @@ -173,6 +174,19 @@ pub enum GossipDataColumnError { max_blobs_per_block: usize, commitments_len: usize, }, + /// The partial data column header does not match the valid one we have already cached. + /// + /// ## Peer scoring + /// The column sidecar is invalid and the peer is faulty + PartialHeaderMismatches, + /// The partial data column header block root does not match the group id. + /// + /// ## Peer scoring + /// The column sidecar is invalid and the peer is faulty + PartialHeaderIncorrectRoot { + group_id: Hash256, + header_hash: Hash256, + }, } impl From for GossipDataColumnError { @@ -389,22 +403,91 @@ impl KzgVerifiedPartialDataColumn { } /// Returns the slot of this partial data column. - /// - /// # Panics - /// Panics if the header is not present. The header is guaranteed to be present - /// for columns that have passed gossip validation. pub fn slot(&self) -> Slot { self.data .sidecar .header .first() - .expect("Header must be present for gossip-validated partial columns") + .expect("Header must be present for validated partial columns") .signed_block_header .message .slot } } +/// Wrapper over a `PartialDataColumnHeader` for which we have completed gossip verification. +#[derive(Debug, Educe, Clone)] +#[educe(PartialEq, Eq)] +pub struct GossipVerifiedPartialDataColumnHeader { + header: PartialDataColumnHeader, + cached: bool, +} + +impl GossipVerifiedPartialDataColumnHeader { + pub fn new>( + group_id: Hash256, + header: PartialDataColumnHeader, + chain: &BeaconChain, + ) -> Result { + let column_slot = header.signed_block_header.message.slot; + if header.kzg_commitments.is_empty() { + return Err(GossipDataColumnError::UnexpectedDataColumn); + } + + let header_hash = header.signed_block_header.message.canonical_root(); + if group_id != header_hash { + return Err(GossipDataColumnError::PartialHeaderIncorrectRoot { + group_id, + header_hash, + }); + } + + verify_sidecar_not_from_future_slot(chain, column_slot)?; + verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; + verify_partial_column_header_inclusion_proof(&header)?; + let parent_block = verify_parent_block_and_finalized_descendant( + header.signed_block_header.message.parent_root, + chain, + )?; + verify_slot_higher_than_parent(&parent_block, column_slot)?; + verify_proposer_and_signature(&header.signed_block_header, &parent_block, chain)?; + + chain + .observed_slashable + .write() + .observe_slashable( + column_slot, + header.signed_block_header.message.proposer_index, + header_hash, + ) + .map_err(|e| GossipDataColumnError::BeaconChainError(Box::new(e.into())))?; + + Ok(Self { + header, + cached: false, + }) + } + + pub fn new_from_cached(header: PartialDataColumnHeader) -> Self { + Self { + header, + cached: true, + } + } + + pub fn as_inner(&self) -> &PartialDataColumnHeader { + &self.header + } + + pub fn into_inner(self) -> PartialDataColumnHeader { + self.header + } + + pub fn was_cached(&self) -> bool { + self.cached + } +} + pub type CustodyDataColumnList = VariableList, ::NumberOfColumns>; @@ -614,9 +697,10 @@ pub fn validate_data_column_sidecar_for_gossip( Ok(()) } +fn verify_partial_column_header_inclusion_proof( + header: &PartialDataColumnHeader, +) -> Result<(), GossipDataColumnError> { + let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_INCLUSION_PROOF_VERIFICATION); + if !header.verify_inclusion_proof() { + return Err(GossipDataColumnError::InvalidInclusionProof); + } + + Ok(()) +} + fn verify_slot_higher_than_parent( parent_block: &Block, data_column_slot: Slot, @@ -759,14 +854,13 @@ fn verify_slot_higher_than_parent( } fn verify_parent_block_and_finalized_descendant( - data_column: Arc>, + block_parent_root: Hash256, chain: &BeaconChain, ) -> Result { let fork_choice = chain.canonical_head.fork_choice_read_lock(); // We have already verified that the column is past finalization, so we can // just check fork choice for the block's parent. - let block_parent_root = data_column.block_parent_root(); let Some(parent_block) = fork_choice.get_block(&block_parent_root) else { return Err(GossipDataColumnError::ParentUnknown { parent_root: block_parent_root, @@ -783,16 +877,15 @@ fn verify_parent_block_and_finalized_descendant( } fn verify_proposer_and_signature( - data_column: &DataColumnSidecar, + signed_block_header: &SignedBeaconBlockHeader, parent_block: &ProtoBlock, chain: &BeaconChain, ) -> Result<(), GossipDataColumnError> { - let column_slot = data_column.slot(); + let column_slot = signed_block_header.message.slot; let slots_per_epoch = T::EthSpec::slots_per_epoch(); let column_epoch = column_slot.epoch(slots_per_epoch); - let column_index = data_column.index; - let block_root = data_column.block_root(); - let block_parent_root = data_column.block_parent_root(); + let block_root = signed_block_header.message.tree_hash_root(); + let block_parent_root = signed_block_header.message.parent_root; let proposer_shuffling_root = parent_block.proposer_shuffling_root_for_child_block(column_epoch, &chain.spec); @@ -804,7 +897,6 @@ fn verify_proposer_and_signature( || { debug!( %block_root, - index = %column_index, "Proposer shuffling cache miss for column verification" ); chain @@ -831,7 +923,6 @@ fn verify_proposer_and_signature( let pubkey = pubkey_cache .get(proposer_index) .ok_or_else(|| GossipDataColumnError::UnknownValidator(proposer_index as u64))?; - let signed_block_header = &data_column.signed_block_header; signed_block_header.verify_signature::( pubkey, &fork, @@ -844,7 +935,7 @@ fn verify_proposer_and_signature( return Err(GossipDataColumnError::ProposalSignatureInvalid); } - let column_proposer_index = data_column.block_proposer_index(); + let column_proposer_index = signed_block_header.message.proposer_index; if proposer_index != column_proposer_index as usize { return Err(GossipDataColumnError::ProposerIndexMismatch { sidecar: column_proposer_index as usize, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 83de13ecee3..a5581431955 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -344,39 +344,44 @@ async fn fetch_and_process_blobs_v2_or_v3( } // Initialize the partial assembler with the columns from the engine - let kzg_commitments = block - .message() - .body() - .blob_kzg_commitments() - .map_err(FetchEngineBlobError::BeaconStateError)? - .clone(); + let assembler = chain_adapter.partial_assembler(); + assembler + .init(block_root, block.as_ref()) + .map_err(FetchEngineBlobError::BeaconStateError)?; - let init_result = chain_adapter.partial_assembler().init_with_engine_blobs( - block_root, - block.slot(), - kzg_commitments, - custody_columns_to_import, - ); + let merge_result = assembler + .merge_partials( + block_root, + custody_columns_to_import + .into_iter() + .map(|c| c.into_inner()) + .collect(), + ) + .ok_or_else(|| { + FetchEngineBlobError::InternalError( + "merging partial failed immediately after initialization".to_string(), + ) + })?; // Publish complete columns and incomplete partials - if !init_result.complete_columns.is_empty() { + if !merge_result.full_columns.is_empty() { publish_fn(EngineGetBlobsOutput::CompleteColumns( - init_result.complete_columns.clone(), + merge_result.full_columns.clone(), )); } - if !init_result.incomplete_partials.is_empty() { + if !merge_result.updated_partials.is_empty() { publish_fn(EngineGetBlobsOutput::PartialColumns( - init_result.incomplete_partials.clone(), + merge_result.updated_partials, )); } // Process complete columns through DA checker - let availability_processing_status = if !init_result.complete_columns.is_empty() { + let availability_processing_status = if !merge_result.full_columns.is_empty() { chain_adapter .process_engine_blobs( block.slot(), block_root, - EngineGetBlobsOutput::CompleteColumns(init_result.complete_columns), + EngineGetBlobsOutput::CompleteColumns(merge_result.full_columns), ) .await? } else { diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 058f8b7b827..0978ba77def 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1647,6 +1647,35 @@ pub static DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_header_processing_requests_total", + "Count of all partial data column sidecars submitted for processing", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_DUPES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_header_processing_successes_total", + "Number of partial data column sidecars verified for gossip (excluding dupes)", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_SUCCESSES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_header_processing_successes_total", + "Number of partial data column sidecars verified for gossip (excluding dupes)", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_HEADER_GOSSIP_VERIFICATION_TIMES: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_histogram( + "beacon_partial_data_column_sidecar_header_gossip_verification_seconds", + "Full runtime of partial data column sidecars gossip verification", + ) +}); pub static PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 2dc9f9292c8..1dd07903eb8 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -1,11 +1,12 @@ -use crate::data_column_verification::KzgVerifiedCustodyPartialDataColumn; use lru::LruCache; use parking_lot::RwLock; +use parking_lot::lock_api::RwLockReadGuard; use std::collections::HashMap; use std::num::NonZeroUsize; +use std::ops::Deref; use std::sync::Arc; -use types::partial_data_column_sidecar::PartialDataColumn; -use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256, KzgCommitments, Slot}; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; +use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -15,26 +16,17 @@ pub struct PartialDataColumnAssembler { /// Tracks partial columns being assembled for a single block struct PartialAssembly { - #[allow(dead_code)] - block_root: Hash256, - slot: Slot, - kzg_commitments: KzgCommitments, + header: PartialDataColumnHeader, /// Map of column_index -> partial column being assembled columns: HashMap>>, } /// Result of merging a partial column -pub enum PartialMergeResult { - /// Merge was successful but column is still incomplete - Incomplete { - updated_partial: Arc>, - }, - /// Merge completed the column - Completed { - full_column: Arc>, - /// The updated partial for publishing (same as full but as partial type) - updated_partial: Arc>, - }, +pub struct PartialMergeResult { + /// Merge that completed the column + pub full_columns: Vec>>, + /// The updated partials for publishing + pub updated_partials: Vec>>, } impl PartialDataColumnAssembler { @@ -44,98 +36,85 @@ impl PartialDataColumnAssembler { } } - /// Initialize assembler with columns from engine getBlobsV3 response. - /// Returns immediately complete columns and partials that need gossip propagation. - pub fn init_with_engine_blobs( - &self, - block_root: Hash256, - slot: Slot, - kzg_commitments: KzgCommitments, - custody_columns: Vec>, - ) -> InitResult { - let mut result = InitResult { - complete_columns: vec![], - incomplete_partials: vec![], - }; + pub fn init(&self, block_root: Hash256, header: T) -> Result + where + T: TryInto>, + { + let assemblies = self.assemblies.write(); - let mut assembly = PartialAssembly { - block_root, - slot, - kzg_commitments: kzg_commitments.clone(), - columns: HashMap::new(), - }; + if assemblies.contains(&block_root) { + return Ok(false); + } - for custody_col in custody_columns { - let partial = custody_col.into_inner(); - let column_index = partial.index; + let header = header.try_into()?; - // Check if this partial is already complete - if partial.sidecar.is_complete() { - // Convert to full column - if let Some(full) = Self::partial_to_full(&partial, &kzg_commitments) { - result.complete_columns.push(full); - continue; - } - } - - // Store incomplete partial - let partial = Arc::new(partial); - result.incomplete_partials.push(partial.clone()); - assembly.columns.insert(column_index, partial); - } + let assembly = PartialAssembly { + header, + columns: HashMap::new(), + }; - // Only cache if we have incomplete columns - if !assembly.columns.is_empty() { - self.assemblies.write().put(block_root, assembly); - } + self.assemblies.write().put(block_root, assembly); - result + Ok(true) } /// Merge a received partial column into the assembly. /// Returns the merge result indicating if column is now complete. - pub fn merge_partial( + pub fn merge_partials( &self, block_root: Hash256, - partial: PartialDataColumn, + partials: Vec>, ) -> Option> { let mut assemblies = self.assemblies.write(); - let assembly = assemblies.get_mut(&block_root)?; + let assembly = assemblies + .try_get_or_insert_mut(block_root, || { + partials + .iter() + .filter_map(|partial| partial.sidecar.header.first()) + .next() + .map(|header| PartialAssembly { + header: header.clone(), + columns: HashMap::new(), + }) + .ok_or(()) + }) + .ok()?; - let column_index = partial.index; + let mut full_columns = Vec::new(); + let mut updated_partials = Vec::new(); - let merged = if let Some(existing) = assembly.columns.get(&column_index) { - // Merge with existing partial - let merged_sidecar = existing.sidecar.merge(&partial.sidecar)?; - Arc::new(PartialDataColumn { - block_root: existing.block_root, - index: existing.index, - sidecar: merged_sidecar, - }) - } else { - // First time seeing this column index for this block - Arc::new(partial.clone()) - }; + for partial in partials { + let column_index = partial.index; - // Check if merged column is now complete - if merged.sidecar.is_complete() { - // Remove from assembly since it's complete - assembly.columns.remove(&column_index); - - // Convert to full column - if let Some(full_column) = Self::partial_to_full(&merged, &assembly.kzg_commitments) { - return Some(PartialMergeResult::Completed { - full_column, - updated_partial: merged, - }); + let merged = if let Some(existing) = assembly.columns.get(&column_index) { + // Merge with existing partial + let Some(merged_sidecar) = existing.sidecar.merge(&partial.sidecar) else { + continue; + }; + PartialDataColumn { + block_root: existing.block_root, + index: existing.index, + sidecar: merged_sidecar, + } + } else { + // First time seeing this column index for this block + partial + }; + + // Check if merged column is now complete by trying to convert into full + if let Some(full_column) = merged.try_clone_full() { + full_columns.push(Arc::new(full_column)); } - } - // Update assembly with merged partial - assembly.columns.insert(column_index, merged.clone()); + // Update assembly with merged partial + let merged = Arc::new(merged); + assembly.columns.insert(column_index, merged.clone()); + updated_partials.push(merged); + } - Some(PartialMergeResult::Incomplete { - updated_partial: merged, + Some(PartialMergeResult { + full_columns, + updated_partials, }) } @@ -153,12 +132,15 @@ impl PartialDataColumnAssembler { .cloned() } - /// Get commitments for a block if we have an active assembly - pub fn get_commitments(&self, block_root: &Hash256) -> Option> { - self.assemblies - .read() - .peek(block_root) - .map(|assembly| assembly.kzg_commitments.clone()) + /// Get header for a block if we have an active assembly + pub fn get_header( + &self, + block_root: &Hash256, + ) -> Option>> { + RwLockReadGuard::try_map(self.assemblies.read(), |assemblies| { + assemblies.peek(block_root).map(|a| &a.header) + }) + .ok() } /// Check if we have an assembly for this block @@ -177,7 +159,14 @@ impl PartialDataColumnAssembler { let mut to_remove = vec![]; for (root, assembly) in assemblies.iter() { - if assembly.slot.epoch(E::slots_per_epoch()) < cutoff_epoch { + if assembly + .header + .signed_block_header + .message + .slot + .epoch(E::slots_per_epoch()) + < cutoff_epoch + { to_remove.push(*root); } } @@ -187,32 +176,6 @@ impl PartialDataColumnAssembler { } } - /// Convert a complete partial to a full DataColumnSidecar - fn partial_to_full( - partial: &PartialDataColumn, - kzg_commitments: &KzgCommitments, - ) -> Option>> { - // Use the as_full method - requires a block for signed header - // For now we'll construct it manually since we have all the data - if !partial.sidecar.is_complete() { - return None; - } - - let expected = partial.sidecar.cells_present_bitmap.len(); - if partial.sidecar.column.len() != expected - || partial.sidecar.kzg_proofs.len() != expected - || kzg_commitments.len() != expected - { - return None; - } - - // We need the signed block header and inclusion proof which requires the full block - // For the assembler, we'll return None here and require callers to use the - // VerifiablePartialDataColumn::as_full method with the actual block - // This is handled in the merge flow where we have access to the block - None - } - /// Get cache size for metrics pub fn cache_size(&self) -> usize { self.assemblies.read().len() diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index d17addea513..3b8a4b267f0 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -9,7 +9,6 @@ use beacon_chain::block_verification_types::AsBlock; use beacon_chain::data_column_verification::{ GossipDataColumnError, GossipVerifiedDataColumn, KzgVerifiedPartialDataColumn, }; -use beacon_chain::partial_data_column_assembler::PartialMergeResult; use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, @@ -34,6 +33,7 @@ use logging::crit; use operation_pool::ReceivedPreCapella; use slot_clock::SlotClock; use ssz::Encode; +use ssz_types::VariableList; use std::fs; use std::io::Write; use std::path::PathBuf; @@ -42,11 +42,12 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tracing::{Instrument, Span, debug, error, info, instrument, trace, warn}; use types::{ - Attestation, AttestationData, AttestationRef, AttesterSlashing, BlobSidecar, DataColumnSidecar, - DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate, - LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, - SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, - Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, + Attestation, AttestationData, AttestationRef, AttesterSlashing, BlobSidecar, ColumnIndex, + DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, + LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, + SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, + SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, Slot, SubnetId, + SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, }; use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; @@ -769,6 +770,10 @@ impl NetworkBeaconProcessor { MessageAcceptance::Ignore, ); } + GossipDataColumnError::PartialHeaderMismatches + | GossipDataColumnError::PartialHeaderIncorrectRoot { .. } => { + error!(error = ?err, "Unexpected error during full column verification"); + } } } } @@ -779,104 +784,57 @@ impl NetworkBeaconProcessor { parent = None, level = "debug", skip_all, - fields(block_root = ?column_sidecar.block_root, index = column_sidecar.index), + fields(block_root = ?column.block_root, index = column.index), )] - pub async fn process_gossip_dangling_partial_data_column_sidecar( - self: &Arc, - peer_id: PeerId, - column_sidecar: PartialDataColumn, - seen_duration: Duration, - ) { - // TODO(dknopik): add check and cache for header - - // TODO(dknopik): make this more elegant with above todo - let slot = column_sidecar - .sidecar - .header - .first() - .unwrap() - .signed_block_header - .message - .slot; - - self.process_gossip_verifiable_partial_data_column_sidecar( - peer_id, - column_sidecar, - slot, - seen_duration, - ) - .await; - } - - /*fn handle_dangling_partial_data_column_sidecar_missing_block( + pub async fn process_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: Arc>, + mut column: PartialDataColumn, seen_duration: Duration, - allow_reprocess: bool, ) { - // TODO(dknopik): is this method of checking adequate?... - if self - .chain - .store - .block_exists(&column_sidecar.block_root) - .unwrap_or(false) - { - debug!("Received partial for already imported block"); - return; - } - if !allow_reprocess { - debug!("Not reprocessing"); - return; - } - debug!("Received partial while not having block"); - metrics::inc_counter( - &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, - ); - // TODO(dknopik): self.send_sync_message() ? - let cloned_self = self.clone(); - if self - .beacon_processor_send - .try_send(WorkEvent { - drop_during_sync: false, - work: Work::Reprocess(ReprocessQueueMessage::UnknownPartialColumn( - QueuedPartialColumn { - beacon_block_root: column_sidecar.block_root, - process_fn: Box::pin(async move { - cloned_self - .process_gossip_dangling_partial_data_column_sidecar( - peer_id, - column_sidecar, - seen_duration, - false, - ) - .await; - }), - }, - )), - }) - .is_err() - { - error!("Failed to send attestation for re-processing") - } - }*/ + let slot = if let Some(header) = column.sidecar.header.first() { + // Header was sent, so it is required to be valid + match self + .chain + .verify_partial_data_column_header_for_gossip(column.block_root, header.clone()) + { + Ok(verified) => { + if !verified.was_cached() { + self.chain + .data_availability_checker + .put_partial_data_column_header(verified.into_inner()) + // TODO(dknopik): trigger getBlobsV3 + } + } + Err(err) => { + self.handle_partial_verification_error( + peer_id, + err, + column.block_root, + column.index, + ); + return; + } + } + header.signed_block_header.message.slot + } else { + // There is no header, so we check if we have a cached one to use + let Some(header) = self + .chain + .data_availability_checker + .partial_assembler() + .get_header(&column.block_root) + else { + error!(block=%column.block_root, index=%column.index, "Received partial column while not having header stored"); + return; + }; + let slot = header.signed_block_header.message.slot; + column.sidecar.header = VariableList::repeat_full((*header).clone()); + slot + }; - #[instrument( - name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, - parent = None, - level = "debug", - skip_all, - fields(slot = %slot, block_root = ?column_sidecar.block_root, index = column_sidecar.index), - )] - pub async fn process_gossip_verifiable_partial_data_column_sidecar( - self: &Arc, - peer_id: PeerId, - column_sidecar: PartialDataColumn, - slot: Slot, - seen_duration: Duration, - ) { - let block_root = column_sidecar.block_root; - let index = column_sidecar.index; + let block_root = column.block_root; + let index = column.index; let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. metrics::observe_duration( @@ -885,7 +843,7 @@ impl NetworkBeaconProcessor { ); match self .chain - .verify_partial_data_column_sidecar_for_gossip(column_sidecar) + .verify_partial_data_column_sidecar_for_gossip(column) { Ok(gossip_verified_data_column) => { metrics::inc_counter( @@ -922,97 +880,117 @@ impl NetworkBeaconProcessor { .await } Err(err) => { - match err { - GossipDataColumnError::PriorKnownUnpublished => { - debug!( - %slot, - %block_root, - %index, - "Gossip data column already processed via the EL. Accepting the column sidecar without re-processing." - ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Accept, - //); - } - // ParentUnknown can't really happen, so we treat it like an internal error - GossipDataColumnError::ParentUnknown { .. } - | GossipDataColumnError::PubkeyCacheTimeout - | GossipDataColumnError::BeaconChainError(_) => { - crit!( - error = ?err, - "Internal error when verifying column sidecar" - ) - } - GossipDataColumnError::ProposalSignatureInvalid - | GossipDataColumnError::UnknownValidator(_) - | GossipDataColumnError::ProposerIndexMismatch { .. } - | GossipDataColumnError::IsNotLaterThanParent { .. } - | GossipDataColumnError::InvalidSubnetId { .. } - | GossipDataColumnError::InvalidInclusionProof - | GossipDataColumnError::InvalidKzgProof { .. } - | GossipDataColumnError::UnexpectedDataColumn - | GossipDataColumnError::InvalidColumnIndex(_) - | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } - | GossipDataColumnError::InconsistentCommitmentsLength { .. } - | GossipDataColumnError::InconsistentProofsLength { .. } - | GossipDataColumnError::NotFinalizedDescendant { .. } => { - debug!( - error = ?err, - %slot, - %block_root, - %index, - "Could not verify partial column for gossip. Rejecting the column sidecar" - ); - // Prevent recurring behaviour by penalizing the peer slightly. - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_data_column_low", - ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Reject, - //); - } - GossipDataColumnError::PriorKnown { .. } => { - // Data column is available via either the EL or reconstruction. - // Do not penalise the peer. - // Gossip filter should filter any duplicates received after this. - debug!( - %slot, - %block_root, - %index, - "Received already available column sidecar. Ignoring the column sidecar" - ) - } - GossipDataColumnError::FutureSlot { .. } - | GossipDataColumnError::PastFinalizedSlot { .. } => { - debug!( - error = ?err, - %slot, - %block_root, - %index, - "Could not verify column sidecar for gossip. Ignoring the column sidecar" - ); - // Prevent recurring behaviour by penalizing the peer slightly. - self.gossip_penalize_peer( - peer_id, - PeerAction::HighToleranceError, - "gossip_data_column_high", - ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Ignore, - //); - } - } + self.handle_partial_verification_error(peer_id, err, block_root, index); + } + } + } + + fn handle_partial_verification_error( + self: &Arc, + peer_id: PeerId, + err: GossipDataColumnError, + block_root: Hash256, + index: ColumnIndex, + ) { + match err { + GossipDataColumnError::PriorKnownUnpublished => { + debug!( + %block_root, + %index, + "Gossip data column already processed via the EL. Accepting the column sidecar without re-processing." + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Accept, + //); + } + // ParentUnknown can't really happen, so we treat it like an internal error + GossipDataColumnError::ParentUnknown { .. } + | GossipDataColumnError::PubkeyCacheTimeout + | GossipDataColumnError::BeaconChainError(_) => { + crit!( + error = ?err, + "Internal error when verifying column sidecar" + ) + } + GossipDataColumnError::ProposalSignatureInvalid + | GossipDataColumnError::UnknownValidator(_) + | GossipDataColumnError::ProposerIndexMismatch { .. } + | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::InvalidSubnetId { .. } + | GossipDataColumnError::InvalidInclusionProof + | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::UnexpectedDataColumn + | GossipDataColumnError::InvalidColumnIndex(_) + | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } + | GossipDataColumnError::InconsistentCommitmentsLength { .. } + | GossipDataColumnError::InconsistentProofsLength { .. } + | GossipDataColumnError::NotFinalizedDescendant { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify partial column for gossip. Rejecting the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_data_column_low", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Reject, + //); + } + GossipDataColumnError::PriorKnown { .. } => { + // Data column is available via either the EL or reconstruction. + // Do not penalise the peer. + // Gossip filter should filter any duplicates received after this. + debug!( + %block_root, + %index, + "Received already available column sidecar. Ignoring the column sidecar" + ) + } + GossipDataColumnError::FutureSlot { .. } + | GossipDataColumnError::PastFinalizedSlot { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify column sidecar for gossip. Ignoring the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::HighToleranceError, + "gossip_data_column_high", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Ignore, + //); + } + GossipDataColumnError::PartialHeaderMismatches + | GossipDataColumnError::PartialHeaderIncorrectRoot { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify partial column header" + ); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_data_column_high", + ); } } } @@ -1394,51 +1372,37 @@ impl NetworkBeaconProcessor { .process_gossip_partial_data_column(verified_partial) .await; - match &result { + match result { Ok(merge_result) => { - match merge_result { - PartialMergeResult::Completed { - full_column, - updated_partial, - } => { - debug!( - %block_root, - index = data_column_index, - "Partial data column completed to full column" - ); + if !merge_result.full_columns.is_empty() { + debug!( + %block_root, + index = data_column_index, + "Partial data column completed to full column" + ); + for full_column in merge_result.full_columns { // Publish both the updated partial and the full column let subnet = DataColumnSubnetId::from_column_index( full_column.index, &self.chain.spec, ); - self.send_network_message(NetworkMessage::PublishPartial { - columns: vec![updated_partial.clone()], - }); self.send_network_message(NetworkMessage::Publish { messages: vec![PubsubMessage::DataColumnSidecar(Box::new(( subnet, full_column.clone(), )))], }); - - // Check if block is now fully available - // This is handled in process_gossip_partial_data_column - } - PartialMergeResult::Incomplete { updated_partial } => { - trace!( - %block_root, - index = data_column_index, - "Partial data column merged but column still incomplete" - ); - - // Publish the updated partial - self.send_network_message(NetworkMessage::PublishPartial { - columns: vec![updated_partial.clone()], - }); } } + self.send_network_message(NetworkMessage::PublishPartial { + columns: merge_result.updated_partials, + }); + + // Check if block is now fully available + // This is handled in process_gossip_partial_data_column + metrics::set_gauge( &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, processing_start_time.elapsed().as_millis() as i64, @@ -1453,14 +1417,6 @@ impl NetworkBeaconProcessor { ); } } - - // Notify sync if block was imported - if result.is_ok() { - self.send_sync_message(SyncMessage::GossipBlockProcessResult { - block_root, - imported: false, // Will be true if completed in the chain method - }); - } } /// Process the beacon block received from the gossip network and: diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 052a716c871..c8a1f02f00e 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -260,11 +260,7 @@ impl NetworkBeaconProcessor { let processor = self.clone(); let process_fn = async move { processor - .process_gossip_dangling_partial_data_column_sidecar( - peer_id, - column_sidecar, - seen_timestamp, - ) + .process_gossip_partial_data_column_sidecar(peer_id, column_sidecar, seen_timestamp) .await }; diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 4a428fef757..2f8ed628ba9 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,12 +1,17 @@ use crate::test_utils::TestRandom; -use crate::{Cell, ColumnIndex, EthSpec, Hash256, KzgCommitments, SignedBeaconBlockHeader}; +use crate::{ + BLOB_KZG_COMMITMENTS_INDEX, BeaconStateError, Cell, ColumnIndex, DataColumnSidecar, EthSpec, + Hash256, KzgCommitments, SignedBeaconBlock, SignedBeaconBlockHeader, +}; use educe::Educe; use kzg::KzgProof; +use merkle_proof::verify_merkle_proof; use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use test_random_derive::TestRandom; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use typenum::U1; @@ -191,6 +196,36 @@ pub struct PartialDataColumnHeader { pub kzg_commitments_inclusion_proof: FixedVector, } +impl PartialDataColumnHeader { + pub fn verify_inclusion_proof(&self) -> bool { + let blob_kzg_commitments_root = self.kzg_commitments.tree_hash_root(); + + verify_merkle_proof( + blob_kzg_commitments_root, + &self.kzg_commitments_inclusion_proof, + E::kzg_commitments_inclusion_proof_depth(), + BLOB_KZG_COMMITMENTS_INDEX, + self.signed_block_header.message.body_root, + ) + } +} + +impl TryFrom<&SignedBeaconBlock> for PartialDataColumnHeader { + type Error = BeaconStateError; + + fn try_from(block: &SignedBeaconBlock) -> Result { + Ok(Self { + kzg_commitments: block.message().body().blob_kzg_commitments()?.clone(), + signed_block_header: block.signed_block_header(), + kzg_commitments_inclusion_proof: block + .message() + .body() + .kzg_commitments_merkle_proof()? + .clone(), + }) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumn { pub block_root: Hash256, @@ -209,4 +244,29 @@ impl PartialDataColumn { index: self.index, }) } + + pub fn try_clone_full(&self) -> Option> { + if !self.sidecar.is_complete() { + return None; + } + let Some(header) = self.sidecar.header.first() else { + return None; + }; + + Some(DataColumnSidecar { + index: self.index, + column: VariableList::new( + self.sidecar + .column + .iter() + .map(|cell| cell.as_ref().clone()) + .collect(), + ) + .expect("Same bounds"), + kzg_commitments: header.kzg_commitments.clone(), + kzg_proofs: self.sidecar.kzg_proofs.clone(), + signed_block_header: header.signed_block_header.clone(), + kzg_commitments_inclusion_proof: header.kzg_commitments_inclusion_proof.clone(), + }) + } } From d42cbbe49ac4795d4fd394b8cae3ff7e4c1a41b3 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 21:37:52 +0100 Subject: [PATCH 061/146] Fix obvious mistakes --- .../beacon_chain/src/data_availability_checker.rs | 10 ++++------ .../beacon_chain/src/partial_data_column_assembler.rs | 4 ++-- crypto/kzg/src/lib.rs | 8 ++++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4cddcbf4922..226c624ee08 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -450,9 +450,8 @@ impl DataAvailabilityChecker { executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { let block = executed_block.as_block(); - self.partial_assembler - .init(block.canonical_root(), block) - .map_err(Error::PartialAssemblerError)?; + // Ignore error which happens at pre-fulu blocks + let _ = self.partial_assembler.init(block.canonical_root(), block); self.availability_cache.put_executed_block(executed_block) } @@ -464,9 +463,8 @@ impl DataAvailabilityChecker { block: Arc>, source: BlockImportSource, ) -> Result<(), Error> { - self.partial_assembler - .init(block_root, block.as_ref()) - .map_err(Error::PartialAssemblerError)?; + // Ignore error which happens at pre-fulu blocks + let _ = self.partial_assembler.init(block_root, block.as_ref()); self.availability_cache .put_pre_execution_block(block_root, block, source) } diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 1dd07903eb8..ddec65c5619 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -40,7 +40,7 @@ impl PartialDataColumnAssembler { where T: TryInto>, { - let assemblies = self.assemblies.write(); + let mut assemblies = self.assemblies.write(); if assemblies.contains(&block_root) { return Ok(false); @@ -53,7 +53,7 @@ impl PartialDataColumnAssembler { columns: HashMap::new(), }; - self.assemblies.write().put(block_root, assembly); + assemblies.put(block_root, assembly); Ok(true) } diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 05b016f596a..642995afb85 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -13,17 +13,17 @@ pub use crate::{ }; pub use c_kzg::{ - BYTES_PER_BLOB, BYTES_PER_COMMITMENT, BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, Blob, Bytes32, - Bytes48, FIELD_ELEMENTS_PER_BLOB, KzgSettings, + Blob, Bytes32, Bytes48, KzgSettings, BYTES_PER_BLOB, BYTES_PER_COMMITMENT, + BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, FIELD_ELEMENTS_PER_BLOB, }; use crate::trusted_setup::load_trusted_setup; use rayon::prelude::*; pub use rust_eth_kzg::{ - Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, + Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, }; -use tracing::{Span, instrument}; +use tracing::{instrument, Span}; /// Disables the fixed-base multi-scalar multiplication optimization for computing /// cell KZG proofs, because `rust-eth-kzg` already handles the precomputation. From 803b1154c983a7a4265f7336ab955d1569aa0a20 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 23:45:34 +0100 Subject: [PATCH 062/146] Remove unnecessary explicit init --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index a5581431955..bf4d738cf59 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -345,10 +345,6 @@ async fn fetch_and_process_blobs_v2_or_v3( // Initialize the partial assembler with the columns from the engine let assembler = chain_adapter.partial_assembler(); - assembler - .init(block_root, block.as_ref()) - .map_err(FetchEngineBlobError::BeaconStateError)?; - let merge_result = assembler .merge_partials( block_root, From 011b1bbb88355503b4ca759c4a210d0838a54f19 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 23:45:34 +0100 Subject: [PATCH 063/146] Remove outdated todo --- consensus/types/src/data/data_column_sidecar.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index b67a0ed4552..8ef65ef2c2d 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -29,7 +29,7 @@ use crate::{ }; pub type ColumnIndex = u64; -pub type Cell = FixedVector::BytesPerCell>; // TODO(dknopik): Arc<[u8; E::BytesPerCell]> ??? cell level arcing acors the codebase seems reasonable +pub type Cell = FixedVector::BytesPerCell>; pub type DataColumn = VariableList, ::MaxBlobCommitmentsPerBlock>; /// Identifies a set of data columns associated with a specific beacon block. From 7ce4d23a7ae5b93ed1e0a76af885a943683089dd Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 23:45:34 +0100 Subject: [PATCH 064/146] Fix fetch blobs tests --- .../beacon_chain/src/fetch_blobs/tests.rs | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 490cd56b7f2..7e2fa694e8f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -3,12 +3,15 @@ use crate::fetch_blobs::fetch_blobs_beacon_adapter::MockFetchBlobsBeaconAdapter; use crate::fetch_blobs::{ EngineGetBlobsOutput, FetchEngineBlobError, fetch_and_process_engine_blobs_inner, }; +use crate::partial_data_column_assembler::PartialDataColumnAssembler; use crate::test_utils::{EphemeralHarnessType, get_kzg}; use bls::Signature; use eth2::types::BlobsBundle; use execution_layer::json_structures::{BlobAndProof, BlobAndProofV1, BlobAndProofV2}; use execution_layer::test_utils::generate_blobs; use maplit::hashset; +use std::mem; +use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; use task_executor::test_utils::TestRuntime; use types::{ @@ -223,8 +226,8 @@ mod get_blobs_v2 { let published_columns = extract_published_blobs(publish_fn_args); assert!( matches!( - published_columns, - EngineGetBlobsOutput::CompleteColumns(columns) if columns.len() == custody_columns.len() + &published_columns[..], + [EngineGetBlobsOutput::CompleteColumns(complete_columns), EngineGetBlobsOutput::PartialColumns(partial_columns)] if complete_columns.len() == custody_columns.len() && complete_columns.len() == partial_columns.len() ), "should publish custody columns" ); @@ -363,8 +366,8 @@ mod get_blobs_v1 { ); assert!( matches!( - extract_published_blobs(publish_fn_args), - EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count - 1 + &extract_published_blobs(publish_fn_args)[..], + [EngineGetBlobsOutput::Blobs(blobs)] if blobs.len() == blob_count - 1 ), "partial blob results should still be published" ); @@ -495,8 +498,8 @@ mod get_blobs_v1 { let published_blobs = extract_published_blobs(publish_fn_args); assert!( matches!( - published_blobs, - EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count + &published_blobs[..], + [EngineGetBlobsOutput::Blobs(blobs)] if blobs.len() == blob_count ), "should publish fetched blobs" ); @@ -521,13 +524,11 @@ mod get_blobs_v1 { } } -/// Extract the `EngineGetBlobsOutput` passed to the `publish_fn`. +/// Extract the `EngineGetBlobsOutput`s passed to the `publish_fn`. fn extract_published_blobs( publish_fn_args: Arc>>>, -) -> EngineGetBlobsOutput { - let mut calls = publish_fn_args.lock().unwrap(); - assert_eq!(calls.len(), 1); - calls.pop().unwrap() +) -> Vec> { + mem::take(publish_fn_args.lock().unwrap().as_mut()) } fn mock_process_engine_blobs_result( @@ -610,6 +611,7 @@ fn mock_beacon_adapter(fork_name: ForkName, get_blobs_v3: bool) -> MockFetchBlob let test_runtime = TestRuntime::default(); let spec = Arc::new(fork_name.make_genesis_spec(E::default_spec())); let kzg = get_kzg(&spec); + let partial_assembler = PartialDataColumnAssembler::new(NonZeroUsize::new(32).unwrap()); let mut mock_adapter = MockFetchBlobsBeaconAdapter::default(); mock_adapter.expect_spec().return_const(spec.clone()); @@ -621,4 +623,7 @@ fn mock_beacon_adapter(fork_name: ForkName, get_blobs_v3: bool) -> MockFetchBlob .expect_supports_get_blobs_v3() .returning(move || Ok(get_blobs_v3)); mock_adapter + .expect_partial_assembler() + .return_const(Arc::new(partial_assembler)); + mock_adapter } From fcdfa4c070749c372f547610fc76912373abc85c Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 21 Jan 2026 23:45:34 +0100 Subject: [PATCH 065/146] Undo buggy changes in block publish flow --- beacon_node/http_api/src/publish_blocks.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 22fdc806720..4ac600e29d0 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -234,34 +234,29 @@ pub async fn publish_block>( tokio::time::sleep(delay).await; } } - // Publish columns to network first (eager push for builder/producer case) publish_column_sidecars(network_tx, &gossip_verified_columns, &chain).map_err(|_| { warp_utils::reject::custom_server_error("unable to publish data column sidecars".into()) })?; - - // Add full columns directly to DA checker (not via gossip verification) let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let sampling_columns_indices = chain.sampling_columns_for_epoch(epoch); - let sampling_columns: Vec<_> = gossip_verified_columns + let sampling_columns = gossip_verified_columns .into_iter() .filter(|data_column| sampling_columns_indices.contains(&data_column.index())) - .map(|gv| gv.into_inner().to_data_column()) - .collect(); + .collect::>(); if !sampling_columns.is_empty() { - // Add columns directly to DA checker (block producer case) - let block_root = block.tree_hash_root(); - if let Err(e) = chain - .data_availability_checker - .put_full_data_columns(block_root, sampling_columns) + // Importing the columns could trigger block import and network publication in the case + // where the block was already seen on gossip. + if let Err(e) = + Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn)).await { - let msg = format!("Failed to add data columns to DA checker: {e:?}"); + let msg = format!("Invalid data column: {e}"); return if let BroadcastValidation::Gossip = validation_level { Err(warp_utils::reject::broadcast_without_import(msg)) } else { error!( reason = &msg, - "Data column addition failed during block publication" + "Invalid data column during block publication" ); Err(warp_utils::reject::custom_bad_request(msg)) }; From d5c57f701cb64a7af3f4658c6c088679360cea52 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 22 Jan 2026 01:09:02 +0100 Subject: [PATCH 066/146] fix SSE --- beacon_node/beacon_chain/src/beacon_chain.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a5b57bc0a1b..1c1bd550db2 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3277,17 +3277,14 @@ impl BeaconChain { { let imported_data_columns = self .data_availability_checker - .get_data_columns(*block_root) + .cached_data_column_indexes(block_root) .unwrap_or_default(); + let new_data_columns = + data_columns_iter.filter(|b| !imported_data_columns.contains(&b.index)); - let new_data_columns_indices = data_columns_iter.map(|c| c.index).collect::>(); - - for data_column in imported_data_columns - .into_iter() - .filter(|c| new_data_columns_indices.contains(&c.index)) - { + for data_column in new_data_columns { event_handler.register(EventKind::DataColumnSidecar( - SseDataColumnSidecar::from_data_column_sidecar(data_column.as_ref()), + SseDataColumnSidecar::from_data_column_sidecar(data_column), )); } } From ba3b6b0b0f17b5a9cb6025e4dc414505ad58c7ca Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 22 Jan 2026 01:09:02 +0100 Subject: [PATCH 067/146] Implement getting blobs after header arrival --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../src/data_availability_checker.rs | 16 ++-- .../src/data_column_verification.rs | 15 ++- .../beacon_chain/src/fetch_blobs/mod.rs | 94 ++++++++----------- .../beacon_chain/src/fetch_blobs/tests.rs | 34 ++++--- beacon_node/beacon_chain/src/kzg_utils.rs | 50 +++------- .../src/partial_data_column_assembler.rs | 22 ++++- .../gossip_methods.rs | 41 ++++---- .../src/network_beacon_processor/mod.rs | 8 +- .../network_beacon_processor/sync_methods.rs | 10 +- .../types/src/block/beacon_block_body.rs | 50 ++-------- consensus/types/src/data/blob_sidecar.rs | 29 +++--- .../src/data/partial_data_column_sidecar.rs | 15 ++- consensus/types/src/kzg_ext/mod.rs | 52 +++++++++- .../generate_random_block_and_blobs.rs | 16 +--- 15 files changed, 224 insertions(+), 230 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1c1bd550db2..1481cd820d4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3147,7 +3147,7 @@ impl BeaconChain { let merge_result = self .data_availability_checker .partial_assembler() - .merge_partials(block_root, vec![partial]) + .merge_partials(block_root, vec![partial], false) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; // If we completed a column, process it through the DA checker diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 226c624ee08..48488b0588f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -450,8 +450,7 @@ impl DataAvailabilityChecker { executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { let block = executed_block.as_block(); - // Ignore error which happens at pre-fulu blocks - let _ = self.partial_assembler.init(block.canonical_root(), block); + self.partial_assembler.init(block.canonical_root(), block); self.availability_cache.put_executed_block(executed_block) } @@ -463,19 +462,20 @@ impl DataAvailabilityChecker { block: Arc>, source: BlockImportSource, ) -> Result<(), Error> { - // Ignore error which happens at pre-fulu blocks - let _ = self.partial_assembler.init(block_root, block.as_ref()); + self.partial_assembler.init(block_root, block.as_ref()); self.availability_cache .put_pre_execution_block(block_root, block, source) } /// Inserts a pre-execution block into the cache. /// This does NOT override an existing executed block. - pub fn put_partial_data_column_header(&self, header: PartialDataColumnHeader) { - let _ = self - .partial_assembler + /// Returns true if this was the first insertion. + pub fn put_partial_data_column_header( + &self, + header: PartialDataColumnHeader, + ) -> bool { + self.partial_assembler .init(header.signed_block_header.message.canonical_root(), header) - .map_err(|e| match e {}); } /// Removes a pre-execution block from the cache. diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index e9fbd4ca9a8..76e52fafb62 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -420,7 +420,7 @@ impl KzgVerifiedPartialDataColumn { #[educe(PartialEq, Eq)] pub struct GossipVerifiedPartialDataColumnHeader { header: PartialDataColumnHeader, - cached: bool, + previously_cached: bool, } impl GossipVerifiedPartialDataColumnHeader { @@ -429,7 +429,7 @@ impl GossipVerifiedPartialDataColumnHeader { header: PartialDataColumnHeader, chain: &BeaconChain, ) -> Result { - let column_slot = header.signed_block_header.message.slot; + let column_slot = header.slot(); if header.kzg_commitments.is_empty() { return Err(GossipDataColumnError::UnexpectedDataColumn); } @@ -452,6 +452,11 @@ impl GossipVerifiedPartialDataColumnHeader { verify_slot_higher_than_parent(&parent_block, column_slot)?; verify_proposer_and_signature(&header.signed_block_header, &parent_block, chain)?; + // Cache the valid header + let newly_cached = chain + .data_availability_checker + .put_partial_data_column_header(header.clone()); + chain .observed_slashable .write() @@ -464,14 +469,14 @@ impl GossipVerifiedPartialDataColumnHeader { Ok(Self { header, - cached: false, + previously_cached: !newly_cached, }) } pub fn new_from_cached(header: PartialDataColumnHeader) -> Self { Self { header, - cached: true, + previously_cached: true, } } @@ -484,7 +489,7 @@ impl GossipVerifiedPartialDataColumnHeader { } pub fn was_cached(&self) -> bool { - self.cached + self.previously_cached } } diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index bf4d738cf59..b6acbf2943f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -13,7 +13,6 @@ mod fetch_blobs_beacon_adapter; mod tests; use crate::blob_verification::{GossipBlobError, KzgVerifiedBlob}; -use crate::block_verification_types::AsBlock; use crate::data_column_verification::{ KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, }; @@ -31,16 +30,12 @@ use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndPr use metrics::{TryExt, inc_counter}; #[cfg(test)] use mockall_double::double; -use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; use types::data::{BlobSidecarError, DataColumnSidecarError}; -use types::partial_data_column_sidecar::PartialDataColumn; -use types::{ - BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, SignedBeaconBlock, - SignedBeaconBlockHeader, VersionedHash, -}; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; +use types::{BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, Hash256, VersionedHash}; /// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the /// gossip network. The blobs / data columns have not been marked as observed yet, as they may not @@ -76,14 +71,14 @@ pub enum FetchEngineBlobError { pub async fn fetch_and_process_engine_blobs( chain: Arc>, block_root: Hash256, - block: Arc>>, + header: &PartialDataColumnHeader, custody_columns: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, ) -> Result, FetchEngineBlobError> { fetch_and_process_engine_blobs_inner( FetchBlobsBeaconAdapter::new(chain), block_root, - block, + header, custody_columns, publish_fn, ) @@ -95,34 +90,28 @@ pub async fn fetch_and_process_engine_blobs( async fn fetch_and_process_engine_blobs_inner( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, - block: Arc>>, + header: &PartialDataColumnHeader, custody_columns: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, ) -> Result, FetchEngineBlobError> { - let versioned_hashes = if let Some(kzg_commitments) = block - .message() - .body() - .blob_kzg_commitments() - .ok() - .filter(|blobs| !blobs.is_empty()) - { - kzg_commitments - .iter() - .map(kzg_commitment_to_versioned_hash) - .collect::>() - } else { + let versioned_hashes = header + .kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect::>(); + if versioned_hashes.is_empty() { debug!("Fetch blobs not triggered - none required"); return Ok(None); }; if chain_adapter .spec() - .is_peer_das_enabled_for_epoch(block.epoch()) + .is_peer_das_enabled_for_epoch(header.slot().epoch(T::EthSpec::slots_per_epoch())) { fetch_and_process_blobs_v2_or_v3( chain_adapter, block_root, - block, + header, versioned_hashes, custody_columns, publish_fn, @@ -132,7 +121,7 @@ async fn fetch_and_process_engine_blobs_inner( fetch_and_process_blobs_v1( chain_adapter, block_root, - block, + header, versioned_hashes, publish_fn, ) @@ -144,7 +133,7 @@ async fn fetch_and_process_engine_blobs_inner( async fn fetch_and_process_blobs_v1( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, - block: Arc>, + header: &PartialDataColumnHeader, versioned_hashes: Vec, publish_fn: impl Fn(EngineGetBlobsOutput) + Send + Sized, ) -> Result, FetchEngineBlobError> { @@ -182,20 +171,12 @@ async fn fetch_and_process_blobs_v1( return Ok(None); } - let (signed_block_header, kzg_commitments_proof) = block - .signed_block_header_and_kzg_commitments_proof() - .map_err(FetchEngineBlobError::BeaconStateError)?; + let mut blob_sidecar_list = build_blob_sidecars(header, response)?; - let mut blob_sidecar_list = build_blob_sidecars( - &block, - response, - signed_block_header, - &kzg_commitments_proof, - )?; - - if let Some(observed_blobs) = - chain_adapter.blobs_known_for_proposal(block.message().proposer_index(), block.slot()) - { + if let Some(observed_blobs) = chain_adapter.blobs_known_for_proposal( + header.signed_block_header.message.proposer_index, + header.slot(), + ) { blob_sidecar_list.retain(|blob| !observed_blobs.contains(&blob.blob_index())); if blob_sidecar_list.is_empty() { debug!( @@ -224,7 +205,7 @@ async fn fetch_and_process_blobs_v1( let availability_processing_status = chain_adapter .process_engine_blobs( - block.slot(), + header.slot(), block_root, EngineGetBlobsOutput::Blobs(blob_sidecar_list), ) @@ -237,7 +218,7 @@ async fn fetch_and_process_blobs_v1( async fn fetch_and_process_blobs_v2_or_v3( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, - block: Arc>, + header: &PartialDataColumnHeader, versioned_hashes: Vec, custody_columns_indices: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, @@ -329,7 +310,7 @@ async fn fetch_and_process_blobs_v2_or_v3( let custody_columns_to_import = compute_custody_columns_to_import( &chain_adapter, block_root, - block.clone(), + header, blobs_and_proofs, custody_columns_indices, ) @@ -352,10 +333,11 @@ async fn fetch_and_process_blobs_v2_or_v3( .into_iter() .map(|c| c.into_inner()) .collect(), + true, ) .ok_or_else(|| { FetchEngineBlobError::InternalError( - "merging partial failed immediately after initialization".to_string(), + "Failed to merge partials into assembler".to_string(), ) })?; @@ -375,14 +357,14 @@ async fn fetch_and_process_blobs_v2_or_v3( let availability_processing_status = if !merge_result.full_columns.is_empty() { chain_adapter .process_engine_blobs( - block.slot(), + header.slot(), block_root, EngineGetBlobsOutput::CompleteColumns(merge_result.full_columns), ) .await? } else { // No complete columns yet, still missing components - AvailabilityProcessingStatus::MissingComponents(block.slot(), block_root) + AvailabilityProcessingStatus::MissingComponents(header.slot(), block_root) }; Ok(Some(availability_processing_status)) @@ -392,7 +374,7 @@ async fn fetch_and_process_blobs_v2_or_v3( async fn compute_custody_columns_to_import( chain_adapter: &Arc>, block_root: Hash256, - block: Arc>>, + header: &PartialDataColumnHeader, blobs_and_proofs: Vec>, custody_columns_indices: &[ColumnIndex], ) -> Result>, FetchEngineBlobError> { @@ -400,6 +382,7 @@ async fn compute_custody_columns_to_import( let spec = chain_adapter.spec().clone(); let chain_adapter_cloned = chain_adapter.clone(); let custody_columns_indices = custody_columns_indices.to_vec(); + let header = header.clone(); let current_span = Span::current(); chain_adapter .executor() @@ -420,7 +403,7 @@ async fn compute_custody_columns_to_import( }) .collect::>(); let data_columns_result = - blobs_to_partial_data_columns(blob_and_cell_refs, &block, &kzg, &spec) + blobs_to_partial_data_columns(blob_and_cell_refs, &header, &kzg, &spec) .discard_timer_on_break(&mut timer); drop(timer); @@ -442,9 +425,12 @@ async fn compute_custody_columns_to_import( .map_err(FetchEngineBlobError::DataColumnSidecarError)?; // Only consider columns that are not already observed on gossip. - if let Some(observed_columns) = chain_adapter_cloned.data_column_known_for_proposal( - ProposalKey::new(block.message().proposer_index(), block.slot()), - ) { + if let Some(observed_columns) = + chain_adapter_cloned.data_column_known_for_proposal(ProposalKey::new( + header.signed_block_header.message.proposer_index, + header.slot(), + )) + { custody_columns.retain(|col| !observed_columns.contains(&col.index())); if custody_columns.is_empty() { return Ok(vec![]); @@ -471,10 +457,8 @@ async fn compute_custody_columns_to_import( } fn build_blob_sidecars( - block: &Arc>>, + header: &PartialDataColumnHeader, response: Vec>>, - signed_block_header: SignedBeaconBlockHeader, - kzg_commitments_inclusion_proof: &FixedVector, ) -> Result>, FetchEngineBlobError> { let mut sidecars = vec![]; for (index, blob_and_proof) in response @@ -485,9 +469,7 @@ fn build_blob_sidecars( let blob_sidecar = BlobSidecar::new_with_existing_proof( index, blob_and_proof.blob, - block, - signed_block_header.clone(), - kzg_commitments_inclusion_proof, + header.clone(), blob_and_proof.proof, ) .map_err(FetchEngineBlobError::BlobSidecarError)?; diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 7e2fa694e8f..b6ceee7173c 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -25,6 +25,7 @@ type T = EphemeralHarnessType; mod get_blobs_v2 { use super::*; use types::ColumnIndex; + use types::partial_data_column_sidecar::PartialDataColumnHeader; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_in_block() { @@ -44,7 +45,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - Arc::new(block), + &(&block).try_into().unwrap(), &custody_columns, publish_fn, ) @@ -69,7 +70,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -97,7 +98,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -130,7 +131,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -169,7 +170,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -211,7 +212,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -256,7 +257,8 @@ mod get_blobs_v1 { use super::*; use crate::block_verification_types::AsBlock; use std::collections::HashSet; - use types::ColumnIndex; + use types::partial_data_column_sidecar::PartialDataColumnHeader; + use types::{ColumnIndex, FullPayload}; const ELECTRA_FORK: ForkName = ForkName::Electra; @@ -265,8 +267,10 @@ mod get_blobs_v1 { let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let spec = mock_adapter.spec(); let (publish_fn, _s) = mock_publish_fn(); - let block_no_blobs = - SignedBeaconBlock::from_block(BeaconBlock::empty(spec), Signature::empty()); + let block_no_blobs = SignedBeaconBlock::>::from_block( + BeaconBlock::empty(spec), + Signature::empty(), + ); let block_root = block_no_blobs.canonical_root(); // Expectations: engine fetch blobs should not be triggered @@ -277,7 +281,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - Arc::new(block_no_blobs), + &PartialDataColumnHeader::try_from(&block_no_blobs).unwrap(), &custody_columns, publish_fn, ) @@ -304,7 +308,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -350,7 +354,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -390,7 +394,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -438,7 +442,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) @@ -482,7 +486,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - block, + &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), &custody_columns, publish_fn, ) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index df093d61429..50de3eedfa9 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -284,7 +284,7 @@ pub fn blobs_to_data_column_sidecars( #[instrument(skip_all, level = "debug", fields(blob_count = blobs_and_proofs.len()))] pub fn blobs_to_partial_data_columns( blobs_and_proofs: Vec, &[KzgProof])>>, - block: &SignedBeaconBlock, + header: &PartialDataColumnHeader, kzg: &Kzg, spec: &ChainSpec, ) -> Result>, DataColumnSidecarError> { @@ -292,14 +292,6 @@ pub fn blobs_to_partial_data_columns( return Ok(vec![]); } - let kzg_commitments = block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_err| DataColumnSidecarError::PreDeneb)?; - let kzg_commitments_inclusion_proof = block.message().body().kzg_commitments_merkle_proof()?; - let signed_block_header = block.signed_block_header(); - let blob_cells_and_proofs_vec = blobs_and_proofs .into_par_iter() .map(|maybe_blob_and_proofs| { @@ -324,14 +316,8 @@ pub fn blobs_to_partial_data_columns( }) .collect::, KzgError>>()?; - build_partial_data_columns( - kzg_commitments.clone(), - kzg_commitments_inclusion_proof, - signed_block_header, - blob_cells_and_proofs_vec, - spec, - ) - .map_err(DataColumnSidecarError::BuildSidecarFailed) + build_partial_data_columns(header, blob_cells_and_proofs_vec, spec) + .map_err(DataColumnSidecarError::BuildSidecarFailed) } pub fn compute_cells(blobs: &[&Blob], kzg: &Kzg) -> Result, KzgError> { @@ -418,16 +404,13 @@ pub(crate) fn build_data_column_sidecars( } pub(crate) fn build_partial_data_columns( - kzg_commitments: KzgCommitments, - kzg_commitments_inclusion_proof: FixedVector, - signed_block_header: SignedBeaconBlockHeader, + header: &PartialDataColumnHeader, blob_cells_and_proofs_vec: Vec>, spec: &ChainSpec, ) -> Result>, String> { let number_of_columns = E::number_of_columns(); - let max_blobs_per_block = spec - .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) - as usize; + let max_blobs_per_block = + spec.max_blobs_per_block(header.slot().epoch(E::slots_per_epoch())) as usize; let mut bitmap = CellBitmap::::with_capacity(blob_cells_and_proofs_vec.len()).map_err(|_| { format!( @@ -481,7 +464,7 @@ pub(crate) fn build_partial_data_columns( .enumerate() .map(|(index, (col, proofs))| { let column = PartialDataColumn { - block_root: signed_block_header.message.canonical_root(), + block_root: header.signed_block_header.message.canonical_root(), index: index as u64, sidecar: types::partial_data_column_sidecar::PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), @@ -489,11 +472,7 @@ pub(crate) fn build_partial_data_columns( .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - header: VariableList::repeat_full(PartialDataColumnHeader { - kzg_commitments: kzg_commitments.clone(), - signed_block_header: signed_block_header.clone(), - kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), - }), + header: VariableList::repeat_full(header.clone()), }, }; Ok(column) @@ -573,16 +552,9 @@ pub fn reconstruct_blobs( let blob = Blob::::new(blob_bytes).map_err(|e| format!("{e:?}"))?; let kzg_proof = KzgProof::empty(); - BlobSidecar::::new_with_existing_proof( - row_index, - blob, - signed_block, - first_data_column.signed_block_header.clone(), - &first_data_column.kzg_commitments_inclusion_proof, - kzg_proof, - ) - .map(Arc::new) - .map_err(|e| format!("{e:?}")) + BlobSidecar::::new_with_existing_proof(row_index, blob, signed_block, kzg_proof) + .map(Arc::new) + .map_err(|e| format!("{e:?}")) }) .collect::, _>>()?; diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index ddec65c5619..f207c34ff4f 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -17,12 +17,15 @@ pub struct PartialDataColumnAssembler { /// Tracks partial columns being assembled for a single block struct PartialAssembly { header: PartialDataColumnHeader, + has_local_blobs: bool, /// Map of column_index -> partial column being assembled columns: HashMap>>, } /// Result of merging a partial column pub struct PartialMergeResult { + /// Have local blobs been added yet + pub local_blobs: bool, /// Merge that completed the column pub full_columns: Vec>>, /// The updated partials for publishing @@ -36,26 +39,30 @@ impl PartialDataColumnAssembler { } } - pub fn init(&self, block_root: Hash256, header: T) -> Result + /// Returns true unless the header was already contained or the passed argument failed to convert to a header + pub fn init(&self, block_root: Hash256, header: T) -> bool where T: TryInto>, { let mut assemblies = self.assemblies.write(); if assemblies.contains(&block_root) { - return Ok(false); + return false; } - let header = header.try_into()?; + let Ok(header) = header.try_into() else { + return false; + }; let assembly = PartialAssembly { header, + has_local_blobs: false, columns: HashMap::new(), }; assemblies.put(block_root, assembly); - Ok(true) + true } /// Merge a received partial column into the assembly. @@ -64,6 +71,7 @@ impl PartialDataColumnAssembler { &self, block_root: Hash256, partials: Vec>, + local_blobs: bool, ) -> Option> { let mut assemblies = self.assemblies.write(); let assembly = assemblies @@ -74,6 +82,7 @@ impl PartialDataColumnAssembler { .next() .map(|header| PartialAssembly { header: header.clone(), + has_local_blobs: local_blobs, columns: HashMap::new(), }) .ok_or(()) @@ -112,7 +121,12 @@ impl PartialDataColumnAssembler { updated_partials.push(merged); } + if local_blobs { + assembly.has_local_blobs = true; + } + Some(PartialMergeResult { + local_blobs: assembly.has_local_blobs, full_columns, updated_partials, }) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 3b8a4b267f0..48fe97ae01e 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -58,7 +58,7 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use types::partial_data_column_sidecar::PartialDataColumn; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -792,19 +792,15 @@ impl NetworkBeaconProcessor { mut column: PartialDataColumn, seen_duration: Duration, ) { - let slot = if let Some(header) = column.sidecar.header.first() { + let mut get_blobs = false; + let header = if let Some(header) = column.sidecar.header.first() { // Header was sent, so it is required to be valid match self .chain .verify_partial_data_column_header_for_gossip(column.block_root, header.clone()) { Ok(verified) => { - if !verified.was_cached() { - self.chain - .data_availability_checker - .put_partial_data_column_header(verified.into_inner()) - // TODO(dknopik): trigger getBlobsV3 - } + get_blobs = !verified.was_cached(); } Err(err) => { self.handle_partial_verification_error( @@ -816,7 +812,7 @@ impl NetworkBeaconProcessor { return; } } - header.signed_block_header.message.slot + header.clone() } else { // There is no header, so we check if we have a cached one to use let Some(header) = self @@ -828,13 +824,13 @@ impl NetworkBeaconProcessor { error!(block=%column.block_root, index=%column.index, "Received partial column while not having header stored"); return; }; - let slot = header.signed_block_header.message.slot; column.sidecar.header = VariableList::repeat_full((*header).clone()); - slot + (*header).clone() }; let block_root = column.block_root; let index = column.index; + let slot = header.slot(); let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. metrics::observe_duration( @@ -883,6 +879,13 @@ impl NetworkBeaconProcessor { self.handle_partial_verification_error(peer_id, err, block_root, index); } } + + if get_blobs { + // We want to publish immediately when this finishes + let publish_blobs = true; + self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) + .await + } } fn handle_partial_verification_error( @@ -1396,9 +1399,11 @@ impl NetworkBeaconProcessor { } } - self.send_network_message(NetworkMessage::PublishPartial { - columns: merge_result.updated_partials, - }); + if merge_result.local_blobs || self.chain.config.disable_get_blobs { + self.send_network_message(NetworkMessage::PublishPartial { + columns: merge_result.updated_partials, + }); + } // Check if block is now fully available // This is handled in process_gossip_partial_data_column @@ -1783,9 +1788,11 @@ impl NetworkBeaconProcessor { let current_span = Span::current(); self.executor.spawn( async move { - self_clone - .fetch_engine_blobs_and_publish(block_clone, block_root, publish_blobs) - .await + if let Ok(header) = PartialDataColumnHeader::try_from(block_clone.as_ref()) { + self_clone + .fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) + .await + } } .instrument(current_span), "fetch_blobs_gossip", diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index c8a1f02f00e..d8ade9ccc7b 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -28,7 +28,7 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; use tracing::{debug, error, instrument, trace, warn}; -use types::partial_data_column_sidecar::PartialDataColumn; +use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -775,14 +775,14 @@ impl NetworkBeaconProcessor { pub async fn fetch_engine_blobs_and_publish( self: &Arc, - block: Arc>>, + header: &PartialDataColumnHeader, block_root: Hash256, publish_blobs: bool, ) { if self.chain.config.disable_get_blobs { return; } - let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); + let epoch = header.slot().epoch(T::EthSpec::slots_per_epoch()); let custody_columns = self.chain.sampling_columns_for_epoch(epoch); let self_cloned = self.clone(); let publish_fn = move |blobs_or_data_column| { @@ -811,7 +811,7 @@ impl NetworkBeaconProcessor { match fetch_and_process_engine_blobs( self.chain.clone(), block_root, - block.clone(), + header, custody_columns, publish_fn, ) diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index a96c2778a8b..e53abceaafd 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -33,6 +33,7 @@ use store::KzgCommitment; use tracing::{debug, debug_span, error, info, instrument, warn}; use types::data::FixedBlobSidecarList; use types::kzg_ext::format_kzg_commitments; +use types::partial_data_column_sidecar::PartialDataColumnHeader; use types::{BlockImportSource, DataColumnSidecarList, Epoch, Hash256}; /// Id associated to a batch processing request, either a sync batch or a parent lookup. @@ -238,9 +239,12 @@ impl NetworkBeaconProcessor { // Block is valid, we can now attempt fetching blobs from EL using version hashes // derived from kzg commitments from the block, without having to wait for all blobs // to be sent from the peers if we already have them. - let publish_blobs = false; - self.fetch_engine_blobs_and_publish(signed_beacon_block, block_root, publish_blobs) - .await + if let Ok(header) = PartialDataColumnHeader::try_from(signed_beacon_block.as_ref()) + { + let publish_blobs = false; + self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) + .await + } } _ => {} } diff --git a/consensus/types/src/block/beacon_block_body.rs b/consensus/types/src/block/beacon_block_body.rs index a113f85fd31..2f381cf7fb1 100644 --- a/consensus/types/src/block/beacon_block_body.rs +++ b/consensus/types/src/block/beacon_block_body.rs @@ -3,14 +3,14 @@ use std::marker::PhantomData; use bls::Signature; use context_deserialize::{ContextDeserialize, context_deserialize}; use educe::Educe; -use merkle_proof::{MerkleTree, MerkleTreeError}; +use merkle_proof::MerkleTree; use metastruct::metastruct; use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use superstruct::superstruct; use test_random_derive::TestRandom; -use tree_hash::{BYTES_PER_CHUNK, TreeHash}; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use crate::{ @@ -18,6 +18,7 @@ use crate::{ attestation::{ AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut, PayloadAttestation, }, + complete_kzg_commitment_merkle_proof, core::{EthSpec, Graffiti, Hash256}, deposit::Deposit, execution::{ @@ -270,46 +271,11 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, | Self::Capella(_) | Self::Gloas(_) => Err(BeaconStateError::IncorrectStateVariant), Self::Deneb(_) | Self::Electra(_) | Self::Fulu(_) => { - // We compute the branches by generating 2 merkle trees: - // 1. Merkle tree for the `blob_kzg_commitments` List object - // 2. Merkle tree for the `BeaconBlockBody` container - // We then merge the branches for both the trees all the way up to the root. - - // Part1 (Branches for the subtree rooted at `blob_kzg_commitments`) - // - // Branches for `blob_kzg_commitments` without length mix-in - let blob_leaves = self - .blob_kzg_commitments()? - .iter() - .map(|commitment| commitment.tree_hash_root()) - .collect::>(); - let depth = E::max_blob_commitments_per_block() - .next_power_of_two() - .ilog2(); - let tree = MerkleTree::create(&blob_leaves, depth as usize); - let (_, mut proof) = tree - .generate_proof(index, depth as usize) - .map_err(BeaconStateError::MerkleTreeError)?; - - // Add the branch corresponding to the length mix-in. - let length = blob_leaves.len(); - let usize_len = std::mem::size_of::(); - let mut length_bytes = [0; BYTES_PER_CHUNK]; - length_bytes - .get_mut(0..usize_len) - .ok_or(BeaconStateError::MerkleTreeError( - MerkleTreeError::PleaseNotifyTheDevs, - ))? - .copy_from_slice(&length.to_le_bytes()); - let length_root = Hash256::from_slice(length_bytes.as_slice()); - proof.push(length_root); - - // Part 2 - // Branches for `BeaconBlockBody` container - // Join the proofs for the subtree and the main tree - proof.extend_from_slice(kzg_commitments_proof); - - Ok(FixedVector::new(proof)?) + complete_kzg_commitment_merkle_proof::( + self.blob_kzg_commitments()?, + index, + kzg_commitments_proof, + ) } } } diff --git a/consensus/types/src/data/blob_sidecar.rs b/consensus/types/src/data/blob_sidecar.rs index 709e556933b..d627ebc55de 100644 --- a/consensus/types/src/data/blob_sidecar.rs +++ b/consensus/types/src/data/blob_sidecar.rs @@ -15,13 +15,14 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use crate::partial_data_column_sidecar::PartialDataColumnHeader; use crate::{ block::{ BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlock, SignedBeaconBlockHeader, }, + complete_kzg_commitment_merkle_proof, core::{ChainSpec, Epoch, EthSpec, Hash256, Slot}, data::Blob, - execution::AbstractExecPayload, fork::ForkName, kzg_ext::KzgProofs, state::BeaconStateError, @@ -140,33 +141,29 @@ impl BlobSidecar { }) } - pub fn new_with_existing_proof>( + pub fn new_with_existing_proof>>( index: usize, blob: Blob, - signed_block: &SignedBeaconBlock, - signed_block_header: SignedBeaconBlockHeader, - kzg_commitments_inclusion_proof: &[Hash256], + header: T, kzg_proof: KzgProof, ) -> Result { - let expected_kzg_commitments = signed_block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_e| BlobSidecarError::PreDeneb)?; - let kzg_commitment = *expected_kzg_commitments + let header = header.try_into().map_err(|_| BlobSidecarError::PreDeneb)?; + let kzg_commitment = *header + .kzg_commitments .get(index) .ok_or(BlobSidecarError::MissingKzgCommitment)?; - let kzg_commitment_inclusion_proof = signed_block - .message() - .body() - .complete_kzg_commitment_merkle_proof(index, kzg_commitments_inclusion_proof)?; + let kzg_commitment_inclusion_proof = complete_kzg_commitment_merkle_proof::( + &header.kzg_commitments, + index, + &header.kzg_commitments_inclusion_proof, + )?; Ok(Self { index: index as u64, blob, kzg_commitment, kzg_proof, - signed_block_header, + signed_block_header: header.signed_block_header, kzg_commitment_inclusion_proof, }) } diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 2f8ed628ba9..699dc34a88e 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,7 +1,8 @@ use crate::test_utils::TestRandom; use crate::{ - BLOB_KZG_COMMITMENTS_INDEX, BeaconStateError, Cell, ColumnIndex, DataColumnSidecar, EthSpec, - Hash256, KzgCommitments, SignedBeaconBlock, SignedBeaconBlockHeader, + AbstractExecPayload, BLOB_KZG_COMMITMENTS_INDEX, BeaconStateError, Cell, ColumnIndex, + DataColumnSidecar, EthSpec, Hash256, KzgCommitments, SignedBeaconBlock, + SignedBeaconBlockHeader, Slot, }; use educe::Educe; use kzg::KzgProof; @@ -197,6 +198,10 @@ pub struct PartialDataColumnHeader { } impl PartialDataColumnHeader { + pub fn slot(&self) -> Slot { + self.signed_block_header.message.slot + } + pub fn verify_inclusion_proof(&self) -> bool { let blob_kzg_commitments_root = self.kzg_commitments.tree_hash_root(); @@ -210,10 +215,12 @@ impl PartialDataColumnHeader { } } -impl TryFrom<&SignedBeaconBlock> for PartialDataColumnHeader { +impl> TryFrom<&SignedBeaconBlock> + for PartialDataColumnHeader +{ type Error = BeaconStateError; - fn try_from(block: &SignedBeaconBlock) -> Result { + fn try_from(block: &SignedBeaconBlock) -> Result { Ok(Self { kzg_commitments: block.message().body().blob_kzg_commitments()?.clone(), signed_block_header: block.signed_block_header(), diff --git a/consensus/types/src/kzg_ext/mod.rs b/consensus/types/src/kzg_ext/mod.rs index 63533ec71f5..a4cc212697e 100644 --- a/consensus/types/src/kzg_ext/mod.rs +++ b/consensus/types/src/kzg_ext/mod.rs @@ -2,9 +2,11 @@ pub mod consts; pub use kzg::{Blob as KzgBlob, Error as KzgError, Kzg, KzgCommitment, KzgProof}; -use ssz_types::VariableList; - use crate::core::EthSpec; +use crate::{BeaconStateError, Hash256}; +use merkle_proof::{MerkleTree, MerkleTreeError}; +use ssz_types::{FixedVector, VariableList}; +use tree_hash::{BYTES_PER_CHUNK, TreeHash}; // Note on List limit: // - Deneb to Electra: `MaxBlobCommitmentsPerBlock` @@ -25,3 +27,49 @@ pub fn format_kzg_commitments(commitments: &[KzgCommitment]) -> String { let surrounded_commitments = format!("[{}]", commitments_joined); surrounded_commitments } + +pub fn complete_kzg_commitment_merkle_proof( + kzg_commitments: &KzgCommitments, + index: usize, + kzg_commitments_proof: &[Hash256], +) -> Result, BeaconStateError> { + // We compute the branches by generating 2 merkle trees: + // 1. Merkle tree for the `blob_kzg_commitments` List object + // 2. Merkle tree for the `BeaconBlockBody` container + // We then merge the branches for both the trees all the way up to the root. + + // Part1 (Branches for the subtree rooted at `blob_kzg_commitments`) + // + // Branches for `blob_kzg_commitments` without length mix-in + let blob_leaves = kzg_commitments + .iter() + .map(|commitment| commitment.tree_hash_root()) + .collect::>(); + let depth = E::max_blob_commitments_per_block() + .next_power_of_two() + .ilog2(); + let tree = MerkleTree::create(&blob_leaves, depth as usize); + let (_, mut proof) = tree + .generate_proof(index, depth as usize) + .map_err(BeaconStateError::MerkleTreeError)?; + + // Add the branch corresponding to the length mix-in. + let length = blob_leaves.len(); + let usize_len = std::mem::size_of::(); + let mut length_bytes = [0; BYTES_PER_CHUNK]; + length_bytes + .get_mut(0..usize_len) + .ok_or(BeaconStateError::MerkleTreeError( + MerkleTreeError::PleaseNotifyTheDevs, + ))? + .copy_from_slice(&length.to_le_bytes()); + let length_root = Hash256::from_slice(length_bytes.as_slice()); + proof.push(length_root); + + // Part 2 + // Branches for `BeaconBlockBody` container + // Join the proofs for the subtree and the main tree + proof.extend_from_slice(kzg_commitments_proof); + + Ok(FixedVector::new(proof)?) +} diff --git a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs index cf7b5df891a..47252205183 100644 --- a/consensus/types/src/test_utils/generate_random_block_and_blobs.rs +++ b/consensus/types/src/test_utils/generate_random_block_and_blobs.rs @@ -100,20 +100,8 @@ mod test { .. } = blob_sidecars.pop().unwrap(); - // Compute the commitments inclusion proof and use it for building blob sidecar. - let (signed_block_header, kzg_commitments_inclusion_proof) = block - .signed_block_header_and_kzg_commitments_proof() - .unwrap(); - - let blob_sidecar = BlobSidecar::new_with_existing_proof( - index as usize, - blob, - &block, - signed_block_header, - &kzg_commitments_inclusion_proof, - kzg_proof, - ) - .unwrap(); + let blob_sidecar = + BlobSidecar::new_with_existing_proof(index as usize, blob, &block, kzg_proof).unwrap(); assert!(blob_sidecar.verify_blob_sidecar_inclusion_proof()); } From 9e4985436c7b48501b4ffbd373396f63d1e534bf Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 22 Jan 2026 03:09:40 +0100 Subject: [PATCH 068/146] Implement 3 of 4 missing metrics --- beacon_node/beacon_chain/src/beacon_chain.rs | 20 ++++++++++++++++++- beacon_node/beacon_chain/src/metrics.rs | 20 ++++++++----------- .../src/partial_data_column_assembler.rs | 9 +++++++++ .../gossip_methods.rs | 2 ++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1481cd820d4..35419a8b837 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3132,6 +3132,13 @@ impl BeaconChain { { let block_root = verified_partial.block_root(); let slot = verified_partial.slot(); + let partial = verified_partial.to_data_column(); + let index_str = partial.index.to_string(); + metrics::inc_counter_vec_by( + &metrics::BEACON_PARTIAL_MESSAGE_CELLS_RECEIVED_TOTAL, + &[index_str.as_str()], + partial.sidecar.column.len() as u64, + ); // If this block has already been imported to forkchoice it must have been available if self @@ -3143,15 +3150,26 @@ impl BeaconChain { } // Merge the partial into the assembler - let partial = verified_partial.clone_data_column(); let merge_result = self .data_availability_checker .partial_assembler() .merge_partials(block_root, vec![partial], false) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; + metrics::inc_counter_vec_by( + &metrics::BEACON_PARTIAL_MESSAGE_USEFUL_CELLS_TOTAL, + &[index_str.as_str()], + merge_result.added_cells as u64, + ); + // If we completed a column, process it through the DA checker if !merge_result.full_columns.is_empty() { + metrics::inc_counter_vec_by( + &metrics::BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL, + &[index_str.as_str()], + merge_result.full_columns.len() as u64, + ); + let availability = self .data_availability_checker .put_full_data_columns(block_root, merge_result.full_columns.clone())?; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 0978ba77def..9657a9838d8 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1807,9 +1807,8 @@ pub static BEACON_PARTIAL_MESSAGE_USEFUL_CELLS_TOTAL: LazyLock { /// Result of merging a partial column pub struct PartialMergeResult { + /// How many cells were added to the store + pub added_cells: usize, /// Have local blobs been added yet pub local_blobs: bool, /// Merge that completed the column @@ -91,6 +93,7 @@ impl PartialDataColumnAssembler { let mut full_columns = Vec::new(); let mut updated_partials = Vec::new(); + let mut added_cells = 0; for partial in partials { let column_index = partial.index; @@ -100,12 +103,17 @@ impl PartialDataColumnAssembler { let Some(merged_sidecar) = existing.sidecar.merge(&partial.sidecar) else { continue; }; + added_cells += merged_sidecar + .column + .len() + .saturating_sub(partial.sidecar.column.len()); PartialDataColumn { block_root: existing.block_root, index: existing.index, sidecar: merged_sidecar, } } else { + added_cells += partial.sidecar.column.len(); // First time seeing this column index for this block partial }; @@ -126,6 +134,7 @@ impl PartialDataColumnAssembler { } Some(PartialMergeResult { + added_cells, local_blobs: assembly.has_local_blobs, full_columns, updated_partials, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 48fe97ae01e..e13e54929f8 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1258,6 +1258,8 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); + // TODO(dknopik): import into assembly and track BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL metric + let result = self .chain .process_gossip_data_columns(vec![verified_data_column], || Ok(())) From 821e7d215a14f3448e322e9f712a04f90d38f2e3 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 22 Jan 2026 03:46:40 +0100 Subject: [PATCH 069/146] add version byte prefix to group id --- beacon_node/lighthouse_network/src/types/partial.rs | 7 +++++-- beacon_node/lighthouse_network/src/types/pubsub.rs | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index afc62125519..97617aac9e6 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -6,9 +6,9 @@ use ssz_types::VariableList; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; -use types::EthSpec; use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; use types::partial_data_column_sidecar::PartialDataColumnSidecar; +use types::{EthSpec, Hash256}; pub type HeaderSentSet = Arc>>; @@ -102,7 +102,10 @@ impl From> for CellBitmapMetadata { impl Partial for OutgoingPartialColumn { fn group_id(&self) -> Vec { - self.partial_column.block_root.as_slice().to_vec() + let mut group_id = Vec::with_capacity(Hash256::len_bytes() + 1); + group_id.push(0); + group_id.extend_from_slice(self.partial_column.block_root.as_slice()); + group_id } fn metadata(&self) -> Vec { diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 472cefe5a9e..96ccca6e1b7 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -427,7 +427,10 @@ pub fn decode_partial( ) -> Result, String> { match topic.kind() { GossipKind::DataColumnSidecar(id) => { - let block_root = Hash256::from_ssz_bytes(group) + if group.first() != Some(&0) { + return Err(format!("Unknown data column format: {:?}", group.first())); + } + let block_root = Hash256::from_ssz_bytes(&group[1..]) .map_err(|e| format!("Error decoding group: {:?}", e))?; let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; From a4526af4d4e0a5cce291860179b292e9f6db06a7 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 12:01:35 +0100 Subject: [PATCH 070/146] update libp2p --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c202fba9504..32e62a0d544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4943,7 +4943,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "bytes", "either", @@ -4974,7 +4974,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4984,7 +4984,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4994,7 +4994,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "either", "fnv", @@ -5018,7 +5018,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "async-trait", "futures", @@ -5033,7 +5033,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5053,7 +5053,7 @@ dependencies = [ "prometheus-client", "quick-protobuf", "quick-protobuf-codec", - "rand 0.8.5", + "rand 0.9.2", "regex", "sha2", "tracing", @@ -5063,7 +5063,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "asynchronous-codec", "either", @@ -5103,7 +5103,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "hickory-proto", @@ -5121,7 +5121,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "libp2p-core", @@ -5137,7 +5137,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "asynchronous-codec", "bytes", @@ -5155,7 +5155,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "asynchronous-codec", "bytes", @@ -5177,7 +5177,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "futures-timer", @@ -5198,7 +5198,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "heck", "quote", @@ -5229,7 +5229,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "futures-timer", @@ -5244,7 +5244,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "futures-rustls", @@ -5262,7 +5262,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "futures-timer", @@ -5276,7 +5276,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "either", "futures", @@ -5972,7 +5972,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "bytes", "futures", @@ -7085,7 +7085,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "asynchronous-codec", "bytes", @@ -7761,7 +7761,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bd02db8b960158883545e6c9137c99e61ab8503e" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" dependencies = [ "futures", "pin-project", From d9726ffd375b4f8c6ca2ed22254f946a2e1179c2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 13:20:06 +0100 Subject: [PATCH 071/146] add logging on send --- .../lighthouse_network/src/types/partial.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 97617aac9e6..18a5fe0d634 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -6,6 +6,7 @@ use ssz_types::VariableList; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; +use tracing::debug; use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; use types::partial_data_column_sidecar::PartialDataColumnSidecar; use types::{EthSpec, Hash256}; @@ -129,6 +130,13 @@ impl Partial for OutgoingPartialColumn { Box::new(CellBitmapMetadata::::Unknown) as Box, ) }); + debug!( + peer=%peer_id, + group_id=%self.partial_column.block_root, + column_index=self.partial_column.index, + sending_header=send.is_some(), + "Partial send: No metadata" + ); Ok(PartialAction { need: false, send }) } @@ -146,6 +154,14 @@ impl Partial for OutgoingPartialColumn { .sidecar .with_missing_cells(&peer_has) .map(|sidecar| { + debug!( + peer=%peer_id, + group_id=%self.partial_column.block_root, + column_index=self.partial_column.index, + metadata=%peer_has, + sending=%sidecar.cells_present_bitmap, + "Partial send: Sending" + ); ( sidecar.as_ssz_bytes(), Box::new(CellBitmapMetadata::::from( @@ -154,6 +170,16 @@ impl Partial for OutgoingPartialColumn { ) }); + if send.is_none() { + debug!( + peer=%peer_id, + group_id=%self.partial_column.block_root, + column_index=self.partial_column.index, + metadata=%peer_has, + "Partial send: Nothing to send" + ); + } + Ok(PartialAction { need, send }) } } From b9e09a92698ef759c51ae56f395110d79c9ee9bc Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 14:47:16 +0100 Subject: [PATCH 072/146] Improve logging --- beacon_node/lighthouse_network/src/service/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 57ba0571ea3..972161946b8 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -875,6 +875,12 @@ impl Network { "Attempted to publish duplicate message" ); } + PublishError::NoPeersSubscribedToTopic if topic.kind().supports_partial_messages() => { + debug!( + kind = %topic.kind(), + "No peers supporting full messages" + ); + } ref e => { warn!( error = ?e, From c817af7cb7e4a9999dcbe8abc26a394bfa4b1682 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 14:47:54 +0100 Subject: [PATCH 073/146] fix metric name --- beacon_node/beacon_chain/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 9657a9838d8..cc8452872a3 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1657,7 +1657,7 @@ pub static PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_REQUESTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( - "beacon_partial_data_column_sidecar_header_processing_successes_total", + "beacon_partial_data_column_sidecar_header_processing_dupes_total", "Number of partial data column sidecars verified for gossip (excluding dupes)", ) }); From 096fb782c062c0002b9922e2745caeed072c3df6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 14:47:54 +0100 Subject: [PATCH 074/146] avoid completing already completed columsn --- .../beacon_chain/src/partial_data_column_assembler.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 5329b77695b..be69adabc3e 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -99,6 +99,11 @@ impl PartialDataColumnAssembler { let column_index = partial.index; let merged = if let Some(existing) = assembly.columns.get(&column_index) { + // Check if the column is already completed + if existing.sidecar.is_complete() { + continue; + } + // Merge with existing partial let Some(merged_sidecar) = existing.sidecar.merge(&partial.sidecar) else { continue; From eb0ac14dd0274dc49d59bdec27904902af5f2808 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 19:35:46 +0100 Subject: [PATCH 075/146] fix: always publish partials after attempted getblobsv2/v3 and avoid useless republishs --- beacon_node/beacon_chain/src/beacon_chain.rs | 15 ++---------- .../beacon_chain/src/fetch_blobs/mod.rs | 19 ++++++--------- .../beacon_chain/src/fetch_blobs/tests.rs | 21 +++++++++-------- .../src/partial_data_column_assembler.rs | 23 ++++++++++++++++++- .../src/network_beacon_processor/mod.rs | 18 +++++++++------ 5 files changed, 53 insertions(+), 43 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a6ab6e1fc75..c19122f6b90 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3248,15 +3248,12 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CompleteColumns(columns) => { + EngineGetBlobsOutput::CustodyColumns(columns) => { self.emit_sse_data_column_sidecar_events( &block_root, columns.iter().map(|column| column.as_ref()), ); } - EngineGetBlobsOutput::PartialColumns(_) => { - // Partials don't emit SSE events until they complete - } } self.check_engine_blobs_availability_and_import(slot, block_root, engine_get_blobs_output) @@ -3737,7 +3734,7 @@ impl BeaconChain { self.data_availability_checker .put_kzg_verified_blobs(block_root, blobs)? } - EngineGetBlobsOutput::CompleteColumns(data_columns) => { + EngineGetBlobsOutput::CustodyColumns(data_columns) => { self.check_data_column_sidecar_header_signature_and_slashability( block_root, data_columns.iter().map(|c| c.as_ref()), @@ -3745,14 +3742,6 @@ impl BeaconChain { self.data_availability_checker .put_full_data_columns(block_root, data_columns)? } - EngineGetBlobsOutput::PartialColumns(_partials) => { - // Partials have already been initialized in the assembler during fetch_and_process - // They don't go through put_kzg_verified - they're assembled via gossip - // Return MissingComponents to indicate we're waiting for more data - return Ok(AvailabilityProcessingStatus::MissingComponents( - slot, block_root, - )); - } }; self.process_availability(slot, availability, || Ok(())) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index b6acbf2943f..907b455121c 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -34,7 +34,7 @@ use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_h use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; use types::data::{BlobSidecarError, DataColumnSidecarError}; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; +use types::partial_data_column_sidecar::PartialDataColumnHeader; use types::{BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, Hash256, VersionedHash}; /// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the @@ -43,10 +43,8 @@ use types::{BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, Hash256, Versio #[derive(Debug)] pub enum EngineGetBlobsOutput { Blobs(Vec>), - /// Complete columns ready to be imported into the `DataAvailabilityChecker`. - CompleteColumns(Vec>>), - /// Incomplete partial columns that need assembly via gossip - PartialColumns(Vec>>), + /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. + CustodyColumns(Vec>>), } #[derive(Debug)] @@ -343,15 +341,12 @@ async fn fetch_and_process_blobs_v2_or_v3( // Publish complete columns and incomplete partials if !merge_result.full_columns.is_empty() { - publish_fn(EngineGetBlobsOutput::CompleteColumns( + publish_fn(EngineGetBlobsOutput::CustodyColumns( merge_result.full_columns.clone(), )); } - if !merge_result.updated_partials.is_empty() { - publish_fn(EngineGetBlobsOutput::PartialColumns( - merge_result.updated_partials, - )); - } + // We publish all partials at the calling site, regardless of result, as previous publishs + // have been blocked, waiting for the results of this call // Process complete columns through DA checker let availability_processing_status = if !merge_result.full_columns.is_empty() { @@ -359,7 +354,7 @@ async fn fetch_and_process_blobs_v2_or_v3( .process_engine_blobs( header.slot(), block_root, - EngineGetBlobsOutput::CompleteColumns(merge_result.full_columns), + EngineGetBlobsOutput::CustodyColumns(merge_result.full_columns), ) .await? } else { diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index b6ceee7173c..d1b34dac4a7 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -10,7 +10,6 @@ use eth2::types::BlobsBundle; use execution_layer::json_structures::{BlobAndProof, BlobAndProofV1, BlobAndProofV2}; use execution_layer::test_utils::generate_blobs; use maplit::hashset; -use std::mem; use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; use task_executor::test_utils::TestRuntime; @@ -227,8 +226,8 @@ mod get_blobs_v2 { let published_columns = extract_published_blobs(publish_fn_args); assert!( matches!( - &published_columns[..], - [EngineGetBlobsOutput::CompleteColumns(complete_columns), EngineGetBlobsOutput::PartialColumns(partial_columns)] if complete_columns.len() == custody_columns.len() && complete_columns.len() == partial_columns.len() + published_columns, + EngineGetBlobsOutput::CustodyColumns(columns) if columns.len() == custody_columns.len() ), "should publish custody columns" ); @@ -370,8 +369,8 @@ mod get_blobs_v1 { ); assert!( matches!( - &extract_published_blobs(publish_fn_args)[..], - [EngineGetBlobsOutput::Blobs(blobs)] if blobs.len() == blob_count - 1 + extract_published_blobs(publish_fn_args), + EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count - 1 ), "partial blob results should still be published" ); @@ -502,8 +501,8 @@ mod get_blobs_v1 { let published_blobs = extract_published_blobs(publish_fn_args); assert!( matches!( - &published_blobs[..], - [EngineGetBlobsOutput::Blobs(blobs)] if blobs.len() == blob_count + published_blobs, + EngineGetBlobsOutput::Blobs(blobs) if blobs.len() == blob_count ), "should publish fetched blobs" ); @@ -528,11 +527,13 @@ mod get_blobs_v1 { } } -/// Extract the `EngineGetBlobsOutput`s passed to the `publish_fn`. +/// Extract the `EngineGetBlobsOutput` passed to the `publish_fn`. fn extract_published_blobs( publish_fn_args: Arc>>>, -) -> Vec> { - mem::take(publish_fn_args.lock().unwrap().as_mut()) +) -> EngineGetBlobsOutput { + let mut calls = publish_fn_args.lock().unwrap(); + assert_eq!(calls.len(), 1); + calls.pop().unwrap() } fn mock_process_engine_blobs_result( diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index be69adabc3e..eb630dcdf1d 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -108,10 +108,18 @@ impl PartialDataColumnAssembler { let Some(merged_sidecar) = existing.sidecar.merge(&partial.sidecar) else { continue; }; - added_cells += merged_sidecar + + let adding_cells = merged_sidecar .column .len() .saturating_sub(partial.sidecar.column.len()); + + added_cells += adding_cells; + + if adding_cells == 0 { + continue; + } + PartialDataColumn { block_root: existing.block_root, index: existing.index, @@ -160,6 +168,19 @@ impl PartialDataColumnAssembler { .cloned() } + /// Get all current partials for a block + pub fn get_partials(&self, block_root: &Hash256) -> Option>>> { + Some( + self.assemblies + .read() + .peek(block_root)? + .columns + .values() + .cloned() + .collect(), + ) + } + /// Get header for a block if we have an active assembly pub fn get_header( &self, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index d8ade9ccc7b..0362d9ce301 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -794,16 +794,10 @@ impl NetworkBeaconProcessor { block_root, ); } - EngineGetBlobsOutput::CompleteColumns(columns) => { + EngineGetBlobsOutput::CustodyColumns(columns) => { // Gradually publish full columns self_cloned.publish_data_columns_gradually(columns, block_root); } - EngineGetBlobsOutput::PartialColumns(partials) => { - // Publish partial columns without eager send - self_cloned.send_network_message(NetworkMessage::PublishPartial { - columns: partials, - }) - } }; } }; @@ -855,6 +849,16 @@ impl NetworkBeaconProcessor { ); } } + + // Publish partial columns without eager send + if let Some(columns) = self + .chain + .data_availability_checker + .partial_assembler() + .get_partials(&block_root) + { + self.send_network_message(NetworkMessage::PublishPartial { columns }) + } } /// Attempts to reconstruct all data columns if the conditions checked in From 671249202107dfe0cf11f9f5ce62476816b4c9f3 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 19:39:50 +0100 Subject: [PATCH 076/146] fmt --- beacon_node/lighthouse_network/src/service/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 972161946b8..ff93de2ebb0 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -875,7 +875,9 @@ impl Network { "Attempted to publish duplicate message" ); } - PublishError::NoPeersSubscribedToTopic if topic.kind().supports_partial_messages() => { + PublishError::NoPeersSubscribedToTopic + if topic.kind().supports_partial_messages() => + { debug!( kind = %topic.kind(), "No peers supporting full messages" From 81cce2cefe0497ffcea66cfe8640e9b2b024d94a Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 27 Jan 2026 19:43:34 +0100 Subject: [PATCH 077/146] add moar logs --- .../network/src/network_beacon_processor/gossip_methods.rs | 3 +++ beacon_node/network/src/network_beacon_processor/mod.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 288a6c13fcb..e5328516966 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -877,6 +877,7 @@ impl NetworkBeaconProcessor { } if get_blobs { + debug!(block= %column.block_root, "Triggering getBlobs after receiving partial header"); // We want to publish immediately when this finishes let publish_blobs = true; self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) @@ -1401,6 +1402,8 @@ impl NetworkBeaconProcessor { self.send_network_message(NetworkMessage::PublishPartial { columns: merge_result.updated_partials, }); + } else { + debug!(block = %block_root, "Not publishing partials - waiting for getBlobs"); } // Check if block is now fully available diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 0362d9ce301..77179e92a6a 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -857,6 +857,7 @@ impl NetworkBeaconProcessor { .partial_assembler() .get_partials(&block_root) { + debug!(block = %block_root, "Publishing all partials after getBlobs"); self.send_network_message(NetworkMessage::PublishPartial { columns }) } } From 7fa4502219e68c5bb87ac315a6b2fe4d8b207fd1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 28 Jan 2026 19:26:22 +0100 Subject: [PATCH 078/146] fix compilation --- .../network/src/network_beacon_processor/gossip_methods.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e5328516966..e657e94a226 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -877,7 +877,7 @@ impl NetworkBeaconProcessor { } if get_blobs { - debug!(block= %column.block_root, "Triggering getBlobs after receiving partial header"); + debug!(block = %block_root, "Triggering getBlobs after receiving partial header"); // We want to publish immediately when this finishes let publish_blobs = true; self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) From f2863ca776083be05a756cb493d8cbae239e6310 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 2 Feb 2026 15:27:57 +0100 Subject: [PATCH 079/146] Cherry pick c3fb9eb0 for devnet retention fix --- consensus/types/src/core/chain_spec.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index a90a39a69d8..dda5241179d 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -810,14 +810,19 @@ impl ChainSpec { /// Returns the min epoch for blob / data column sidecar requests based on the current epoch. /// Switch to use the column sidecar config once the `blob_retention_epoch` has passed Fulu fork epoch. + /// Never uses the `blob_retention_epoch` for networks that started with Fulu enabled. pub fn min_epoch_data_availability_boundary(&self, current_epoch: Epoch) -> Option { let fork_epoch = self.deneb_fork_epoch?; let blob_retention_epoch = current_epoch.saturating_sub(self.min_epochs_for_blob_sidecars_requests); match self.fulu_fork_epoch { - Some(fulu_fork_epoch) if blob_retention_epoch > fulu_fork_epoch => Some( - current_epoch.saturating_sub(self.min_epochs_for_data_column_sidecars_requests), - ), + Some(fulu_fork_epoch) + if blob_retention_epoch > fulu_fork_epoch || fulu_fork_epoch == Epoch::new(0) => + { + Some( + current_epoch.saturating_sub(self.min_epochs_for_data_column_sidecars_requests), + ) + } _ => Some(std::cmp::max(fork_epoch, blob_retention_epoch)), } } From 6251407f4d6bb1785e26675da51b3d419e2238c6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 30 Jan 2026 15:41:41 +0100 Subject: [PATCH 080/146] update libp2p --- Cargo.lock | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19983a6de85..1a1c0ba977d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3055,7 +3055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4674,7 +4674,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4941,7 +4941,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "bytes", "either", @@ -4972,7 +4972,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4982,7 +4982,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4992,7 +4992,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "either", "fnv", @@ -5016,7 +5016,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "async-trait", "futures", @@ -5031,7 +5031,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5061,7 +5061,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "asynchronous-codec", "either", @@ -5101,7 +5101,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "hickory-proto", @@ -5119,7 +5119,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "libp2p-core", @@ -5135,7 +5135,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "asynchronous-codec", "bytes", @@ -5153,7 +5153,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "asynchronous-codec", "bytes", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "futures-timer", @@ -5196,7 +5196,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "either", "fnv", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "heck", "quote", @@ -5227,7 +5227,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "futures-timer", @@ -5242,7 +5242,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "futures-rustls", @@ -5260,7 +5260,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "futures-timer", @@ -5274,7 +5274,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "either", "futures", @@ -5966,7 +5966,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "bytes", "futures", @@ -6232,7 +6232,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7078,7 +7078,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "asynchronous-codec", "bytes", @@ -7140,7 +7140,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7648,7 +7648,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7754,7 +7754,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#9a49c4604f0844b6fb0e54e9c724b2f76bd41fb7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" dependencies = [ "futures", "pin-project", @@ -8713,7 +8713,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10058,7 +10058,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From c2a7c402841cf238fc7414de89e25b2d761fdae1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 6 Feb 2026 09:48:30 +0100 Subject: [PATCH 081/146] invert CLI param --- beacon_node/lighthouse_network/src/config.rs | 10 +++------- beacon_node/lighthouse_network/src/service/mod.rs | 15 ++++++++------- .../lighthouse_network/src/types/topics.rs | 9 ++++++--- beacon_node/src/cli.rs | 15 ++++----------- beacon_node/src/config.rs | 8 ++------ book/src/help_bn.md | 7 +++---- 6 files changed, 26 insertions(+), 38 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 890ea5b5891..8917e3da0e5 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -141,11 +141,8 @@ pub struct Config { /// Flag for advertising a fake CGC to peers for testing ONLY. pub advertise_false_custody_group_count: Option, - /// Whether to disable signaling partial message support. Implies disable_partial_messages_request - pub disable_partial_messages_support: bool, - - /// Whether to disable requesting partial messages - pub disable_partial_messages_request: bool, + /// Whether to enable partial data column support. + pub enable_partial_columns: bool, } impl Config { @@ -370,8 +367,7 @@ impl Default for Config { inbound_rate_limiter_config: None, idontwant_message_size_threshold: DEFAULT_IDONTWANT_MESSAGE_SIZE_THRESHOLD, advertise_false_custody_group_count: None, - disable_partial_messages_support: false, - disable_partial_messages_request: false, + enable_partial_columns: false, } } } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index ff93de2ebb0..c9589eea605 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -821,14 +821,13 @@ impl Network { .write() .insert(topic.clone()); - let config = &self.network_globals.config; - let partial = topic.kind().supports_partial_messages(); + let partial = topic + .kind() + .use_partial_messages(self.network_globals.config.as_ref()); let topic: Topic = topic.into(); - let subscribe_result = if partial && !config.disable_partial_messages_support { - let request_partials = !config.disable_partial_messages_request; - self.gossipsub_mut() - .subscribe_partial(&topic, request_partials) + let subscribe_result = if partial { + self.gossipsub_mut().subscribe_partial(&topic, true) } else { self.gossipsub_mut().subscribe(&topic) }; @@ -876,7 +875,9 @@ impl Network { ); } PublishError::NoPeersSubscribedToTopic - if topic.kind().supports_partial_messages() => + if topic + .kind() + .use_partial_messages(self.network_globals.config.as_ref()) => { debug!( kind = %topic.kind(), diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index c0f7bde0d2d..a82b07d6690 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -11,7 +11,7 @@ use types::{ sync_committee::SyncSubnetId, }; -use crate::Subnet; +use crate::{NetworkConfig, Subnet}; /// The gossipsub topic names. // These constants form a topic name of the form /TOPIC_PREFIX/TOPIC/ENCODING_POSTFIX @@ -178,8 +178,11 @@ pub enum GossipKind { } impl GossipKind { - pub fn supports_partial_messages(&self) -> bool { - matches!(self, GossipKind::DataColumnSidecar(_)) + pub fn use_partial_messages(&self, config: &NetworkConfig) -> bool { + match self { + GossipKind::DataColumnSidecar(_) => config.enable_partial_columns, + _ => false, + } } } diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 1bf6303b3ab..340140a162b 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -671,17 +671,10 @@ pub fn cli_app() -> Command { .display_order(0) ) .arg( - Arg::new("disable-partial-messages-request") - .long("disable-partial-messages-request") - .help("Do not request partial messages for data columns.") - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .display_order(0) - ) - .arg( - Arg::new("disable-partial-messages-support") - .long("disable-partial-messages-support") - .help("Do not support receiving partial messages for data columns.") + Arg::new("enable-partial-columns") + .long("enable-partial-columns") + .help("Enable partial messages for data columns. This can reduce the amount of \ + data sent over the network.") .action(ArgAction::SetTrue) .help_heading(FLAG_HEADER) .display_order(0) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 208fce98e99..a43bbca9074 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1492,12 +1492,8 @@ pub fn set_network_config( })?; } - if parse_flag(cli_args, "disable-partial-messages-request") { - config.disable_partial_messages_request = true; - } - - if parse_flag(cli_args, "disable-partial-messages-support") { - config.disable_partial_messages_support = true; + if parse_flag(cli_args, "enable-partial-columns") { + config.enable_partial_columns = true; } Ok(()) diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 9965938006c..735de613b3b 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -477,10 +477,6 @@ Flags: --disable-packet-filter Disables the discovery packet filter. Useful for testing in smaller networks - --disable-partial-messages-request - Do not request partial messages for data columns. - --disable-partial-messages-support - Do not support receiving partial messages for data columns. --disable-proposer-reorgs Do not attempt to reorg late blocks from other validators when proposing. @@ -496,6 +492,9 @@ Flags: Sets the local ENR IP address and port to match those set for lighthouse. Specifically, the IP address will be the value of --listen-address and the UDP port will be --discovery-port. + --enable-partial-columns + Enable partial messages for data columns. This can reduce the amount + of data sent over the network. --enable-private-discovery Lighthouse by default does not discover private IP addresses. Set this flag to enable connection attempts to local addresses. From 166bbfb2849604e474c72c361023fbc2fec62d03 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 30 Jan 2026 15:56:43 +0100 Subject: [PATCH 082/146] clean up --- beacon_node/beacon_chain/src/beacon_chain.rs | 88 +++---- .../src/data_availability_checker.rs | 44 +--- .../src/data_availability_checker/error.rs | 2 - .../src/data_column_verification.rs | 175 +++++++------- .../beacon_chain/src/fetch_blobs/mod.rs | 29 ++- .../beacon_chain/src/fetch_blobs/tests.rs | 6 +- beacon_node/beacon_chain/src/kzg_utils.rs | 26 ++- .../src/partial_data_column_assembler.rs | 66 +++--- beacon_node/beacon_processor/src/lib.rs | 7 +- .../src/scheduler/work_reprocessing_queue.rs | 117 +--------- .../lighthouse_network/src/service/mod.rs | 30 ++- .../lighthouse_network/src/types/partial.rs | 4 +- .../lighthouse_network/src/types/pubsub.rs | 13 +- .../gossip_methods.rs | 219 +++++++++++------- .../src/network_beacon_processor/mod.rs | 14 +- .../network_beacon_processor/sync_methods.rs | 18 +- beacon_node/network/src/router.rs | 5 +- beacon_node/network/src/service.rs | 5 +- consensus/types/src/data/blob_sidecar.rs | 3 +- .../types/src/data/data_column_sidecar.rs | 2 +- consensus/types/src/data/mod.rs | 5 +- 21 files changed, 368 insertions(+), 510 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c19122f6b90..bf99972ddf4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -27,7 +27,8 @@ use crate::data_availability_checker::{ }; use crate::data_column_verification::{ GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumnHeader, - KzgVerifiedPartialDataColumn, validate_partial_data_column_sidecar_for_gossip, + KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + validate_partial_data_column_sidecar_for_gossip, }; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; @@ -59,6 +60,7 @@ use crate::observed_block_producers::ObservedBlockProducers; use crate::observed_data_sidecars::ObservedDataSidecars; use crate::observed_operations::{ObservationOutcome, ObservedOperations}; use crate::observed_slashable::ObservedSlashable; +use crate::partial_data_column_assembler::PartialMergeResult; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::persisted_custody::persist_custody_context; use crate::persisted_fork_choice::PersistedForkChoice; @@ -136,7 +138,6 @@ use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace use tree_hash::TreeHash; use types::data::{ColumnIndex, FixedBlobSidecarList}; use types::execution::BlockProductionVersion; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -339,7 +340,6 @@ struct PartialBeaconBlock { pub enum BlockProcessStatus { /// Block is not in any pre-import cache. Block may be in the data-base or in the fork-choice. Unknown, - /// Only the header /// Block is currently processing but not yet validated. NotValidated(Arc>, BlockImportSource), /// Block is fully valid, but not yet imported. It's cached in the da_checker while awaiting @@ -544,6 +544,9 @@ impl FinalizationAndCanonicity { } } +type ProcessedPartialColumnStatus = + Option<(AvailabilityProcessingStatus, PartialMergeResult)>; + impl BeaconChain { /// Checks if a block is finalized. /// The finalization check is done with the block slot. The block root is used to verify that @@ -2204,7 +2207,9 @@ impl BeaconChain { data_column_header: PartialDataColumnHeader, ) -> Result, GossipDataColumnError> { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_REQUESTS); - let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + let _timer = metrics::start_timer( + &metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_GOSSIP_VERIFICATION_TIMES, + ); if let Some(cached_header) = self .data_availability_checker .partial_assembler() @@ -2222,7 +2227,9 @@ impl BeaconChain { GossipVerifiedPartialDataColumnHeader::new(block_root, data_column_header, self).inspect( |_| { - metrics::inc_counter(&metrics::DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); + metrics::inc_counter( + &metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_SUCCESSES, + ); }, ) } @@ -3127,11 +3134,10 @@ impl BeaconChain { pub async fn process_gossip_partial_data_column( self: &Arc, verified_partial: KzgVerifiedPartialDataColumn, - ) -> Result, BlockError> - { + slot: Slot, + ) -> Result, BlockError> { let block_root = verified_partial.block_root(); - let slot = verified_partial.slot(); - let partial = verified_partial.to_data_column(); + let partial = verified_partial.as_data_column(); let index_str = partial.index.to_string(); metrics::inc_counter_vec_by( &metrics::BEACON_PARTIAL_MESSAGE_CELLS_RECEIVED_TOTAL, @@ -3139,6 +3145,17 @@ impl BeaconChain { partial.sidecar.column.len() as u64, ); + // Check if we have custody of this column + let sampling_columns = self + .data_availability_checker + .custody_context() + .sampling_columns_for_epoch(slot.epoch(T::EthSpec::slots_per_epoch()), &self.spec); + let verified_partial = if sampling_columns.contains(&partial.index) { + KzgVerifiedCustodyPartialDataColumn::from_asserted_custody(verified_partial) + } else { + return Ok(None); + }; + // If this block has already been imported to forkchoice it must have been available if self .canonical_head @@ -3152,7 +3169,7 @@ impl BeaconChain { let merge_result = self .data_availability_checker .partial_assembler() - .merge_partials(block_root, vec![partial], false) + .merge_partials(block_root, vec![verified_partial], false) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; metrics::inc_counter_vec_by( @@ -3161,30 +3178,21 @@ impl BeaconChain { merge_result.added_cells as u64, ); - // If we completed a column, process it through the DA checker - if !merge_result.full_columns.is_empty() { - metrics::inc_counter_vec_by( - &metrics::BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL, - &[index_str.as_str()], - merge_result.full_columns.len() as u64, - ); + metrics::inc_counter_vec_by( + &metrics::BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL, + &[index_str.as_str()], + merge_result.full_columns.len() as u64, + ); - let availability = self - .data_availability_checker - .put_full_data_columns(block_root, merge_result.full_columns.clone())?; + let availability = self + .data_availability_checker + .put_kzg_verified_custody_data_columns(block_root, merge_result.full_columns.clone())?; - // If we achieved full availability, process it - if let Availability::Available(available_block) = availability { - self.process_availability( - slot, - Availability::Available(available_block), - || Ok(()), - ) - .await?; - } - } + let availability = self + .process_availability(slot, availability, || Ok(())) + .await?; - Ok(merge_result) + Ok(Some((availability, merge_result))) } /// Cache the blobs in the processing cache, process it, then evict it from the cache if it was @@ -3251,7 +3259,7 @@ impl BeaconChain { EngineGetBlobsOutput::CustodyColumns(columns) => { self.emit_sse_data_column_sidecar_events( &block_root, - columns.iter().map(|column| column.as_ref()), + columns.iter().map(|column| column.as_data_column()), ); } } @@ -3281,11 +3289,13 @@ impl BeaconChain { } } - fn emit_sse_data_column_sidecar_events<'a>( + fn emit_sse_data_column_sidecar_events<'a, I>( self: &Arc, block_root: &Hash256, - data_columns_iter: impl Iterator>, - ) { + data_columns_iter: I, + ) where + I: Iterator>, + { if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_data_column_sidecar_subscribers() { @@ -3656,8 +3666,8 @@ impl BeaconChain { publish_fn: impl FnOnce() -> Result<(), BlockError>, ) -> Result { if let Some(slasher) = self.slasher.as_ref() { - for header in data_columns.iter().map(|c| c.signed_block_header()) { - slasher.accept_block_header(header.clone()); + for data_colum in &data_columns { + slasher.accept_block_header(data_colum.signed_block_header()); } } @@ -3737,10 +3747,10 @@ impl BeaconChain { EngineGetBlobsOutput::CustodyColumns(data_columns) => { self.check_data_column_sidecar_header_signature_and_slashability( block_root, - data_columns.iter().map(|c| c.as_ref()), + data_columns.iter().map(|c| c.as_data_column()), )?; self.data_availability_checker - .put_full_data_columns(block_root, data_columns)? + .put_kzg_verified_custody_data_columns(block_root, data_columns)? } }; diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 48488b0588f..887dc2550a4 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -40,9 +40,7 @@ use crate::metrics::{ }; use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; -use types::DataColumnSidecar; -use types::new_non_zero_usize; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; +use types::{DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader, new_non_zero_usize}; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -216,7 +214,7 @@ impl DataAvailabilityChecker { // Check assembler for partial columns if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { // Compare: find which cells from incoming full column aren't in cached partial - return compare_full_to_partial(cell_count, cached_partial.as_ref()); + return compare_full_to_partial(cell_count, cached_partial.as_data_column()); } // No cached data, all cells are "missing" (new data we want) @@ -248,7 +246,7 @@ impl DataAvailabilityChecker { // Check assembler for partial columns if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { // Compare: find which cells from incoming partial aren't in cached partial - return compare_partial_to_partial(data_column, cached_partial.as_ref()); + return compare_partial_to_partial(data_column, cached_partial.as_data_column()); } // No cached data, return all present cells as "missing" (new data we want) @@ -410,39 +408,6 @@ impl DataAvailabilityChecker { .put_kzg_verified_data_columns(block_root, custody_columns) } - /// Put complete data columns directly into the DA checker (no assembly needed) - #[instrument(skip_all, level = "trace")] - pub fn put_full_data_columns( - &self, - block_root: Hash256, - data_columns: Vec>>, - ) -> Result, AvailabilityCheckError> { - // Convert to custody columns - filter by custody requirement - let Some(first_column) = data_columns.first() else { - // No columns to process - return missing components - return Ok(Availability::MissingComponents(block_root)); - }; - let epoch = first_column.slot().epoch(T::EthSpec::slots_per_epoch()); - - let sampling_columns = self - .custody_context - .sampling_columns_for_epoch(epoch, &self.spec); - - let custody_columns = data_columns - .into_iter() - .filter(|col| sampling_columns.contains(&col.index)) - .map(|c| { - // These are from EL, already KZG verified - KzgVerifiedCustodyDataColumn::from_asserted_custody( - KzgVerifiedDataColumn::from_execution_verified(c), - ) - }) - .collect::>(); - - self.availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) - } - /// Check if we have all the blobs for a block. Returns `Availability` which has information /// about whether all components have been received or more are required. pub fn put_executed_block( @@ -467,8 +432,7 @@ impl DataAvailabilityChecker { .put_pre_execution_block(block_root, block, source) } - /// Inserts a pre-execution block into the cache. - /// This does NOT override an existing executed block. + /// Inserts a partial data column header into the assembler. /// Returns true if this was the first insertion. pub fn put_partial_data_column_header( &self, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/error.rs b/beacon_node/beacon_chain/src/data_availability_checker/error.rs index a5448ece383..c9efb7a4149 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/error.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/error.rs @@ -21,7 +21,6 @@ pub enum Error { ParentStateMissing(Hash256), BlockReplayError(state_processing::BlockReplayError), RebuildingStateCaches(BeaconStateError), - PartialAssemblerError(BeaconStateError), SlotClockError, } @@ -45,7 +44,6 @@ impl Error { | Error::ParentStateMissing(_) | Error::BlockReplayError(_) | Error::RebuildingStateCaches(_) - | Error::PartialAssemblerError(_) | Error::SlotClockError => ErrorCategory::Internal, Error::InvalidBlobs { .. } | Error::InvalidColumn { .. } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 76e52fafb62..2b71ac0f8c2 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -18,11 +18,9 @@ use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; use tree_hash::TreeHash; -use types::data::ColumnIndex; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::{ - BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, - SignedBeaconBlockHeader, Slot, + BeaconStateError, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSubnetId, EthSpec, + Hash256, PartialDataColumn, PartialDataColumnHeader, SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip data column. @@ -219,21 +217,14 @@ impl GossipVerifiedDataColumn let header = column_sidecar.signed_block_header.clone(); // We only process slashing info if the gossip verification failed // since we do not process the data column any further in that case. - validate_data_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { - process_block_slash_info::<_, GossipDataColumnError>( - chain, - BlockSlashInfo::from_early_error_data_column(header.clone(), e), - ) - }) - } - - /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { - Self { - block_root: column_sidecar.block_root(), - data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), - _phantom: Default::default(), - } + validate_data_column_sidecar_for_gossip::(column_sidecar, subnet_id, chain).map_err( + |e| { + process_block_slash_info::<_, GossipDataColumnError>( + chain, + BlockSlashInfo::from_early_error_data_column(header.clone(), e), + ) + }, + ) } /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY. @@ -272,6 +263,15 @@ impl GossipVerifiedDataColumn }) } + /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. + pub fn __new_for_testing(column_sidecar: Arc>) -> Self { + Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), + _phantom: Default::default(), + } + } + pub fn as_data_column(&self) -> &DataColumnSidecar { self.data_column.as_data_column() } @@ -286,15 +286,15 @@ impl GossipVerifiedDataColumn } pub fn slot(&self) -> Slot { - self.data_column.as_data_column().slot() + self.data_column.data.slot() } pub fn index(&self) -> ColumnIndex { - self.data_column.index() + self.data_column.data.index } - pub fn signed_block_header(&self) -> &SignedBeaconBlockHeader { - &self.data_column.as_data_column().signed_block_header + pub fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.data_column.data.signed_block_header.clone() } pub fn into_inner(self) -> KzgVerifiedDataColumn { @@ -302,9 +302,6 @@ impl GossipVerifiedDataColumn } } -/// Alias for backwards compatibility -pub type GossipVerifiedFullDataColumn = GossipVerifiedDataColumn; - /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] @@ -366,13 +363,6 @@ pub struct KzgVerifiedPartialDataColumn { } impl KzgVerifiedPartialDataColumn { - pub fn new( - data_column: PartialDataColumn, - kzg: &Kzg, - ) -> Result, KzgError)> { - verify_kzg_for_partial_data_column(data_column, kzg) - } - /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. pub(crate) fn __new_for_testing(data_column: PartialDataColumn) -> Self { Self { data: data_column } @@ -387,12 +377,10 @@ impl KzgVerifiedPartialDataColumn { pub fn to_data_column(self) -> PartialDataColumn { self.data } + pub fn as_data_column(&self) -> &PartialDataColumn { &self.data } - pub fn clone_data_column(&self) -> PartialDataColumn { - self.data.clone() - } pub fn index(&self) -> ColumnIndex { self.data.index @@ -401,18 +389,6 @@ impl KzgVerifiedPartialDataColumn { pub fn block_root(&self) -> Hash256 { self.data.block_root } - - /// Returns the slot of this partial data column. - pub fn slot(&self) -> Slot { - self.data - .sidecar - .header - .first() - .expect("Header must be present for validated partial columns") - .signed_block_header - .message - .slot - } } /// Wrapper over a `PartialDataColumnHeader` for which we have completed gossip verification. @@ -480,14 +456,6 @@ impl GossipVerifiedPartialDataColumnHeader { } } - pub fn as_inner(&self) -> &PartialDataColumnHeader { - &self.header - } - - pub fn into_inner(self) -> PartialDataColumnHeader { - self.header - } - pub fn was_cached(&self) -> bool { self.previously_cached } @@ -535,17 +503,6 @@ pub struct KzgVerifiedCustodyDataColumn { } impl KzgVerifiedCustodyDataColumn { - /// Verify a column already marked as custody column - pub fn new( - data_column: CustodyDataColumn, - kzg: &Kzg, - ) -> Result, KzgError)> { - verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; - Ok(Self { - data: data_column.into_inner(), - }) - } - /// Mark a column as custody column. Caller must ensure that our current custody requirements /// include this column pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { @@ -554,34 +511,22 @@ impl KzgVerifiedCustodyDataColumn { } } - pub fn into_inner(self) -> Arc> { - self.data - } - - pub fn as_data_column(&self) -> &DataColumnSidecar { - &self.data - } - - pub fn clone_arc(&self) -> Arc> { - self.data.clone() - } - - pub fn index(&self) -> ColumnIndex { - self.data.index - } - - pub fn into_partial(self) -> KzgVerifiedCustodyPartialDataColumn { - let column = Arc::try_unwrap(self.data).unwrap_or_else(|column| (*column).clone()); - KzgVerifiedCustodyPartialDataColumn { - data: column.into_partial(), - } + /// Verify a column already marked as custody column + pub fn new( + data_column: CustodyDataColumn, + kzg: &Kzg, + ) -> Result, KzgError)> { + verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; + Ok(Self { + data: data_column.data, + }) } pub fn reconstruct_columns( kzg: &Kzg, partial_set_of_columns: &[Self], spec: &ChainSpec, - ) -> Result, KzgError> { + ) -> Result>, KzgError> { let all_data_columns = reconstruct_data_columns( kzg, partial_set_of_columns @@ -598,6 +543,20 @@ impl KzgVerifiedCustodyDataColumn { }) .collect::>()) } + + pub fn into_inner(self) -> Arc> { + self.data + } + + pub fn as_data_column(&self) -> &DataColumnSidecar { + &self.data + } + pub fn clone_arc(&self) -> Arc> { + self.data.clone() + } + pub fn index(&self) -> ColumnIndex { + self.data.index + } } /// Partial data column that we must custody and has completed kzg verification. @@ -605,7 +564,7 @@ impl KzgVerifiedCustodyDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedCustodyPartialDataColumn { - data: PartialDataColumn, + data: Arc>, } impl KzgVerifiedCustodyPartialDataColumn { @@ -613,11 +572,11 @@ impl KzgVerifiedCustodyPartialDataColumn { /// include this column pub fn from_asserted_custody(kzg_verified: KzgVerifiedPartialDataColumn) -> Self { Self { - data: kzg_verified.to_data_column(), + data: Arc::new(kzg_verified.to_data_column()), } } - pub fn into_inner(self) -> PartialDataColumn { + pub fn into_inner(self) -> Arc> { self.data } @@ -628,6 +587,27 @@ impl KzgVerifiedCustodyPartialDataColumn { pub fn index(&self) -> ColumnIndex { self.data.index } + + pub fn merge(&self, other: &Self) -> Option { + self.data + .sidecar + .merge(&other.data.sidecar) + .map(|sidecar| Self { + data: Arc::new(PartialDataColumn { + block_root: self.data.block_root, + index: self.data.index, + sidecar, + }), + }) + } + + pub fn try_clone_full(&self) -> Option> { + self.data + .try_clone_full() + .map(|data| KzgVerifiedCustodyDataColumn { + data: Arc::new(data), + }) + } } /// Complete kzg verification for a `DataColumnSidecar`. @@ -661,10 +641,13 @@ pub fn verify_kzg_for_partial_data_column( /// /// Note: This function should be preferred over calling `verify_kzg_for_data_column` /// in a loop since this function kzg verifies a list of data columns more efficiently. -pub fn verify_kzg_for_data_column_list<'a, E: EthSpec + 'a>( - data_column_iter: impl Iterator>>, - kzg: &Kzg, -) -> Result<(), (Option, KzgError)> { +pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I>( + data_column_iter: I, + kzg: &'a Kzg, +) -> Result<(), (Option, KzgError)> +where + I: Iterator>> + Clone, +{ let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES); validate_full_data_columns(kzg, data_column_iter)?; Ok(()) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 907b455121c..55f65c08be9 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -14,7 +14,7 @@ mod tests; use crate::blob_verification::{GossipBlobError, KzgVerifiedBlob}; use crate::data_column_verification::{ - KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + KzgVerifiedCustodyDataColumn, KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, }; #[cfg_attr(test, double)] use crate::fetch_blobs::fetch_blobs_beacon_adapter::FetchBlobsBeaconAdapter; @@ -33,9 +33,8 @@ use mockall_double::double; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; -use types::data::{BlobSidecarError, DataColumnSidecarError}; -use types::partial_data_column_sidecar::PartialDataColumnHeader; -use types::{BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, Hash256, VersionedHash}; +use types::data::{BlobSidecarError, ColumnIndex, DataColumnSidecarError, PartialDataColumnHeader}; +use types::{BeaconStateError, BlobSidecar, EthSpec, Hash256, VersionedHash}; /// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the /// gossip network. The blobs / data columns have not been marked as observed yet, as they may not @@ -44,7 +43,7 @@ use types::{BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, Hash256, Versio pub enum EngineGetBlobsOutput { Blobs(Vec>), /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. - CustodyColumns(Vec>>), + CustodyColumns(Vec>), } #[derive(Debug)] @@ -102,6 +101,11 @@ async fn fetch_and_process_engine_blobs_inner( return Ok(None); }; + debug!( + num_expected_blobs = versioned_hashes.len(), + "Fetching blobs from the EL" + ); + if chain_adapter .spec() .is_peer_das_enabled_for_epoch(header.slot().epoch(T::EthSpec::slots_per_epoch())) @@ -325,21 +329,14 @@ async fn fetch_and_process_blobs_v2_or_v3( // Initialize the partial assembler with the columns from the engine let assembler = chain_adapter.partial_assembler(); let merge_result = assembler - .merge_partials( - block_root, - custody_columns_to_import - .into_iter() - .map(|c| c.into_inner()) - .collect(), - true, - ) + .merge_partials(block_root, custody_columns_to_import, true) .ok_or_else(|| { FetchEngineBlobError::InternalError( "Failed to merge partials into assembler".to_string(), ) })?; - // Publish complete columns and incomplete partials + // Publish complete columns if !merge_result.full_columns.is_empty() { publish_fn(EngineGetBlobsOutput::CustodyColumns( merge_result.full_columns.clone(), @@ -389,7 +386,7 @@ async fn compute_custody_columns_to_import( &[&blobs_and_proofs.len().to_string()], ); - let blob_and_cell_refs = blobs_and_proofs + let blob_and_proof_refs = blobs_and_proofs .iter() .map(|option| { option @@ -398,7 +395,7 @@ async fn compute_custody_columns_to_import( }) .collect::>(); let data_columns_result = - blobs_to_partial_data_columns(blob_and_cell_refs, &header, &kzg, &spec) + blobs_to_partial_data_columns(blob_and_proof_refs, &header, &kzg, &spec) .discard_timer_on_break(&mut timer); drop(timer); diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index d1b34dac4a7..0299796b582 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -23,8 +23,7 @@ type T = EphemeralHarnessType; mod get_blobs_v2 { use super::*; - use types::ColumnIndex; - use types::partial_data_column_sidecar::PartialDataColumnHeader; + use types::{ColumnIndex, PartialDataColumnHeader}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_in_block() { @@ -256,8 +255,7 @@ mod get_blobs_v1 { use super::*; use crate::block_verification_types::AsBlock; use std::collections::HashSet; - use types::partial_data_column_sidecar::PartialDataColumnHeader; - use types::{ColumnIndex, FullPayload}; + use types::{ColumnIndex, FullPayload, PartialDataColumnHeader}; const ELECTRA_FORK: ForkName = ForkName::Electra; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 50de3eedfa9..ef0aa26e103 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -6,9 +6,11 @@ use rayon::prelude::*; use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; -use types::data::{Cell, DataColumn, DataColumnSidecarError}; +use types::data::{ + Cell, CellBitmap, DataColumn, DataColumnSidecarError, PartialDataColumn, + PartialDataColumnHeader, +}; use types::kzg_ext::KzgCommitments; -use types::partial_data_column_sidecar::{CellBitmap, PartialDataColumn, PartialDataColumnHeader}; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, @@ -65,17 +67,17 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( return Err((Some(col_index), KzgError::KzgVerificationFailed)); } - // Full columns have all cells present - iterate over column, commitments, and proofs - for ((cell, commitment), proof) in data_column - .column - .iter() - .zip(data_column.kzg_commitments.iter()) - .zip(data_column.kzg_proofs.iter()) - { + for cell in &data_column.column { cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); column_indices.push(col_index); - proofs.push(Bytes48::from(*proof)); - commitments.push(Bytes48::from(*commitment)); + } + + for &proof in &data_column.kzg_proofs { + proofs.push(Bytes48::from(proof)); + } + + for &commitment in &data_column.kzg_commitments { + commitments.push(Bytes48::from(commitment)); } let expected_len = column_indices.len(); @@ -466,7 +468,7 @@ pub(crate) fn build_partial_data_columns( let column = PartialDataColumn { block_root: header.signed_block_header.message.canonical_root(), index: index as u64, - sidecar: types::partial_data_column_sidecar::PartialDataColumnSidecar { + sidecar: types::data::PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), column: VariableList::try_from(col) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index eb630dcdf1d..f0e70f77de4 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -1,3 +1,6 @@ +use crate::data_column_verification::{ + KzgVerifiedCustodyDataColumn, KzgVerifiedCustodyPartialDataColumn, +}; use lru::LruCache; use parking_lot::RwLock; use parking_lot::lock_api::RwLockReadGuard; @@ -5,8 +8,8 @@ use std::collections::HashMap; use std::num::NonZeroUsize; use std::ops::Deref; use std::sync::Arc; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; -use types::{ColumnIndex, DataColumnSidecar, EthSpec, Hash256}; +use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; +use types::{EthSpec, Hash256}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -19,7 +22,7 @@ struct PartialAssembly { header: PartialDataColumnHeader, has_local_blobs: bool, /// Map of column_index -> partial column being assembled - columns: HashMap>>, + columns: HashMap>, } /// Result of merging a partial column @@ -29,9 +32,9 @@ pub struct PartialMergeResult { /// Have local blobs been added yet pub local_blobs: bool, /// Merge that completed the column - pub full_columns: Vec>>, + pub full_columns: Vec>, /// The updated partials for publishing - pub updated_partials: Vec>>, + pub updated_partials: Vec>, } impl PartialDataColumnAssembler { @@ -72,7 +75,7 @@ impl PartialDataColumnAssembler { pub fn merge_partials( &self, block_root: Hash256, - partials: Vec>, + partials: Vec>, local_blobs: bool, ) -> Option> { let mut assemblies = self.assemblies.write(); @@ -80,7 +83,7 @@ impl PartialDataColumnAssembler { .try_get_or_insert_mut(block_root, || { partials .iter() - .filter_map(|partial| partial.sidecar.header.first()) + .filter_map(|partial| partial.as_data_column().sidecar.header.first()) .next() .map(|header| PartialAssembly { header: header.clone(), @@ -96,23 +99,29 @@ impl PartialDataColumnAssembler { let mut added_cells = 0; for partial in partials { - let column_index = partial.index; + let partial_column = partial.as_data_column(); + let column_index = partial_column.index; let merged = if let Some(existing) = assembly.columns.get(&column_index) { + let column = existing.as_data_column(); // Check if the column is already completed - if existing.sidecar.is_complete() { + if column.sidecar.is_complete() { continue; } + let old_len = column.sidecar.column.len(); + // Merge with existing partial - let Some(merged_sidecar) = existing.sidecar.merge(&partial.sidecar) else { + let Some(merged) = existing.merge(&partial) else { continue; }; - let adding_cells = merged_sidecar + let adding_cells = merged + .as_data_column() + .sidecar .column .len() - .saturating_sub(partial.sidecar.column.len()); + .saturating_sub(old_len); added_cells += adding_cells; @@ -120,24 +129,19 @@ impl PartialDataColumnAssembler { continue; } - PartialDataColumn { - block_root: existing.block_root, - index: existing.index, - sidecar: merged_sidecar, - } + merged } else { - added_cells += partial.sidecar.column.len(); + added_cells += partial_column.sidecar.column.len(); // First time seeing this column index for this block partial }; // Check if merged column is now complete by trying to convert into full if let Some(full_column) = merged.try_clone_full() { - full_columns.push(Arc::new(full_column)); + full_columns.push(full_column); } // Update assembly with merged partial - let merged = Arc::new(merged); assembly.columns.insert(column_index, merged.clone()); updated_partials.push(merged); } @@ -159,7 +163,7 @@ impl PartialDataColumnAssembler { &self, block_root: &Hash256, column_index: ColumnIndex, - ) -> Option>> { + ) -> Option> { self.assemblies .read() .peek(block_root)? @@ -169,7 +173,10 @@ impl PartialDataColumnAssembler { } /// Get all current partials for a block - pub fn get_partials(&self, block_root: &Hash256) -> Option>>> { + pub fn get_partials( + &self, + block_root: &Hash256, + ) -> Option>> { Some( self.assemblies .read() @@ -192,16 +199,6 @@ impl PartialDataColumnAssembler { .ok() } - /// Check if we have an assembly for this block - pub fn has_assembly(&self, block_root: &Hash256) -> bool { - self.assemblies.read().contains(block_root) - } - - /// Remove assembly for a block (called when block is finalized or evicted) - pub fn remove_assembly(&self, block_root: &Hash256) { - self.assemblies.write().pop(block_root); - } - /// Maintenance: remove assemblies older than cutoff epoch pub fn do_maintenance(&self, cutoff_epoch: types::Epoch) { let mut assemblies = self.assemblies.write(); @@ -224,11 +221,6 @@ impl PartialDataColumnAssembler { assemblies.pop(&root); } } - - /// Get cache size for metrics - pub fn cache_size(&self) -> usize { - self.assemblies.read().len() - } } /// Result of initializing with engine blobs diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index ec43f517992..c2c985be73b 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -41,8 +41,7 @@ pub use crate::scheduler::BeaconProcessorQueueLengths; use crate::scheduler::work_queue::WorkQueues; use crate::work_reprocessing_queue::{ - QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, QueuedPartialColumn, - ReprocessQueueMessage, + QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; @@ -289,10 +288,6 @@ impl From for WorkEvent { work: Work::ColumnReconstruction(process_fn), } } - ReadyWork::PartialColumn(QueuedPartialColumn { process_fn, .. }) => Self { - drop_during_sync: true, - work: Work::GossipPartialDataColumnSidecar(process_fn), - }, } } } diff --git a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs index 04286714ec0..c99388287c0 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs @@ -30,7 +30,7 @@ use strum::AsRefStr; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, trace, warn}; use types::{EthSpec, Hash256, Slot}; const TASK_NAME: &str = "beacon_processor_reprocess_queue"; @@ -60,9 +60,6 @@ pub const QUEUED_SAMPLING_REQUESTS_DELAY: Duration = Duration::from_secs(12); /// For how long to queue delayed column reconstruction. pub const QUEUED_RECONSTRUCTION_DELAY: Duration = Duration::from_millis(150); -/// For how long to queue partial columns. -pub const QUEUED_PARTIAL_COLUMN_DELAY: Duration = Duration::from_secs(4); - /// Set an arbitrary upper-bound on the number of queued blocks to avoid DoS attacks. The fact that /// we signature-verify blocks before putting them in the queue *should* protect against this, but /// it's nice to have extra protection. @@ -74,9 +71,6 @@ const MAXIMUM_QUEUED_ATTESTATIONS: usize = 16_384; /// How many light client updates we keep before new ones get dropped. const MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES: usize = 128; -/// How many partial column messages we keep before new ones get dropped. -const MAXIMUM_QUEUED_PARTIAL_COLUMNS: usize = 256; - // Process backfill batch 50%, 60%, 80% through each slot. // // Note: use caution to set these fractions in a way that won't cause panic-y @@ -121,10 +115,6 @@ pub enum ReprocessQueueMessage { BackfillSync(QueuedBackfillBatch), /// A delayed column reconstruction that needs checking DelayColumnReconstruction(QueuedColumnReconstruction), - /// A block with pending data availability was added. - BlockPending { block_root: Hash256 }, - /// A partial data column that references an unknown block. - UnknownPartialColumn(QueuedPartialColumn), } /// Events sent by the scheduler once they are ready for re-processing. @@ -137,7 +127,6 @@ pub enum ReadyWork { LightClientUpdate(QueuedLightClientUpdate), BackfillSync(QueuedBackfillBatch), ColumnReconstruction(QueuedColumnReconstruction), - PartialColumn(QueuedPartialColumn), } /// An Attestation for which the corresponding block was not seen while processing, queued for @@ -193,11 +182,6 @@ pub struct QueuedColumnReconstruction { pub process_fn: AsyncFn, } -pub struct QueuedPartialColumn { - pub beacon_block_root: Hash256, - pub process_fn: AsyncFn, -} - impl TryFrom> for QueuedBackfillBatch { type Error = WorkEvent; @@ -236,8 +220,6 @@ enum InboundEvent { ReadyBackfillSync(QueuedBackfillBatch), /// A column reconstruction that was queued is ready for processing. ReadyColumnReconstruction(QueuedColumnReconstruction), - /// A partial column that was queued has timed out. - ReadyPartialColumn((DelayKey, QueuedPartialColumn)), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -260,8 +242,6 @@ struct ReprocessQueue { lc_updates_delay_queue: DelayQueue, /// Queue to manage scheduled column reconstructions. column_reconstructions_delay_queue: DelayQueue, - /// Queue to manage scheduled partial columns. - partial_columns_delay_queue: DelayQueue, /* Queued items */ /// Queued blocks. @@ -280,8 +260,6 @@ struct ReprocessQueue { queued_column_reconstructions: HashMap, /// Queued backfill batches queued_backfill_batches: Vec, - /// Partial columns per root. - awaiting_partial_columns_per_root: HashMap>, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations @@ -377,16 +355,6 @@ impl Stream for ReprocessQueue { Poll::Ready(None) | Poll::Pending => (), } - match self.partial_columns_delay_queue.poll_expired(cx) { - Poll::Ready(Some(partial_column)) => { - return Poll::Ready(Some(InboundEvent::ReadyPartialColumn(( - partial_column.key(), - partial_column.into_inner(), - )))); - } - Poll::Ready(None) | Poll::Pending => (), - } - if let Some(next_backfill_batch_event) = self.next_backfill_batch_event.as_mut() { match next_backfill_batch_event.as_mut().poll(cx) { Poll::Ready(_) => { @@ -454,7 +422,6 @@ impl ReprocessQueue { attestations_delay_queue: DelayQueue::new(), lc_updates_delay_queue: DelayQueue::new(), column_reconstructions_delay_queue: DelayQueue::new(), - partial_columns_delay_queue: DelayQueue::new(), queued_gossip_block_roots: HashSet::new(), queued_lc_updates: FnvHashMap::default(), queued_aggregates: FnvHashMap::default(), @@ -463,7 +430,6 @@ impl ReprocessQueue { awaiting_lc_updates_per_parent_root: HashMap::new(), queued_backfill_batches: Vec::new(), queued_column_reconstructions: HashMap::new(), - awaiting_partial_columns_per_root: HashMap::new(), next_attestation: 0, next_lc_update: 0, early_block_debounce: TimeLatch::default(), @@ -736,15 +702,6 @@ impl ReprocessQueue { ); } } - - // Remove waiting partial columns - as the block is imported they will not be needed - if let Some(queued_partials) = - self.awaiting_partial_columns_per_root.remove(&block_root) - { - for key in queued_partials { - self.partial_columns_delay_queue.remove(&key); - } - } } InboundEvent::Msg(NewLightClientOptimisticUpdate { parent_root }) => { // Unqueue the light client optimistic updates we have for this root, if any. @@ -827,62 +784,6 @@ impl ReprocessQueue { } } } - InboundEvent::Msg(BlockPending { block_root }) => { - if let Some(queued_partials) = - self.awaiting_partial_columns_per_root.remove(&block_root) - { - let mut sent_count = 0; - let mut failed_count = 0; - for key in queued_partials { - let partial_column = self.partial_columns_delay_queue.remove(&key); - - if self - .ready_work_tx - .try_send(ReadyWork::PartialColumn(partial_column.into_inner())) - .is_ok() - { - sent_count += 1; - } else { - failed_count += 1; - } - } - - if failed_count > 0 { - error!( - hint = "system may be overloaded", - ?block_root, - failed_count, - sent_count, - "Ignored scheduled partial column(s) for block" - ); - } else { - info!(?block_root, sent_count, "Sent partial column(s) for block") - } - } - } - InboundEvent::Msg(UnknownPartialColumn(queued_partial_column)) => { - if self.partial_columns_delay_queue.len() >= MAXIMUM_QUEUED_PARTIAL_COLUMNS { - error!( - queue_size = MAXIMUM_QUEUED_PARTIAL_COLUMNS, - msg = "system resources may be saturated", - "Partial columns delay queue is full" - ); - return; - } - - let block_root = queued_partial_column.beacon_block_root; - - // Register the delay. - let delay_key = self - .partial_columns_delay_queue - .insert(queued_partial_column, QUEUED_ATTESTATION_DELAY); - - // Register this attestation for the corresponding root. - self.awaiting_partial_columns_per_root - .entry(block_root) - .or_default() - .push(delay_key); - } // A block that was queued for later processing is now ready to be processed. InboundEvent::ReadyGossipBlock(ready_block) => { let block_root = ready_block.beacon_block_root; @@ -1033,22 +934,6 @@ impl ReprocessQueue { ); } } - InboundEvent::ReadyPartialColumn((key, partial_column)) => { - if let Entry::Occupied(mut queued_cols) = self - .awaiting_partial_columns_per_root - .entry(partial_column.beacon_block_root) - && let Some(index) = queued_cols.get().iter().position(|&k| k == key) - { - let queued_atts_mut = queued_cols.get_mut(); - queued_atts_mut.swap_remove(index); - - if queued_atts_mut.is_empty() { - queued_cols.remove_entry(); - } - } - - // Do not send partial column, we timed out already. - } } metrics::set_gauge_vec( diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index c9589eea605..1e4972ceb6b 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -43,11 +43,9 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; -use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; -use types::{ChainSpec, ForkName}; use types::{ - DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, Slot, SubnetId, - consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, + CellBitmap, ChainSpec, DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, ForkName, + PartialDataColumn, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, }; use utils::{Context as ServiceContext, build_transport, strip_peer_id}; @@ -863,10 +861,11 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let bytes = message.encode(); - let gossipsub = self.gossipsub_mut(); - let publish_topic: Topic = topic.clone().into(); - if let Err(e) = gossipsub.publish(publish_topic, bytes.clone()) { + let message_data = message.encode(GossipEncoding::default()); + if let Err(e) = self + .gossipsub_mut() + .publish(Topic::from(topic.clone()), message_data.clone()) + { match e { PublishError::Duplicate => { debug!( @@ -914,7 +913,7 @@ impl Network { } if let PublishError::NoPeersSubscribedToTopic = e { - self.gossip_cache.insert(topic, bytes); + self.gossip_cache.insert(topic, message_data); } } } @@ -1908,12 +1907,9 @@ impl Network { fn inject_upnp_event(&mut self, event: libp2p::upnp::Event) { match event { - libp2p::upnp::Event::NewExternalAddr { - external_addr: addr, - .. - } => { - info!(%addr, "UPnP route established"); - let mut iter = addr.iter(); + libp2p::upnp::Event::NewExternalAddr { external_addr, .. } => { + info!(%external_addr, "UPnP route established"); + let mut iter = external_addr.iter(); let is_ip6 = { let addr = iter.next(); matches!(addr, Some(MProtocol::Ip6(_))) @@ -1928,7 +1924,7 @@ impl Network { } } _ => { - trace!(%addr, "UPnP address mapped multiaddr from unknown transport"); + trace!(%external_addr, "UPnP address mapped multiaddr from unknown transport"); } }, Some(multiaddr::Protocol::Tcp(tcp_port)) => { @@ -1937,7 +1933,7 @@ impl Network { } } _ => { - trace!(%addr, "UPnP address mapped multiaddr from unknown transport"); + trace!(%external_addr, "UPnP address mapped multiaddr from unknown transport"); } } } diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 18a5fe0d634..9e31c6ab057 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -7,8 +7,8 @@ use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use tracing::debug; -use types::data::partial_data_column_sidecar::{CellBitmap, PartialDataColumn}; -use types::partial_data_column_sidecar::PartialDataColumnSidecar; +use types::PartialDataColumnSidecar; +use types::{CellBitmap, PartialDataColumn}; use types::{EthSpec, Hash256}; pub type HeaderSentSet = Arc>>; diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 96ccca6e1b7..e61273d1a63 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -6,13 +6,13 @@ use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use types::data::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, - SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, - SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, + LightClientFinalityUpdate, LightClientOptimisticUpdate, PartialDataColumn, + PartialDataColumnSidecar, ProposerSlashing, SignedAggregateAndProof, + SignedAggregateAndProofBase, SignedAggregateAndProofElectra, SignedBeaconBlock, + SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockFulu, SignedBeaconBlockGloas, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, SubnetId, @@ -393,8 +393,9 @@ impl PubsubMessage { } } - /// Encodes a `PubsubMessage`. - pub fn encode(&self) -> Vec { + /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If + /// no encoding is known, and error is returned. + pub fn encode(&self, _encoding: GossipEncoding) -> Vec { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e657e94a226..7f3024d84d2 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -40,10 +40,10 @@ use tracing::{Instrument, Span, debug, error, info, instrument, trace, warn}; use types::{ Attestation, AttestationData, AttestationRef, AttesterSlashing, BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, - LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, Slot, SubnetId, - SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, + LightClientFinalityUpdate, LightClientOptimisticUpdate, PartialDataColumn, + PartialDataColumnHeader, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, + Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, }; use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; @@ -54,7 +54,6 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -867,6 +866,7 @@ impl NetworkBeaconProcessor { self.process_gossip_verified_partial_data_column( peer_id, gossip_verified_data_column, + slot, seen_duration, ) .await @@ -1285,44 +1285,7 @@ impl NetworkBeaconProcessor { "Processed data column, waiting for other components" ); - if self - .chain - .data_availability_checker - .custody_context() - .should_attempt_reconstruction( - slot.epoch(T::EthSpec::slots_per_epoch()), - &self.chain.spec, - ) - { - // Instead of triggering reconstruction immediately, schedule it to be run. If - // another column arrives, it either completes availability or pushes - // reconstruction back a bit. - let cloned_self = Arc::clone(self); - let block_root = *block_root; - - if self - .beacon_processor_send - .try_send(WorkEvent { - drop_during_sync: false, - work: Work::Reprocess( - ReprocessQueueMessage::DelayColumnReconstruction( - QueuedColumnReconstruction { - block_root, - slot: *slot, - process_fn: Box::pin(async move { - cloned_self - .attempt_data_column_reconstruction(block_root) - .await; - }), - }, - ), - ), - }) - .is_err() - { - warn!("Unable to send reconstruction to reprocessing"); - } - } + self.check_reconstruction_trigger(*slot, block_root).await; } }, Err(BlockError::DuplicateFullyImported(_)) => { @@ -1363,6 +1326,7 @@ impl NetworkBeaconProcessor { self: &Arc, _peer_id: PeerId, verified_partial: KzgVerifiedPartialDataColumn, + slot: Slot, _seen_duration: Duration, ) { let processing_start_time = Instant::now(); @@ -1371,11 +1335,12 @@ impl NetworkBeaconProcessor { let result = self .chain - .process_gossip_partial_data_column(verified_partial) + .process_gossip_partial_data_column(verified_partial, slot) .await; - match result { - Ok(merge_result) => { + // First, handle merge results (if any) + let result = match result { + Ok(Some((avail, merge_result))) => { if !merge_result.full_columns.is_empty() { debug!( %block_root, @@ -1383,44 +1348,140 @@ impl NetworkBeaconProcessor { "Partial data column completed to full column" ); - for full_column in merge_result.full_columns { - // Publish both the updated partial and the full column - let subnet = DataColumnSubnetId::from_column_index( - full_column.index, - &self.chain.spec, - ); - self.send_network_message(NetworkMessage::Publish { - messages: vec![PubsubMessage::DataColumnSidecar(Box::new(( - subnet, - full_column.clone(), - )))], - }); - } + self.send_network_message(NetworkMessage::Publish { + messages: merge_result + .full_columns + .into_iter() + .map(|col| { + let subnet = DataColumnSubnetId::from_column_index( + col.index(), + &self.chain.spec, + ); + PubsubMessage::DataColumnSidecar(Box::new(( + subnet, + col.into_inner(), + ))) + }) + .collect(), + }); } if merge_result.local_blobs || self.chain.config.disable_get_blobs { self.send_network_message(NetworkMessage::PublishPartial { - columns: merge_result.updated_partials, + columns: merge_result + .updated_partials + .into_iter() + .map(|partial| partial.into_inner()) + .collect(), }); } else { debug!(block = %block_root, "Not publishing partials - waiting for getBlobs"); } + Ok(avail) + } + Ok(None) => { + // Column was not merged because it is not a custody column. + return; + } + Err(err) => Err(err), + }; - // Check if block is now fully available - // This is handled in process_gossip_partial_data_column + register_process_result_metrics( + &result, + metrics::BlockSource::Gossip, + "partial_data_column", + ); - metrics::set_gauge( - &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, - processing_start_time.elapsed().as_millis() as i64, + match &result { + Ok(availability) => match availability { + AvailabilityProcessingStatus::Imported(block_root) => { + debug!( + %block_root, + "Data column from partial processed, imported fully available block" + ); + self.chain.recompute_head_at_current_slot().await; + + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, + processing_start_time.elapsed().as_millis() as i64, + ); + } + AvailabilityProcessingStatus::MissingComponents(slot, block_root) => { + trace!( + %slot, + %data_column_index, + %block_root, + "Processed data column from partial, waiting for other components" + ); + + self.check_reconstruction_trigger(*slot, block_root).await; + } + }, + Err(BlockError::DuplicateFullyImported(_)) => { + debug!( + ?block_root, + data_column_index, "Ignoring completed gossip column already imported" ); } - Err(e) => { - warn!( - error = ?e, - %block_root, - index = data_column_index, - "Error processing partial data column" + Err(err) => { + debug!( + outcome = ?err, + ?block_root, + block_slot = %slot, + data_column_index, + "Invalid completed gossip data column" ); + // We can't really penalize here, as the error might be the fault of another peer + // contributing to the partial. + } + } + + // If a block is in the da_checker, sync maybe awaiting for an event when block is finally + // imported. A block can become imported both after processing a block or data column. If a + // importing a block results in `Imported`, notify. Do not notify of data column errors. + if matches!(result, Ok(AvailabilityProcessingStatus::Imported(_))) { + self.send_sync_message(SyncMessage::GossipBlockProcessResult { + block_root, + imported: true, + }); + } + } + + async fn check_reconstruction_trigger(self: &Arc, slot: Slot, block_root: &Hash256) { + if self + .chain + .data_availability_checker + .custody_context() + .should_attempt_reconstruction( + slot.epoch(T::EthSpec::slots_per_epoch()), + &self.chain.spec, + ) + { + // Instead of triggering reconstruction immediately, schedule it to be run. If + // another column arrives, it either completes availability or pushes + // reconstruction back a bit. + let cloned_self = Arc::clone(self); + let block_root = *block_root; + + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::DelayColumnReconstruction( + QueuedColumnReconstruction { + block_root, + slot, + process_fn: Box::pin(async move { + cloned_self + .attempt_data_column_reconstruction(block_root) + .await; + }), + }, + )), + }) + .is_err() + { + warn!("Unable to send reconstruction to reprocessing"); } } } @@ -1850,22 +1911,6 @@ impl NetworkBeaconProcessor { %block_root, "Processed block, waiting for other components" ); - if self - .beacon_processor_send - .try_send(WorkEvent { - drop_during_sync: false, - work: Work::Reprocess(ReprocessQueueMessage::BlockPending { - block_root: *block_root, - }), - }) - .is_err() - { - error!( - source = "gossip", - ?block_root, - "Failed to inform block pending" - ) - }; } Err(BlockError::ParentUnknown { .. }) => { // This should not occur. It should be checked by `should_forward_block`. diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 77179e92a6a..4d2a376c7e2 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -28,7 +28,6 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; use tracing::{debug, error, instrument, trace, warn}; -use types::partial_data_column_sidecar::{PartialDataColumn, PartialDataColumnHeader}; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -795,8 +794,10 @@ impl NetworkBeaconProcessor { ); } EngineGetBlobsOutput::CustodyColumns(columns) => { - // Gradually publish full columns - self_cloned.publish_data_columns_gradually(columns, block_root); + self_cloned.publish_data_columns_gradually( + columns.into_iter().map(|c| c.clone_arc()).collect(), + block_root, + ); } }; } @@ -858,7 +859,12 @@ impl NetworkBeaconProcessor { .get_partials(&block_root) { debug!(block = %block_root, "Publishing all partials after getBlobs"); - self.send_network_message(NetworkMessage::PublishPartial { columns }) + self.send_network_message(NetworkMessage::PublishPartial { + columns: columns + .into_iter() + .map(|partial| partial.into_inner()) + .collect(), + }); } } diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index eca9308f0da..ce9a4aea19f 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -26,9 +26,8 @@ use std::sync::Arc; use std::time::Duration; use store::KzgCommitment; use tracing::{debug, debug_span, error, info, instrument, warn}; -use types::data::FixedBlobSidecarList; +use types::data::{FixedBlobSidecarList, PartialDataColumnHeader}; use types::kzg_ext::format_kzg_commitments; -use types::partial_data_column_sidecar::PartialDataColumnHeader; use types::{BlockImportSource, DataColumnSidecarList, Epoch, Hash256}; /// Id associated to a batch processing request, either a sync batch or a parent lookup. @@ -216,21 +215,6 @@ impl NetworkBeaconProcessor { self.chain.recompute_head_at_current_slot().await; } Ok(AvailabilityProcessingStatus::MissingComponents(..)) => { - if self - .beacon_processor_send - .try_send(WorkEvent { - drop_during_sync: false, - work: Work::Reprocess(ReprocessQueueMessage::BlockPending { block_root }), - }) - .is_err() - { - error!( - source = "rpc", - ?block_root, - "Failed to inform block pending" - ) - }; - // Block is valid, we can now attempt fetching blobs from EL using version hashes // derived from kzg commitments from the block, without having to wait for all blobs // to be sent from the peers if we already have them. diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 4012a58246b..d0987558c6b 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -24,8 +24,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, trace, warn}; -use types::data::partial_data_column_sidecar::PartialDataColumn; -use types::{BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, SignedBeaconBlock}; +use types::{ + BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, PartialDataColumn, SignedBeaconBlock, +}; /// Handles messages from the network and routes them to the appropriate service to be handled. pub struct Router { diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 29715cfa340..05283ec7321 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -38,10 +38,9 @@ use tokio::sync::mpsc; use tokio::time::Sleep; use tracing::{debug, error, info, trace, warn}; use typenum::Unsigned; -use types::data::partial_data_column_sidecar::PartialDataColumn; use types::{ - EthSpec, ForkContext, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, - ValidatorSubscription, + EthSpec, ForkContext, PartialDataColumn, Slot, SubnetId, SyncCommitteeSubscription, + SyncSubnetId, ValidatorSubscription, }; mod tests; diff --git a/consensus/types/src/data/blob_sidecar.rs b/consensus/types/src/data/blob_sidecar.rs index 3adfc009838..3de4d6a7f00 100644 --- a/consensus/types/src/data/blob_sidecar.rs +++ b/consensus/types/src/data/blob_sidecar.rs @@ -15,14 +15,13 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::partial_data_column_sidecar::PartialDataColumnHeader; use crate::{ block::{ BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlock, SignedBeaconBlockHeader, }, complete_kzg_commitment_merkle_proof, core::{ChainSpec, Epoch, EthSpec, Hash256, Slot}, - data::Blob, + data::{Blob, PartialDataColumnHeader}, fork::ForkName, kzg_ext::KzgProofs, state::BeaconStateError, diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 6bc70b42f8d..b4fdf524b7c 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -18,10 +18,10 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::partial_data_column_sidecar::PartialDataColumnHeader; use crate::{ block::{BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlockHeader}, core::{Epoch, EthSpec, Hash256, Slot}, + data::PartialDataColumnHeader, fork::ForkName, kzg_ext::{KzgCommitments, KzgError}, state::BeaconStateError, diff --git a/consensus/types/src/data/mod.rs b/consensus/types/src/data/mod.rs index e8cb833e99b..6e03e98a532 100644 --- a/consensus/types/src/data/mod.rs +++ b/consensus/types/src/data/mod.rs @@ -2,7 +2,7 @@ mod blob_sidecar; mod data_column_custody_group; mod data_column_sidecar; mod data_column_subnet_id; -pub mod partial_data_column_sidecar; +mod partial_data_column_sidecar; pub use blob_sidecar::{ BlobIdentifier, BlobSidecar, BlobSidecarError, BlobSidecarList, BlobsList, FixedBlobSidecarList, @@ -17,6 +17,9 @@ pub use data_column_sidecar::{ DataColumnSidecarList, DataColumnsByRootIdentifier, }; pub use data_column_subnet_id::{DataColumnSubnetId, all_data_column_sidecar_subnets_from_spec}; +pub use partial_data_column_sidecar::{ + CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, +}; use crate::core::EthSpec; use ssz_types::FixedVector; From af796a328aef11800f5c37e9acf5f6106eb8cbe2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 6 Feb 2026 12:14:55 +0100 Subject: [PATCH 083/146] Do maintenance and removal for partial assembler --- beacon_node/beacon_chain/src/data_availability_checker.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 887dc2550a4..558021f45f7 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -759,8 +759,12 @@ pub fn start_availability_cache_maintenance_service( // this cache only needs to be maintained if deneb is configured if chain.spec.deneb_fork_epoch.is_some() { let overflow_cache = chain.data_availability_checker.availability_cache.clone(); + let partial_assembler = chain.data_availability_checker.partial_assembler.clone(); executor.spawn( - async move { availability_cache_maintenance_service(chain, overflow_cache).await }, + async move { + availability_cache_maintenance_service(chain, overflow_cache, partial_assembler) + .await + }, "availability_cache_service", ); } else { @@ -771,6 +775,7 @@ pub fn start_availability_cache_maintenance_service( async fn availability_cache_maintenance_service( chain: Arc>, overflow_cache: Arc>, + partial_assembler: Arc>, ) { let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32; loop { @@ -822,6 +827,7 @@ async fn availability_cache_maintenance_service( if let Err(e) = overflow_cache.do_maintenance(cutoff_epoch) { error!(error = ?e,"Failed to maintain availability cache"); } + partial_assembler.do_maintenance(cutoff_epoch); } None => { error!("Failed to read slot clock"); From 39cf8ec715db689e1ad532ad3d899ef1b8d0a25d Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 11 Feb 2026 12:26:27 +0100 Subject: [PATCH 084/146] implement spec change: request bitmap --- .../lighthouse_network/src/types/partial.rs | 101 ++++++++++++------ consensus/types/src/data/mod.rs | 3 +- .../src/data/partial_data_column_sidecar.rs | 17 +++ 3 files changed, 87 insertions(+), 34 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 9e31c6ab057..0904211bbcb 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,4 +1,5 @@ use crate::PeerId; +use itertools::Itertools; use libp2p::gossipsub::partial_messages::{Metadata, Partial, PartialAction, PartialError}; use parking_lot::Mutex; use ssz::{Decode, Encode}; @@ -7,9 +8,11 @@ use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use tracing::debug; -use types::PartialDataColumnSidecar; use types::{CellBitmap, PartialDataColumn}; use types::{EthSpec, Hash256}; +use types::{PartialDataColumnPartsMetadata, PartialDataColumnSidecar}; + +const PARTIAL_COLUMNS_VERSION_BYTE: u8 = 0; pub type HeaderSentSet = Arc>>; @@ -18,6 +21,7 @@ pub struct NoHeaderInColumnError; #[derive(Debug, Clone)] pub struct OutgoingPartialColumn { pub partial_column: Arc>, + pub metadata: Vec, pub header_message: Vec, pub header_sent_set: HeaderSentSet, } @@ -31,6 +35,19 @@ impl OutgoingPartialColumn { return Err(NoHeaderInColumnError); }; + // For now, always request all cells + let mut request = partial_column.sidecar.cells_present_bitmap.clone(); + for idx in 0..request.len() { + request + .set(idx, true) + .expect("Bound asserted via `len` above"); + } + let metadata = PartialDataColumnPartsMetadata:: { + available: partial_column.sidecar.cells_present_bitmap.clone(), + request: partial_column.sidecar.cells_present_bitmap.clone(), + } + .as_ssz_bytes(); + let header_message = PartialDataColumnSidecar { cells_present_bitmap: CellBitmap::::with_capacity( partial_column.sidecar.cells_present_bitmap.len(), @@ -44,6 +61,7 @@ impl OutgoingPartialColumn { Ok(OutgoingPartialColumn { partial_column, + metadata, header_message, header_sent_set, }) @@ -51,52 +69,62 @@ impl OutgoingPartialColumn { } #[derive(Debug, Clone, PartialEq, Eq)] -enum CellBitmapMetadata { +enum MaybeKnownMetadata { Unknown, Known { - bitmap: CellBitmap, + metadata: Box>, encoded: Vec, }, } -impl Metadata for CellBitmapMetadata { +impl Metadata for MaybeKnownMetadata { fn as_slice(&self) -> &[u8] { match self { - CellBitmapMetadata::Unknown => &[], - CellBitmapMetadata::Known { encoded, .. } => encoded, + MaybeKnownMetadata::Unknown => &[], + MaybeKnownMetadata::Known { encoded, .. } => encoded, } } fn update(&mut self, data: &[u8]) -> Result { - let peer_bitmap = - CellBitmap::::from_ssz_bytes(data).map_err(|_| PartialError::InvalidFormat)?; + let received = PartialDataColumnPartsMetadata::from_ssz_bytes(data) + .map_err(|_| PartialError::InvalidFormat)?; - let CellBitmapMetadata::Known { bitmap, encoded } = self else { - *self = CellBitmapMetadata::Known { - bitmap: peer_bitmap, + let MaybeKnownMetadata::Known { metadata, encoded } = self else { + *self = MaybeKnownMetadata::Known { + metadata: Box::new(received), encoded: data.to_vec(), }; return Ok(true); }; - if peer_bitmap.len() != bitmap.len() { + if ![ + received.available.len(), + received.request.len(), + metadata.available.len(), + metadata.request.len(), + ] + .into_iter() + .all_equal() + { return Err(PartialError::OutOfRange); } - let new_bitmap = bitmap.union(&peer_bitmap); - if *bitmap == new_bitmap { + let new_available = metadata.available.union(&received.available); + let new_request = metadata.request.union(&received.request); + if metadata.available == new_available && metadata.request == new_request { return Ok(false); } - *bitmap = new_bitmap; - *encoded = bitmap.as_ssz_bytes(); + metadata.available = new_available; + metadata.request = new_request; + *encoded = metadata.as_ssz_bytes(); Ok(true) } } -impl From> for CellBitmapMetadata { - fn from(value: CellBitmap) -> Self { +impl From> for MaybeKnownMetadata { + fn from(metadata: PartialDataColumnPartsMetadata) -> Self { Self::Known { - encoded: value.as_ssz_bytes(), - bitmap: value, + encoded: metadata.as_ssz_bytes(), + metadata: Box::new(metadata), } } } @@ -104,16 +132,13 @@ impl From> for CellBitmapMetadata { impl Partial for OutgoingPartialColumn { fn group_id(&self) -> Vec { let mut group_id = Vec::with_capacity(Hash256::len_bytes() + 1); - group_id.push(0); + group_id.push(PARTIAL_COLUMNS_VERSION_BYTE); group_id.extend_from_slice(self.partial_column.block_root.as_slice()); group_id } fn metadata(&self) -> Vec { - self.partial_column - .sidecar - .cells_present_bitmap - .as_ssz_bytes() + self.metadata.clone() } fn partial_action_from_metadata( @@ -127,7 +152,7 @@ impl Partial for OutgoingPartialColumn { let send = self.header_sent_set.lock().insert(peer_id).then(|| { ( self.header_message.clone(), - Box::new(CellBitmapMetadata::::Unknown) as Box, + Box::new(MaybeKnownMetadata::::Unknown) as Box, ) }); debug!( @@ -145,27 +170,37 @@ impl Partial for OutgoingPartialColumn { send: None, }), Some(metadata) => { - let peer_has = CellBitmap::::from_ssz_bytes(metadata) + let peer_metadata = PartialDataColumnPartsMetadata::::from_ssz_bytes(metadata) .map_err(|_| PartialError::InvalidFormat)?; - let need = !peer_has.is_subset(&self.partial_column.sidecar.cells_present_bitmap); + let need = !peer_metadata + .available + .is_subset(&self.partial_column.sidecar.cells_present_bitmap); + let want = peer_metadata.request.difference(&peer_metadata.available); let send = self .partial_column .sidecar - .with_missing_cells(&peer_has) + .with_missing_cells(&want) .map(|sidecar| { debug!( peer=%peer_id, group_id=%self.partial_column.block_root, column_index=self.partial_column.index, - metadata=%peer_has, + metadata=%peer_metadata, sending=%sidecar.cells_present_bitmap, "Partial send: Sending" ); ( sidecar.as_ssz_bytes(), - Box::new(CellBitmapMetadata::::from( - peer_has.union(&sidecar.cells_present_bitmap), + Box::new(MaybeKnownMetadata::::from( + PartialDataColumnPartsMetadata { + available: peer_metadata + .available + .union(&sidecar.cells_present_bitmap), + request: peer_metadata + .request + .union(&sidecar.cells_present_bitmap), + }, )) as Box, ) }); @@ -175,7 +210,7 @@ impl Partial for OutgoingPartialColumn { peer=%peer_id, group_id=%self.partial_column.block_root, column_index=self.partial_column.index, - metadata=%peer_has, + metadata=%peer_metadata, "Partial send: Nothing to send" ); } diff --git a/consensus/types/src/data/mod.rs b/consensus/types/src/data/mod.rs index b0c3c19c2b5..c920d2c2ae2 100644 --- a/consensus/types/src/data/mod.rs +++ b/consensus/types/src/data/mod.rs @@ -19,7 +19,8 @@ pub use data_column_sidecar::{ }; pub use data_column_subnet_id::{DataColumnSubnetId, all_data_column_sidecar_subnets_from_spec}; pub use partial_data_column_sidecar::{ - CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, + CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, + PartialDataColumnSidecar, }; use crate::core::EthSpec; diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index e9caf2cc6ec..d88273bf984 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -10,6 +10,7 @@ use merkle_proof::verify_merkle_proof; use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; +use std::fmt::Display; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; @@ -233,6 +234,22 @@ impl> TryFrom<&SignedBeaconBlock> } } +#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +pub struct PartialDataColumnPartsMetadata { + pub available: CellBitmap, + pub request: CellBitmap, +} + +impl Display for PartialDataColumnPartsMetadata { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "(available: {}, requested: {})", + self.available, self.request + ) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumn { pub block_root: Hash256, From 57befbd835f0e4895b95887f13a1f41d793cafe1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 11 Feb 2026 12:27:35 +0100 Subject: [PATCH 085/146] Remove column from assembly when complete This enables - last metric - correct handling of full column --- beacon_node/beacon_chain/src/beacon_chain.rs | 36 ++++++++---- .../src/data_availability_checker.rs | 48 +++++++++------- beacon_node/beacon_chain/src/metrics.rs | 9 --- .../src/partial_data_column_assembler.rs | 56 +++++++++++++++---- beacon_node/http_api/src/publish_blocks.rs | 2 +- beacon_node/network/src/metrics.rs | 8 +++ .../gossip_methods.rs | 41 ++++++++++---- .../types/src/data/data_column_sidecar.rs | 12 ++-- 8 files changed, 143 insertions(+), 69 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 7620613b823..348f743413e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3199,19 +3199,33 @@ impl BeaconChain { merge_result.added_cells as u64, ); - metrics::inc_counter_vec_by( - &metrics::BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL, - &[index_str.as_str()], - merge_result.full_columns.len() as u64, - ); + let availability = if !merge_result.full_columns.is_empty() { + metrics::inc_counter_vec_by( + &metrics::BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL, + &[index_str.as_str()], + merge_result.full_columns.len() as u64, + ); - let availability = self - .data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, merge_result.full_columns.clone())?; + self.emit_sse_data_column_sidecar_events( + &block_root, + merge_result + .full_columns + .iter() + .map(|column| column.as_data_column()), + ); - let availability = self - .process_availability(slot, availability, || Ok(())) - .await?; + let availability = self + .data_availability_checker + .put_kzg_verified_custody_data_columns( + block_root, + merge_result.full_columns.clone(), + )?; + + self.process_availability(slot, availability, || Ok(())) + .await? + } else { + AvailabilityProcessingStatus::MissingComponents(slot, block_root) + }; Ok(Some((availability, merge_result))) } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 779a5c39c9e..b0704e62a99 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -5,7 +5,7 @@ use crate::block_verification_types::{AvailabilityPendingExecutedBlock, Availabl use crate::data_availability_checker::overflow_lru_cache::{ DataAvailabilityCheckerInner, ReconstructColumnsDecision, }; -use crate::partial_data_column_assembler::PartialDataColumnAssembler; +use crate::partial_data_column_assembler::{AssemblyColumn, PartialDataColumnAssembler}; use crate::{BeaconChain, BeaconChainTypes, BlockProcessStatus, CustodyContext, metrics}; use educe::Educe; use kzg::Kzg; @@ -206,13 +206,17 @@ impl DataAvailabilityChecker { } // Check assembler for partial columns - if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { - // Compare: find which cells from incoming full column aren't in cached partial - return compare_full_to_partial(cell_count, cached_partial.as_data_column()); + match self.partial_assembler.get_partial(block_root, column_index) { + Some(AssemblyColumn::Incomplete(cached_partial)) => { + // Compare: find which cells from incoming full column aren't in cached partial + compare_full_to_partial(cell_count, cached_partial.as_data_column()) + } + Some(AssemblyColumn::Complete) => Some(vec![]), + None => { + // No cached data, all cells are "missing" (new data we want) + Some((0..cell_count).collect()) + } } - - // No cached data, all cells are "missing" (new data we want) - Some((0..cell_count).collect()) } /// Check if the partial data column is in the availability cache. @@ -238,20 +242,24 @@ impl DataAvailabilityChecker { } // Check assembler for partial columns - if let Some(cached_partial) = self.partial_assembler.get_partial(block_root, column_index) { - // Compare: find which cells from incoming partial aren't in cached partial - return compare_partial_to_partial(data_column, cached_partial.as_data_column()); + match self.partial_assembler.get_partial(block_root, column_index) { + Some(AssemblyColumn::Incomplete(cached_partial)) => { + // Compare: find which cells from incoming full column aren't in cached partial + compare_partial_to_partial(data_column, cached_partial.as_data_column()) + } + Some(AssemblyColumn::Complete) => Some(vec![]), + None => { + // No cached data, return all present cells as "missing" (new data we want) + let incoming_cells: Vec = data_column + .sidecar + .cells_present_bitmap + .iter() + .enumerate() + .filter_map(|(idx, present)| present.then_some(idx)) + .collect(); + Some(incoming_cells) + } } - - // No cached data, return all present cells as "missing" (new data we want) - let incoming_cells: Vec = data_column - .sidecar - .cells_present_bitmap - .iter() - .enumerate() - .filter_map(|(idx, present)| present.then_some(idx)) - .collect(); - Some(incoming_cells) } /// Get a blob from the availability cache. diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 730dfea25a0..922d70e959e 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1821,15 +1821,6 @@ pub static BEACON_PARTIAL_MESSAGE_CELLS_RECEIVED_TOTAL: LazyLock> = - LazyLock::new(|| { - try_create_int_counter_vec( - "beacon_useful_full_columns_received_total", - "Number of useful full columns (any cell being useful) received", - &["column_index"], - ) - }); - pub static BEACON_PARTIAL_MESSAGE_COLUMN_COMPLETIONS_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index f0e70f77de4..9ec73054136 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -9,7 +9,7 @@ use std::num::NonZeroUsize; use std::ops::Deref; use std::sync::Arc; use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; -use types::{EthSpec, Hash256}; +use types::{DataColumnSidecarFulu, EthSpec, Hash256}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -22,7 +22,13 @@ struct PartialAssembly { header: PartialDataColumnHeader, has_local_blobs: bool, /// Map of column_index -> partial column being assembled - columns: HashMap>, + columns: HashMap>, +} + +#[derive(Clone, Debug)] +pub enum AssemblyColumn { + Complete, + Incomplete(KzgVerifiedCustodyPartialDataColumn), } /// Result of merging a partial column @@ -103,11 +109,11 @@ impl PartialDataColumnAssembler { let column_index = partial_column.index; let merged = if let Some(existing) = assembly.columns.get(&column_index) { - let column = existing.as_data_column(); - // Check if the column is already completed - if column.sidecar.is_complete() { + let AssemblyColumn::Incomplete(existing) = existing else { + // Already complete. continue; - } + }; + let column = existing.as_data_column(); let old_len = column.sidecar.column.len(); @@ -137,12 +143,15 @@ impl PartialDataColumnAssembler { }; // Check if merged column is now complete by trying to convert into full - if let Some(full_column) = merged.try_clone_full() { + let column = if let Some(full_column) = merged.try_clone_full() { full_columns.push(full_column); - } + AssemblyColumn::Complete + } else { + AssemblyColumn::Incomplete(merged.clone()) + }; // Update assembly with merged partial - assembly.columns.insert(column_index, merged.clone()); + assembly.columns.insert(column_index, column); updated_partials.push(merged); } @@ -158,12 +167,31 @@ impl PartialDataColumnAssembler { }) } + /// Mark a column as assembled. Returns true if the column was previously incomplete or not + /// in the assembly at all. + pub fn mark_as_complete(&self, block_root: Hash256, column: &DataColumnSidecarFulu) -> bool { + let mut assemblies = self.assemblies.write(); + let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { + header: PartialDataColumnHeader { + kzg_commitments: column.kzg_commitments.clone(), + signed_block_header: column.signed_block_header.clone(), + kzg_commitments_inclusion_proof: column.kzg_commitments_inclusion_proof.clone(), + }, + has_local_blobs: false, + columns: Default::default(), + }); + let prev = assembly + .columns + .insert(column.index, AssemblyColumn::Complete); + !matches!(prev, Some(AssemblyColumn::Complete)) + } + /// Get the current partial for a specific column if it exists in assembly pub fn get_partial( &self, block_root: &Hash256, column_index: ColumnIndex, - ) -> Option> { + ) -> Option> { self.assemblies .read() .peek(block_root)? @@ -183,7 +211,13 @@ impl PartialDataColumnAssembler { .peek(block_root)? .columns .values() - .cloned() + .filter_map(|value| { + if let AssemblyColumn::Incomplete(partial) = value { + Some(partial.clone()) + } else { + None + } + }) .collect(), ) } diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 210a9329e57..acb95bfe954 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -533,7 +533,7 @@ fn publish_column_sidecars( for data_col in data_column_sidecars { if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { - partial_columns.push(Arc::new((*fulu_data_col).clone().into_partial())); + partial_columns.push(Arc::new(fulu_data_col.to_partial())); } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 7a05bfe560e..dfe99455ae8 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -606,6 +606,14 @@ pub static BEACON_PARTIAL_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = + LazyLock::new(|| { + try_create_int_counter_vec( + "beacon_useful_full_columns_received_total", + "Number of useful full columns (any cell being useful) received", + &["column_index"], + ) + }); pub static BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: LazyLock> = LazyLock::new( || { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 8edcb6727fa..3c222c489a6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1275,7 +1275,22 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); - // TODO(dknopik): import into assembly and track BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL metric + if let DataColumnSidecar::Fulu(col) = verified_data_column.as_data_column() + && self + .chain + .data_availability_checker + .partial_assembler() + .mark_as_complete(block_root, col) + { + metrics::inc_counter_vec( + &metrics::BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL, + &[&data_column_index.to_string()], + ); + + self.send_network_message(NetworkMessage::PublishPartial { + columns: vec![Arc::new(col.to_partial())], + }); + } let result = self .chain @@ -1386,16 +1401,20 @@ impl NetworkBeaconProcessor { }); } - if merge_result.local_blobs || self.chain.config.disable_get_blobs { - self.send_network_message(NetworkMessage::PublishPartial { - columns: merge_result - .updated_partials - .into_iter() - .map(|partial| partial.into_inner()) - .collect(), - }); - } else { - debug!(block = %block_root, "Not publishing partials - waiting for getBlobs"); + let only_send_completed_partials = + merge_result.local_blobs || self.chain.config.disable_get_blobs; + debug!(block = %block_root, "Not incomplete publishing partials before getBlobs"); + let columns = merge_result + .updated_partials + .into_iter() + .map(|partial| partial.into_inner()) + .filter(|partial| { + !only_send_completed_partials || partial.sidecar.is_complete() + }) + .collect::>(); + + if !columns.is_empty() { + self.send_network_message(NetworkMessage::PublishPartial { columns }); } Ok(avail) } diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index da2564a3646..7dbe14060f1 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -210,7 +210,7 @@ impl DataColumnSidecarFulu { } /// Convert this full data column into a verifiable partial data column. - pub fn into_partial(self) -> PartialDataColumn { + pub fn to_partial(&self) -> PartialDataColumn { let cell_count = self.column.len(); let mut bitmap = CellBitmap::::with_capacity(cell_count).expect("our column has the same bound"); @@ -224,8 +224,8 @@ impl DataColumnSidecarFulu { let column = self .column - .into_iter() - .map(Arc::new) + .iter() + .map(|col| Arc::new(col.clone())) .collect::>() .try_into() .expect("our column has the same bound"); @@ -236,11 +236,11 @@ impl DataColumnSidecarFulu { sidecar: PartialDataColumnSidecar { cells_present_bitmap: bitmap, column, - kzg_proofs: self.kzg_proofs, + kzg_proofs: self.kzg_proofs.clone(), header: VariableList::repeat_full(PartialDataColumnHeader { kzg_commitments: self.kzg_commitments.clone(), - signed_block_header: self.signed_block_header, - kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof, + signed_block_header: self.signed_block_header.clone(), + kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof.clone(), }), }, } From c6494f29845d673f986f953a71b64bc716ad3613 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 13 Feb 2026 18:17:03 +0100 Subject: [PATCH 086/146] log silent if no peers support partial no special case for duplicates --- beacon_node/lighthouse_network/src/service/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index eafd7e99758..1f1ddf4b178 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -944,10 +944,10 @@ impl Network { .publish_partial(publish_topic, partial_message) { match e { - PublishError::Duplicate => { + PublishError::NoPeersSubscribedToTopic => { debug!( kind = %topic.kind(), - "Attempted to publish duplicate partial message" + "No peers supporting partial messages" ); } ref e => { From 58cdcf4e196a872215adca0ccac61442007de93e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 13 Feb 2026 18:17:03 +0100 Subject: [PATCH 087/146] only send partial if enabled --- beacon_node/lighthouse_network/src/service/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 1f1ddf4b178..9a3ff57dff6 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -922,6 +922,10 @@ impl Network { /// Publishes partial data column sidecars to the gossipsub network. pub fn publish_partial(&mut self, columns: Vec>>) { + if !self.network_globals.config.enable_partial_columns { + return; + } + for column in columns { let subnet = DataColumnSubnetId::from_column_index(column.index, &self.fork_context.spec); From d794d4f221574cbe5666f5c253cc12ddd46e311e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 16 Feb 2026 14:40:37 +0100 Subject: [PATCH 088/146] update libp2p --- Cargo.lock | 56 +++++++++---------- .../lighthouse_network/src/types/partial.rs | 14 ++--- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8099c9cfb82..d4054a7cd8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 1.0.109", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4963,7 +4963,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "bytes", "either", @@ -4994,7 +4994,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5004,7 +5004,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5014,7 +5014,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "either", "fnv", @@ -5038,7 +5038,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "async-trait", "futures", @@ -5053,7 +5053,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5083,7 +5083,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "asynchronous-codec", "either", @@ -5123,7 +5123,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "hickory-proto", @@ -5141,7 +5141,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "libp2p-core", @@ -5157,7 +5157,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "asynchronous-codec", "bytes", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "asynchronous-codec", "bytes", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "futures-timer", @@ -5218,7 +5218,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "either", "fnv", @@ -5239,7 +5239,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "heck", "quote", @@ -5249,7 +5249,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "futures-timer", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "futures-rustls", @@ -5282,7 +5282,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "futures-timer", @@ -5296,7 +5296,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "either", "futures", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "bytes", "futures", @@ -6195,7 +6195,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7065,7 +7065,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "asynchronous-codec", "bytes", @@ -7127,7 +7127,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7637,7 +7637,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7743,7 +7743,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#f5da15bf834e3eac8a13d22eebebb6a987f2909f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" dependencies = [ "futures", "pin-project", @@ -8683,7 +8683,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10012,7 +10012,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 0904211bbcb..9fa52499def 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -20,10 +20,10 @@ pub struct NoHeaderInColumnError; #[derive(Debug, Clone)] pub struct OutgoingPartialColumn { - pub partial_column: Arc>, - pub metadata: Vec, - pub header_message: Vec, - pub header_sent_set: HeaderSentSet, + partial_column: Arc>, + metadata: MaybeKnownMetadata, + header_message: Vec, + header_sent_set: HeaderSentSet, } impl OutgoingPartialColumn { @@ -46,7 +46,7 @@ impl OutgoingPartialColumn { available: partial_column.sidecar.cells_present_bitmap.clone(), request: partial_column.sidecar.cells_present_bitmap.clone(), } - .as_ssz_bytes(); + .into(); let header_message = PartialDataColumnSidecar { cells_present_bitmap: CellBitmap::::with_capacity( @@ -137,8 +137,8 @@ impl Partial for OutgoingPartialColumn { group_id } - fn metadata(&self) -> Vec { - self.metadata.clone() + fn metadata(&self) -> Box { + Box::new(self.metadata.clone()) } fn partial_action_from_metadata( From a102f6c3f960283e06e0bf8379ef41e95c9dd256 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 18 Feb 2026 11:38:11 +0100 Subject: [PATCH 089/146] update libp2p --- Cargo.lock | 72 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d59bb02f49..eb266ef9197 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.111", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3978,11 +3978,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -4966,7 +4966,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "bytes", "either", @@ -4997,7 +4997,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5007,7 +5007,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5016,8 +5016,8 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +version = "0.43.2" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "either", "fnv", @@ -5041,7 +5041,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "async-trait", "futures", @@ -5056,7 +5056,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5068,7 +5068,7 @@ dependencies = [ "futures", "futures-timer", "getrandom 0.2.16", - "hashlink 0.10.0", + "hashlink 0.11.0", "hex_fmt", "libp2p-core", "libp2p-identity", @@ -5086,7 +5086,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "asynchronous-codec", "either", @@ -5126,7 +5126,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "hickory-proto", @@ -5144,7 +5144,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "libp2p-core", @@ -5160,7 +5160,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "asynchronous-codec", "bytes", @@ -5178,7 +5178,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "asynchronous-codec", "bytes", @@ -5200,7 +5200,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "futures-timer", @@ -5220,14 +5220,14 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +version = "0.47.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "either", "fnv", "futures", "futures-timer", - "hashlink 0.10.0", + "hashlink 0.11.0", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", @@ -5242,7 +5242,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "heck", "quote", @@ -5251,8 +5251,8 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +version = "0.44.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "futures-timer", @@ -5267,7 +5267,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "futures-rustls", @@ -5285,7 +5285,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "futures-timer", @@ -5299,7 +5299,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "either", "futures", @@ -5974,7 +5974,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "bytes", "futures", @@ -6199,7 +6199,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7069,7 +7069,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "asynchronous-codec", "bytes", @@ -7131,7 +7131,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7641,7 +7641,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7747,7 +7747,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#5c2e99b45371baed28b9bd647e10f7fb7e7662d7" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" dependencies = [ "futures", "pin-project", @@ -8687,7 +8687,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10016,7 +10016,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From 970ce627d1ad5095ee12c56fc267cf343b8f5363 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 19 Feb 2026 16:26:25 +0100 Subject: [PATCH 090/146] implement optional metadata function --- .../lighthouse_network/src/types/partial.rs | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 9fa52499def..3446b05e46a 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -77,22 +77,15 @@ enum MaybeKnownMetadata { }, } -impl Metadata for MaybeKnownMetadata { - fn as_slice(&self) -> &[u8] { - match self { - MaybeKnownMetadata::Unknown => &[], - MaybeKnownMetadata::Known { encoded, .. } => encoded, - } - } - - fn update(&mut self, data: &[u8]) -> Result { - let received = PartialDataColumnPartsMetadata::from_ssz_bytes(data) - .map_err(|_| PartialError::InvalidFormat)?; - +impl MaybeKnownMetadata { + fn do_update( + &mut self, + received: PartialDataColumnPartsMetadata, + ) -> Result { let MaybeKnownMetadata::Known { metadata, encoded } = self else { *self = MaybeKnownMetadata::Known { + encoded: received.as_ssz_bytes(), metadata: Box::new(received), - encoded: data.to_vec(), }; return Ok(true); }; @@ -120,6 +113,37 @@ impl Metadata for MaybeKnownMetadata { } } +impl Metadata for MaybeKnownMetadata { + fn as_slice(&self) -> &[u8] { + match self { + MaybeKnownMetadata::Unknown => &[], + MaybeKnownMetadata::Known { encoded, .. } => encoded, + } + } + + fn update(&mut self, data: &[u8]) -> Result { + let received = PartialDataColumnPartsMetadata::from_ssz_bytes(data) + .map_err(|_| PartialError::InvalidFormat)?; + + self.do_update(received) + } + + fn update_from_data(&mut self, data: &[u8]) -> Result<(), PartialError> { + if data.is_empty() { + return Ok(()); + } + + let sidecar = PartialDataColumnSidecar::::from_ssz_bytes(data) + .map_err(|_| PartialError::InvalidFormat)?; + + self.do_update(PartialDataColumnPartsMetadata { + available: sidecar.cells_present_bitmap.clone(), + request: sidecar.cells_present_bitmap, + }) + .map(|_| ()) + } +} + impl From> for MaybeKnownMetadata { fn from(metadata: PartialDataColumnPartsMetadata) -> Self { Self::Known { From 2a93e79c7a0418d5a1fd634c92135beee495b0d0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 19 Feb 2026 16:40:27 +0100 Subject: [PATCH 091/146] implement reporting of failures to libp2p --- .../lighthouse_network/src/service/mod.rs | 21 ++++++--- .../gossip_methods.rs | 45 +++++++++---------- .../src/network_beacon_processor/mod.rs | 10 ++++- beacon_node/network/src/router.rs | 13 ++++-- beacon_node/network/src/service.rs | 24 +++++++++- 5 files changed, 75 insertions(+), 38 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9a3ff57dff6..bca870c4018 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -110,6 +110,8 @@ pub enum NetworkEvent { source: PeerId, /// The partial column data. column: PartialDataColumn, + /// The topic that this message was sent on. + topic: GossipTopic, }, /// Inform the network to send a Status to this peer. StatusPeer(PeerId), @@ -1006,6 +1008,16 @@ impl Network { ); } + /// Informs the gossipsub about the failure of a partial message validation. + pub fn report_partial_message_validation_failure( + &mut self, + propagation_source: PeerId, + topic: GossipTopic, + ) { + self.gossipsub_mut() + .report_invalid_partial(propagation_source, &TopicHash::from(Topic::from(topic))); + } + /// Updates the current gossipsub scoring parameters based on the validator count and current /// slot. pub fn update_gossipsub_parameters( @@ -1415,12 +1427,8 @@ impl Network { "Could not decode gossipsub partial message" ); //reject the message - // TODO(dknopik): implement when ready in libp2p - //self.gossipsub_mut().report_message_validation_result( - // &todo!(), - // &propagation_source, - // MessageAcceptance::Reject, - //); + self.gossipsub_mut() + .report_invalid_partial(peer_id, &topic_hash); } Ok(column) => { debug!( @@ -1433,6 +1441,7 @@ impl Network { return Some(NetworkEvent::PartialDataColumnSidecar { source: peer_id, column, + topic, }); } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 1d5d80363e3..6bf25456850 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -23,7 +23,8 @@ use beacon_chain::{ }; use beacon_processor::{Work, WorkEvent}; use lighthouse_network::{ - Client, MessageAcceptance, MessageId, PeerAction, PeerId, PubsubMessage, ReportSource, + Client, GossipTopic, MessageAcceptance, MessageId, PeerAction, PeerId, PubsubMessage, + ReportSource, }; use logging::crit; use operation_pool::ReceivedPreCapella; @@ -194,6 +195,19 @@ impl NetworkBeaconProcessor { }) } + /// Send a message on `message_tx` that `peer_id` should be propagated on + /// the gossip network. + pub(crate) fn propagate_partial_validation_failure( + &self, + propagation_source: PeerId, + gossip_topic: GossipTopic, + ) { + self.send_network_message(NetworkMessage::PartialValidationFailure { + propagation_source, + gossip_topic, + }) + } + /* Processing functions */ /// Process the unaggregated attestation received from the gossip network and: @@ -798,6 +812,7 @@ impl NetworkBeaconProcessor { peer_id: PeerId, mut column: PartialDataColumn, seen_duration: Duration, + topic: GossipTopic, ) { let mut get_blobs = false; let header = if let Some(header) = column.sidecar.header.first() { @@ -815,6 +830,7 @@ impl NetworkBeaconProcessor { err, column.block_root, column.index, + topic, ); return; } @@ -860,9 +876,6 @@ impl NetworkBeaconProcessor { "Successfully verified gossip partial data column sidecar" ); - // TODO(dknopik): wait for joao's validation result impl - //self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); - // Log metrics to keep track of propagation delay times. if let Some(duration) = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -884,7 +897,7 @@ impl NetworkBeaconProcessor { .await } Err(err) => { - self.handle_partial_verification_error(peer_id, err, block_root, index); + self.handle_partial_verification_error(peer_id, err, block_root, index, topic); } } @@ -903,6 +916,7 @@ impl NetworkBeaconProcessor { err: GossipDataColumnError, block_root: Hash256, index: ColumnIndex, + topic: GossipTopic, ) { match err { GossipDataColumnError::InvalidVariant => { @@ -917,14 +931,8 @@ impl NetworkBeaconProcessor { debug!( %block_root, %index, - "Gossip data column already processed via the EL. Accepting the column sidecar without re-processing." + "Gossip data column already processed via the EL." ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Accept, - //); } // ParentUnknown can't really happen, so we treat it like an internal error GossipDataColumnError::ParentUnknown { .. } @@ -960,12 +968,7 @@ impl NetworkBeaconProcessor { PeerAction::LowToleranceError, "gossip_data_column_low", ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Reject, - //); + self.propagate_partial_validation_failure(peer_id, topic); } GossipDataColumnError::PriorKnown { .. } => { // Data column is available via either the EL or reconstruction. @@ -991,12 +994,6 @@ impl NetworkBeaconProcessor { PeerAction::HighToleranceError, "gossip_data_column_high", ); - // TODO(dknopik): Joao - //self.propagate_validation_result( - // message_id, - // peer_id, - // MessageAcceptance::Ignore, - //); } GossipDataColumnError::PartialHeaderMismatches | GossipDataColumnError::PartialHeaderIncorrectRoot { .. } => { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index fa3a30f1d8c..9581727dbb9 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -18,7 +18,7 @@ use lighthouse_network::rpc::methods::{ }; use lighthouse_network::service::api_types::CustodyBackfillBatchId; use lighthouse_network::{ - Client, MessageId, NetworkGlobals, PeerId, PubsubMessage, + Client, GossipTopic, MessageId, NetworkGlobals, PeerId, PubsubMessage, rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, }; use rand::prelude::SliceRandom; @@ -255,11 +255,17 @@ impl NetworkBeaconProcessor { peer_id: PeerId, column_sidecar: PartialDataColumn, seen_timestamp: Duration, + topic: GossipTopic, ) -> Result<(), Error> { let processor = self.clone(); let process_fn = async move { processor - .process_gossip_partial_data_column_sidecar(peer_id, column_sidecar, seen_timestamp) + .process_gossip_partial_data_column_sidecar( + peer_id, + column_sidecar, + seen_timestamp, + topic, + ) .await }; diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 3f6b21b8ade..a102eb02bd2 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -14,7 +14,7 @@ use beacon_processor::{BeaconProcessorSend, DuplicateCache}; use futures::prelude::*; use lighthouse_network::rpc::*; use lighthouse_network::{ - MessageId, NetworkGlobals, PeerId, PubsubMessage, Response, + GossipTopic, MessageId, NetworkGlobals, PeerId, PubsubMessage, Response, service::api_types::{AppRequestId, SyncRequestId}, }; use logging::TimeLatch; @@ -72,7 +72,7 @@ pub enum RouterMessage { /// by the beacon chain after successful verification. PubsubMessage(MessageId, PeerId, PubsubMessage, bool), /// A partial data column sidecar has been received via gossipsub partial protocol. - PartialDataColumnSidecar(PeerId, PartialDataColumn), + PartialDataColumnSidecar(PeerId, PartialDataColumn, GossipTopic), /// The peer manager has requested we re-status a peer. StatusPeer(PeerId), /// The peer has an updated custody group count from METADATA. @@ -184,10 +184,15 @@ impl Router { RouterMessage::PubsubMessage(id, peer_id, gossip, should_process) => { self.handle_gossip(id, peer_id, gossip, should_process); } - RouterMessage::PartialDataColumnSidecar(peer_id, column) => self + RouterMessage::PartialDataColumnSidecar(peer_id, column, topic) => self .handle_beacon_processor_send_result( self.network_beacon_processor - .send_gossip_partial_data_column_sidecar(peer_id, column, timestamp_now()), + .send_gossip_partial_data_column_sidecar( + peer_id, + column, + timestamp_now(), + topic, + ), ), } } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 48fdb97e3dc..2e3198042fd 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -96,6 +96,13 @@ pub enum NetworkMessage { /// The result of the validation validation_result: MessageAcceptance, }, + /// Reports validation failure of a partial message. + PartialValidationFailure { + /// The peer that sent us the message. + propagation_source: PeerId, + /// The topic of the message. + gossip_topic: GossipTopic, + }, /// Reports a peer to the peer manager for performing an action. ReportPeer { peer_id: PeerId, @@ -564,8 +571,14 @@ impl NetworkService { } } } - NetworkEvent::PartialDataColumnSidecar { source, column } => { - self.send_to_router(RouterMessage::PartialDataColumnSidecar(source, column)); + NetworkEvent::PartialDataColumnSidecar { + source, + column, + topic, + } => { + self.send_to_router(RouterMessage::PartialDataColumnSidecar( + source, column, topic, + )); } NetworkEvent::NewListenAddr(multiaddr) => { self.network_globals @@ -647,6 +660,13 @@ impl NetworkService { validation_result, ); } + NetworkMessage::PartialValidationFailure { + propagation_source, + gossip_topic, + } => { + self.libp2p + .report_partial_message_validation_failure(propagation_source, gossip_topic); + } NetworkMessage::Publish { messages } => { let mut topic_kinds = Vec::new(); for message in &messages { From 9cbde0bb0e71d24a7336e2b2921b28f6863b2536 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 25 Feb 2026 02:55:50 +0100 Subject: [PATCH 092/146] fix lookup on unknown parent --- .../gossip_methods.rs | 39 ++++++++++++++----- .../network/src/sync/block_lookups/mod.rs | 18 ++++----- .../sync/block_lookups/single_block_lookup.rs | 4 +- beacon_node/network/src/sync/manager.rs | 33 ++++++++++++++-- 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 6bf25456850..db2c0fe8c71 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -830,6 +830,7 @@ impl NetworkBeaconProcessor { err, column.block_root, column.index, + Some(column), topic, ); return; @@ -897,7 +898,9 @@ impl NetworkBeaconProcessor { .await } Err(err) => { - self.handle_partial_verification_error(peer_id, err, block_root, index, topic); + self.handle_partial_verification_error( + peer_id, err, block_root, index, None, topic, + ); } } @@ -916,6 +919,7 @@ impl NetworkBeaconProcessor { err: GossipDataColumnError, block_root: Hash256, index: ColumnIndex, + partial_data_column: Option>, topic: GossipTopic, ) { match err { @@ -924,23 +928,40 @@ impl NetworkBeaconProcessor { debug!( %block_root, %index, - "Invalid gossip data column variant." + "Invalid gossip partial data column variant." ) } GossipDataColumnError::PriorKnownUnpublished => { debug!( %block_root, %index, - "Gossip data column already processed via the EL." + "Gossip partial data column already processed via the EL." ); } - // ParentUnknown can't really happen, so we treat it like an internal error - GossipDataColumnError::ParentUnknown { .. } - | GossipDataColumnError::PubkeyCacheTimeout + GossipDataColumnError::ParentUnknown { parent_root } => { + if let Some(partial_data_column) = partial_data_column { + debug!( + action = "requesting parent", + %block_root, + %parent_root, + "Unknown parent hash for partial column" + ); + self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( + peer_id, + partial_data_column, + )); + } else { + crit!( + %parent_root, + "Unknown parent hash for partial column - but got no partial column to reprocess" + ) + } + } + GossipDataColumnError::PubkeyCacheTimeout | GossipDataColumnError::BeaconChainError(_) => { crit!( error = ?err, - "Internal error when verifying column sidecar" + "Internal error when verifying partial column sidecar" ) } GossipDataColumnError::ProposalSignatureInvalid @@ -977,7 +998,7 @@ impl NetworkBeaconProcessor { debug!( %block_root, %index, - "Received already available column sidecar. Ignoring the column sidecar" + "Received already available column sidecar. Ignoring the partial column sidecar" ) } GossipDataColumnError::FutureSlot { .. } @@ -986,7 +1007,7 @@ impl NetworkBeaconProcessor { error = ?err, %block_root, %index, - "Could not verify column sidecar for gossip. Ignoring the column sidecar" + "Could not verify column sidecar for gossip. Ignoring the partial column sidecar" ); // Prevent recurring behaviour by penalizing the peer slightly. self.gossip_penalize_peer( diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index cbf65505efd..0d395b78bb2 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -45,7 +45,7 @@ use std::sync::Arc; use std::time::Duration; use store::Hash256; use tracing::{debug, error, warn}; -use types::{BlobSidecar, DataColumnSidecar, EthSpec, SignedBeaconBlock}; +use types::{EthSpec, SignedBeaconBlock}; pub mod common; pub mod parent_chain; @@ -77,22 +77,21 @@ const LOOKUP_MAX_DURATION_NO_PEERS_SECS: u64 = 10; /// take at most 2 GB. 200 lookups allow 3 parallel chains of depth 64 (current maximum). const MAX_LOOKUPS: usize = 200; +/// The values for `Blob`, `DataColumn` and `PartialDataColumn` is the parent root of the column. pub enum BlockComponent { Block(DownloadResult>>), - Blob(DownloadResult>>), - DataColumn(DownloadResult>>), + Blob(DownloadResult), + DataColumn(DownloadResult), + PartialDataColumn(DownloadResult), } impl BlockComponent { fn parent_root(&self) -> Hash256 { match self { BlockComponent::Block(block) => block.value.parent_root(), - BlockComponent::Blob(blob) => blob.value.block_parent_root(), - BlockComponent::DataColumn(column) => match column.value.as_ref() { - DataColumnSidecar::Fulu(column) => column.block_parent_root(), - // TODO(gloas) we don't have a parent root post gloas, not sure what to do here - DataColumnSidecar::Gloas(column) => column.beacon_block_root, - }, + BlockComponent::Blob(parent_root) + | BlockComponent::DataColumn(parent_root) + | BlockComponent::PartialDataColumn(parent_root) => parent_root.value, } } fn get_type(&self) -> &'static str { @@ -100,6 +99,7 @@ impl BlockComponent { BlockComponent::Block(_) => "block", BlockComponent::Blob(_) => "blob", BlockComponent::DataColumn(_) => "data_column", + BlockComponent::PartialDataColumn(_) => "partial_data_column", } } } diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 919526c2386..23bfd531f0f 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -156,7 +156,9 @@ impl SingleBlockLookup { .block_request_state .state .insert_verified_response(block), - BlockComponent::Blob(_) | BlockComponent::DataColumn(_) => { + BlockComponent::Blob(_) + | BlockComponent::DataColumn(_) + | BlockComponent::PartialDataColumn(_) => { // For now ignore single blobs and columns, as the blob request state assumes all blobs are // attributed to the same peer = the peer serving the remaining blobs. Ignoring this // block component has a minor effect, causing the node to re-request this blob diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index c1ab6221dd2..b06799cd1ac 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -74,7 +74,8 @@ use strum::IntoStaticStr; use tokio::sync::mpsc; use tracing::{debug, error, info, trace}; use types::{ - BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, SignedBeaconBlock, Slot, + BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, PartialDataColumn, + SignedBeaconBlock, Slot, }; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync @@ -142,6 +143,9 @@ pub enum SyncMessage { /// A data column with an unknown parent has been received. UnknownParentDataColumn(PeerId, Arc>), + /// A partial data column with an unknown parent has been received. + UnknownParentPartialDataColumn(PeerId, PartialDataColumn), + /// A peer has sent an attestation that references a block that is unknown. This triggers the /// manager to attempt to find the block matching the unknown hash. UnknownBlockHashFromAttestation(PeerId, Hash256), @@ -864,7 +868,7 @@ impl SyncManager { parent_root, blob_slot, BlockComponent::Blob(DownloadResult { - value: blob, + value: parent_root, block_root, seen_timestamp: timestamp_now(), peer_group: PeerGroup::from_single(peer_id), @@ -884,7 +888,7 @@ impl SyncManager { parent_root, data_column_slot, BlockComponent::DataColumn(DownloadResult { - value: data_column, + value: parent_root, block_root, seen_timestamp: timestamp_now(), peer_group: PeerGroup::from_single(peer_id), @@ -897,6 +901,29 @@ impl SyncManager { } } } + SyncMessage::UnknownParentPartialDataColumn(peer_id, partial_data_column) => { + let block_root = partial_data_column.block_root; + let Some(header) = partial_data_column.sidecar.header.first() else { + // This should not happen - we need to know the header to know that we don't know the parent + crit!(%block_root, "Got unknown parent partial column without header"); + return; + }; + let data_column_slot = header.slot(); + let parent_root = header.signed_block_header.message.parent_root; + debug!(%block_root, %parent_root, "Received unknown parent partial column message"); + self.handle_unknown_parent( + peer_id, + block_root, + parent_root, + data_column_slot, + BlockComponent::PartialDataColumn(DownloadResult { + value: parent_root, + block_root, + seen_timestamp: timestamp_now(), + peer_group: PeerGroup::from_single(peer_id), + }), + ); + } SyncMessage::UnknownBlockHashFromAttestation(peer_id, block_root) => { if !self.notified_unknown_roots.contains(&(peer_id, block_root)) { self.notified_unknown_roots.insert((peer_id, block_root)); From 0651fc37719bfdd1a9af08ebc9a45bced458aa7f Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 25 Feb 2026 02:58:17 +0100 Subject: [PATCH 093/146] update libp2p --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc1e6184443..eab4276a721 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 1.0.109", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4967,7 +4967,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "bytes", "either", @@ -4998,7 +4998,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5008,7 +5008,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5018,7 +5018,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "either", "fnv", @@ -5042,7 +5042,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "async-trait", "futures", @@ -5057,7 +5057,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5077,7 +5077,7 @@ dependencies = [ "prometheus-client", "quick-protobuf", "quick-protobuf-codec", - "rand 0.9.2", + "rand 0.8.5", "regex", "sha2", "tracing", @@ -5087,7 +5087,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "asynchronous-codec", "either", @@ -5127,7 +5127,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "hickory-proto", @@ -5145,7 +5145,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "libp2p-core", @@ -5161,7 +5161,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "asynchronous-codec", "bytes", @@ -5179,7 +5179,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "asynchronous-codec", "bytes", @@ -5201,7 +5201,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "futures-timer", @@ -5222,7 +5222,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "either", "fnv", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "heck", "quote", @@ -5253,7 +5253,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "futures-timer", @@ -5268,7 +5268,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "futures-rustls", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "futures-timer", @@ -5300,7 +5300,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "either", "futures", @@ -5975,7 +5975,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "bytes", "futures", @@ -6201,7 +6201,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7071,7 +7071,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "asynchronous-codec", "bytes", @@ -7133,7 +7133,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7643,7 +7643,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7749,7 +7749,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#05350ddbbefd673229a6280fbdd3dcacf9647f54" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dc2b0a429b5f6389944a4424bc47f4c5dd1628f9" dependencies = [ "futures", "pin-project", @@ -8689,7 +8689,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10021,7 +10021,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 76c9e9278e10cf47a9717a831ae2166f5a42bae8 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 26 Feb 2026 16:13:17 +0100 Subject: [PATCH 094/146] fix inverted logic for sending --- .../lighthouse_network/src/service/mod.rs | 19 ++++--------------- .../lighthouse_network/src/types/partial.rs | 9 ++++++++- .../src/data/partial_data_column_sidecar.rs | 7 ------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index d623d231330..10f8909c456 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -36,7 +36,6 @@ use libp2p::upnp::tokio::Behaviour as Upnp; use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; -use ssz::Decode; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; use std::pin::Pin; @@ -44,8 +43,8 @@ use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; use types::{ - CellBitmap, ChainSpec, DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, ForkName, - PartialDataColumn, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, + ChainSpec, DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, ForkName, PartialDataColumn, + Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, }; use utils::{Context as ServiceContext, build_transport, strip_peer_id}; @@ -1395,7 +1394,7 @@ impl Network { peer_id, group_id, message, - metadata, + .. } => { let topic = GossipTopic::decode(topic_hash.as_str()) .inspect_err(|error| { @@ -1407,17 +1406,6 @@ impl Network { }) .ok()?; - // TODO(dknopik): Remove as soon as partial messages stable? - if let Some(metadata) = metadata { - if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { - debug!(%metadata, %topic, %peer_id, "Got metadata") - } else { - warn!(?metadata, %topic, %peer_id, "Got weird metadata"); - } - } else { - debug!(%peer_id, "Got no metadata") - } - if let Some(message) = message { match decode_partial::(&topic, &group_id, &message) { Err(error) => { @@ -1435,6 +1423,7 @@ impl Network { block_root = %column.block_root, index = column.index, %peer_id, + cells_present = %column.sidecar.cells_present_bitmap, "Decoded partial message" ); // Notify the network diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 3446b05e46a..de471cf9fc0 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -196,6 +196,13 @@ impl Partial for OutgoingPartialColumn { Some(metadata) => { let peer_metadata = PartialDataColumnPartsMetadata::::from_ssz_bytes(metadata) .map_err(|_| PartialError::InvalidFormat)?; + let expected_len = self.partial_column.sidecar.cells_present_bitmap.len(); + if peer_metadata.available.len() != expected_len + || peer_metadata.request.len() != expected_len + { + return Err(PartialError::InvalidFormat); + } + let need = !peer_metadata .available .is_subset(&self.partial_column.sidecar.cells_present_bitmap); @@ -204,7 +211,7 @@ impl Partial for OutgoingPartialColumn { let send = self .partial_column .sidecar - .with_missing_cells(&want) + .clone_filter(|idx| want.get(idx).expect("Bound checked above")) .map(|sidecar| { debug!( peer=%peer_id, diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index d88273bf984..073c028e653 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -73,13 +73,6 @@ impl PartialDataColumnSidecar { self.cells_present_bitmap.iter().all(|bit| bit) } - pub fn with_missing_cells(&self, bitmap: &CellBitmap) -> Option { - if self.cells_present_bitmap.len() != bitmap.len() { - return None; - } - self.clone_filter(|idx| !bitmap.get(idx).expect("Bounds checked above")) - } - /// Creates a new partial data column sidecar containing only the blob indices for which the /// passed closure returns `true` and were present in `self`. Will return `None` if there is no /// overlap. From bc00446621f74123adf8ef42e4c63db80b66b102 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 9 Mar 2026 10:19:39 +0100 Subject: [PATCH 095/146] Add metric for headerless partial verification --- beacon_node/network/src/metrics.rs | 8 ++++++++ .../src/network_beacon_processor/gossip_methods.rs | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 4d530b6a2ea..b09dc95db4a 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -151,6 +151,14 @@ pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: L "Total number of gossip partial data column sidecar verified for propagation.", ) }); +pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_MISSING_HEADER_TOTAL: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_int_counter( + "beacon_processor_gossip_partial_data_column_missing_header_total", + "Total number of gossip partial data column sidecar received without a (cached) header.", + ) +}); // Gossip Exits. pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e1a6c14dc14..f26237b15ff 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -848,7 +848,10 @@ impl NetworkBeaconProcessor { .partial_assembler() .get_header(&column.block_root) else { - error!(block=%column.block_root, index=%column.index, "Received partial column while not having header stored"); + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_MISSING_HEADER_TOTAL, + ); + warn!(block=%column.block_root, index=%column.index, "Received partial column while not having header stored"); return; }; column.sidecar.header = VariableList::repeat_full((*header).clone()); From 6e23ad55f0ed885bb4fb2d0942b2864e5ae6712b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 9 Mar 2026 10:19:39 +0100 Subject: [PATCH 096/146] ensure atomic and guaranteed setting of has_local_blobs --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../beacon_chain/src/fetch_blobs/mod.rs | 2 +- .../src/partial_data_column_assembler.rs | 52 ++++++++++--------- .../src/network_beacon_processor/mod.rs | 2 +- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5b440d4fca8..124ae68c96c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3211,7 +3211,7 @@ impl BeaconChain { let merge_result = self .data_availability_checker .partial_assembler() - .merge_partials(block_root, vec![verified_partial], false) + .merge_partials(block_root, vec![verified_partial]) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; metrics::inc_counter_vec_by( diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 005382f9578..d4f3d1addfe 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -331,7 +331,7 @@ async fn fetch_and_process_blobs_v2_or_v3( // Initialize the partial assembler with the columns from the engine let assembler = chain_adapter.partial_assembler(); let merge_result = assembler - .merge_partials(block_root, custody_columns_to_import, true) + .merge_partials(block_root, custody_columns_to_import) .ok_or_else(|| { FetchEngineBlobError::InternalError( "Failed to merge partials into assembler".to_string(), diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 9ec73054136..8932b2b1bee 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -82,7 +82,6 @@ impl PartialDataColumnAssembler { &self, block_root: Hash256, partials: Vec>, - local_blobs: bool, ) -> Option> { let mut assemblies = self.assemblies.write(); let assembly = assemblies @@ -93,7 +92,7 @@ impl PartialDataColumnAssembler { .next() .map(|header| PartialAssembly { header: header.clone(), - has_local_blobs: local_blobs, + has_local_blobs: false, columns: HashMap::new(), }) .ok_or(()) @@ -155,10 +154,6 @@ impl PartialDataColumnAssembler { updated_partials.push(merged); } - if local_blobs { - assembly.has_local_blobs = true; - } - Some(PartialMergeResult { added_cells, local_blobs: assembly.has_local_blobs, @@ -200,26 +195,35 @@ impl PartialDataColumnAssembler { .cloned() } - /// Get all current partials for a block - pub fn get_partials( + /// Get all current partials for a block for publishing after fetching local blobs. + /// To unlock future publishing, mark blobs as fetched locally. + /// We do this within one write lock to avoid useless double publishes. + pub fn get_partials_and_mark_as_local_fetched( &self, - block_root: &Hash256, + block_root: Hash256, + header: &PartialDataColumnHeader, ) -> Option>> { - Some( - self.assemblies - .read() - .peek(block_root)? - .columns - .values() - .filter_map(|value| { - if let AssemblyColumn::Incomplete(partial) = value { - Some(partial.clone()) - } else { - None - } - }) - .collect(), - ) + let mut assemblies = self.assemblies.write(); + let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { + header: header.clone(), + has_local_blobs: true, + columns: Default::default(), + }); + + assembly.has_local_blobs = true; + + let columns = assembly + .columns + .values() + .filter_map(|value| { + if let AssemblyColumn::Incomplete(partial) = value { + Some(partial.clone()) + } else { + None + } + }) + .collect(); + Some(columns) } /// Get header for a block if we have an active assembly diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 917a00b26de..971e9988e46 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -957,7 +957,7 @@ impl NetworkBeaconProcessor { .chain .data_availability_checker .partial_assembler() - .get_partials(&block_root) + .get_partials_and_mark_as_local_fetched(block_root, header) { debug!(block = %block_root, "Publishing all partials after getBlobs"); self.send_network_message(NetworkMessage::PublishPartial { From 994177788fcc1e7d9b30e6d6d44531cb8ca35515 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 13 Mar 2026 12:16:52 +0100 Subject: [PATCH 097/146] move send log to more sensible location --- beacon_node/lighthouse_network/src/service/mod.rs | 5 +++++ beacon_node/network/src/service.rs | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 10f8909c456..f5b92b0cfee 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -927,6 +927,11 @@ impl Network { return; } + debug!( + count = columns.len(), + "Sending partial data column sidecars" + ); + for column in columns { let subnet = DataColumnSubnetId::from_column_index(column.index, &self.fork_context.spec); diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 2e3198042fd..9d5d25204cf 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -683,10 +683,6 @@ impl NetworkService { self.libp2p.publish(messages); } NetworkMessage::PublishPartial { columns } => { - debug!( - count = columns.len(), - "Sending partial data column sidecars" - ); self.libp2p.publish_partial(columns); } NetworkMessage::ReportPeer { From a2643c90f8d698a17f4618e2f7d2950f3582c8e0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 18 Mar 2026 11:52:28 +0100 Subject: [PATCH 098/146] update libp2p --- Cargo.lock | 62 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbfc51104e9..5c9f6a1ea67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1965,7 +1965,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 1.0.109", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4956,7 +4956,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "bytes", "either", @@ -4987,7 +4987,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4997,7 +4997,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5007,7 +5007,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "either", "fnv", @@ -5031,7 +5031,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "async-trait", "futures", @@ -5046,7 +5046,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5076,7 +5076,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "asynchronous-codec", "either", @@ -5116,7 +5116,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "hickory-proto", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "libp2p-core", @@ -5150,7 +5150,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "asynchronous-codec", "bytes", @@ -5168,7 +5168,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "asynchronous-codec", "bytes", @@ -5190,7 +5190,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "futures-timer", @@ -5211,7 +5211,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "either", "fnv", @@ -5232,7 +5232,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "heck", "quote", @@ -5242,7 +5242,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "futures-timer", @@ -5257,7 +5257,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "futures-rustls", @@ -5275,7 +5275,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "futures-timer", @@ -5289,7 +5289,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "either", "futures", @@ -5964,7 +5964,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "bytes", "futures", @@ -6190,7 +6190,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7060,7 +7060,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "asynchronous-codec", "bytes", @@ -7082,7 +7082,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -7117,9 +7117,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7641,7 +7641,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7747,7 +7747,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#6ec13d75c3310092b6e885513072e2a039716278" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" dependencies = [ "futures", "pin-project", @@ -8699,7 +8699,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10024,7 +10024,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] From b75f5030f3a04979c3c78d291653cb450d8c611c Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 19 Mar 2026 17:24:15 +0100 Subject: [PATCH 099/146] refactor: encode without cloning, no cell level arc, better validation flow --- beacon_node/beacon_chain/src/beacon_chain.rs | 29 +- .../src/data_availability_checker.rs | 23 +- .../src/data_column_verification.rs | 182 +++++++--- .../beacon_chain/src/fetch_blobs/mod.rs | 17 +- .../beacon_chain/src/fetch_blobs/tests.rs | 24 +- beacon_node/beacon_chain/src/kzg_utils.rs | 27 +- .../src/partial_data_column_assembler.rs | 56 +-- beacon_node/http_api/src/publish_blocks.rs | 30 +- .../lighthouse_network/src/service/mod.rs | 13 +- .../lighthouse_network/src/types/partial.rs | 31 +- .../gossip_methods.rs | 320 +++++++++--------- .../src/network_beacon_processor/mod.rs | 9 +- .../network_beacon_processor/sync_methods.rs | 13 +- beacon_node/network/src/service.rs | 11 +- .../types/src/data/data_column_sidecar.rs | 10 +- consensus/types/src/data/mod.rs | 2 +- .../src/data/partial_data_column_sidecar.rs | 65 ++-- 17 files changed, 469 insertions(+), 393 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e6d2ecdf97a..3976c1c58e9 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -23,8 +23,9 @@ use crate::data_availability_checker::{ DataAvailabilityChecker, DataColumnReconstructionResult, }; use crate::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedPartialDataColumnHeader, - KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, + GossipVerifiedPartialDataColumnHeader, KzgVerifiedCustodyPartialDataColumn, + KzgVerifiedPartialDataColumn, PartialColumnVerificationResult, validate_partial_data_column_sidecar_for_gossip, }; use crate::early_attester_cache::EarlyAttesterCache; @@ -2244,10 +2245,11 @@ impl BeaconChain { } pub fn verify_partial_data_column_header_for_gossip( - self: &Arc, + &self, block_root: Hash256, data_column_header: PartialDataColumnHeader, - ) -> Result, GossipDataColumnError> { + ) -> Result, GossipPartialDataColumnError> + { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_REQUESTS); let _timer = metrics::start_timer( &metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_GOSSIP_VERIFICATION_TIMES, @@ -2260,10 +2262,10 @@ impl BeaconChain { return if *cached_header == data_column_header { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_DUPES); Ok(GossipVerifiedPartialDataColumnHeader::new_from_cached( - data_column_header, + cached_header, )) } else { - Err(GossipDataColumnError::PartialHeaderMismatches) + Err(GossipPartialDataColumnError::HeaderMismatches) }; } @@ -2280,13 +2282,15 @@ impl BeaconChain { pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, data_column_sidecar: PartialDataColumn, - ) -> Result, GossipDataColumnError> { + ) -> PartialColumnVerificationResult { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); - validate_partial_data_column_sidecar_for_gossip(data_column_sidecar, self).inspect(|_| { + let ret = validate_partial_data_column_sidecar_for_gossip(data_column_sidecar, self); + if matches!(ret, PartialColumnVerificationResult::Ok { .. }) { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); - }) + } + ret } #[instrument(skip_all, level = "trace")] @@ -3176,6 +3180,7 @@ impl BeaconChain { pub async fn process_gossip_partial_data_column( self: &Arc, verified_partial: KzgVerifiedPartialDataColumn, + verified_header: GossipVerifiedPartialDataColumnHeader, slot: Slot, ) -> Result, BlockError> { let block_root = verified_partial.block_root(); @@ -3211,7 +3216,11 @@ impl BeaconChain { let merge_result = self .data_availability_checker .partial_assembler() - .merge_partials(block_root, vec![verified_partial]) + .merge_partials( + block_root, + vec![verified_partial], + verified_header.into_header(), + ) .ok_or_else(|| BlockError::InternalError("No assembly found for block".to_string()))?; metrics::inc_counter_vec_by( diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 48e766cfc86..88d13f754f4 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -21,8 +21,8 @@ use tracing::{debug, error, instrument}; use types::data::{BlobIdentifier, FixedBlobSidecarList}; use types::{ BlobSidecar, BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecar, - DataColumnSidecarList, Epoch, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, - SignedBeaconBlock, Slot, new_non_zero_usize, + DataColumnSidecarList, Epoch, EthSpec, Hash256, PartialDataColumn, SignedBeaconBlock, Slot, + new_non_zero_usize, }; mod error; @@ -417,7 +417,10 @@ impl DataAvailabilityChecker { executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { let block = executed_block.as_block(); - self.partial_assembler.init(block.canonical_root(), block); + if let Ok(header) = block.try_into() { + self.partial_assembler + .init(block.canonical_root(), Arc::new(header)); + } self.availability_cache.put_executed_block(executed_block) } @@ -429,21 +432,13 @@ impl DataAvailabilityChecker { block: Arc>, source: BlockImportSource, ) -> Result<(), Error> { - self.partial_assembler.init(block_root, block.as_ref()); + if let Ok(header) = block.as_ref().try_into() { + self.partial_assembler.init(block_root, Arc::new(header)); + } self.availability_cache .put_pre_execution_block(block_root, block, source) } - /// Inserts a partial data column header into the assembler. - /// Returns true if this was the first insertion. - pub fn put_partial_data_column_header( - &self, - header: PartialDataColumnHeader, - ) -> bool { - self.partial_assembler - .init(header.signed_block_header.message.canonical_root(), header) - } - /// Removes a pre-execution block from the cache. /// This does NOT remove an existing executed block. pub fn remove_block_on_execution_error(&self, block_root: &Hash256) { diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 4cf1d40da89..6f069893301 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -187,30 +187,60 @@ pub enum GossipDataColumnError { max_blobs_per_block: usize, commitments_len: usize, }, +} + +impl From for GossipDataColumnError { + fn from(e: BeaconChainError) -> Self { + GossipDataColumnError::BeaconChainError(e.into()) + } +} + +impl From for GossipDataColumnError { + fn from(e: BeaconStateError) -> Self { + GossipDataColumnError::BeaconChainError(BeaconChainError::BeaconStateError(e).into()) + } +} + +#[derive(Debug)] +pub enum GossipPartialDataColumnError { + GossipDataColumnError(GossipDataColumnError), + /// The partial data column does not contain a header, and we do not have it cached. + /// + /// ## Peer scoring + /// The peer SHOULD send us the header on the first partial message, but is not required to. + /// Still, the peer incorrectly assumed that we have the header, and sent us data we can not + /// process due to that. Penalize it slightly. + MissingHeader, /// The partial data column header does not match the valid one we have already cached. /// /// ## Peer scoring /// The column sidecar is invalid and the peer is faulty - PartialHeaderMismatches, + HeaderMismatches, /// The partial data column header block root does not match the group id. /// /// ## Peer scoring /// The column sidecar is invalid and the peer is faulty - PartialHeaderIncorrectRoot { + HeaderIncorrectRoot { group_id: Hash256, header_hash: Hash256, }, } -impl From for GossipDataColumnError { +impl From for GossipPartialDataColumnError { + fn from(e: GossipDataColumnError) -> Self { + GossipPartialDataColumnError::GossipDataColumnError(e) + } +} + +impl From for GossipPartialDataColumnError { fn from(e: BeaconChainError) -> Self { - GossipDataColumnError::BeaconChainError(e.into()) + GossipDataColumnError::from(e).into() } } -impl From for GossipDataColumnError { +impl From for GossipPartialDataColumnError { fn from(e: BeaconStateError) -> Self { - GossipDataColumnError::BeaconChainError(BeaconChainError::BeaconStateError(e).into()) + GossipDataColumnError::from(e).into() } } @@ -415,7 +445,7 @@ impl KzgVerifiedPartialDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct GossipVerifiedPartialDataColumnHeader { - header: PartialDataColumnHeader, + header: Arc>, previously_cached: bool, } @@ -424,15 +454,15 @@ impl GossipVerifiedPartialDataColumnHeader { group_id: Hash256, header: PartialDataColumnHeader, chain: &BeaconChain, - ) -> Result { + ) -> Result { let column_slot = header.slot(); if header.kzg_commitments.is_empty() { - return Err(GossipDataColumnError::UnexpectedDataColumn); + return Err(GossipDataColumnError::UnexpectedDataColumn.into()); } let header_hash = header.signed_block_header.message.canonical_root(); if group_id != header_hash { - return Err(GossipDataColumnError::PartialHeaderIncorrectRoot { + return Err(GossipPartialDataColumnError::HeaderIncorrectRoot { group_id, header_hash, }); @@ -448,10 +478,13 @@ impl GossipVerifiedPartialDataColumnHeader { verify_slot_higher_than_parent(&parent_block, column_slot)?; verify_proposer_and_signature(&header.signed_block_header, &parent_block, chain)?; + let header = Arc::new(header); + // Cache the valid header let newly_cached = chain .data_availability_checker - .put_partial_data_column_header(header.clone()); + .partial_assembler() + .init(group_id, header.clone()); chain .observed_slashable @@ -461,7 +494,7 @@ impl GossipVerifiedPartialDataColumnHeader { header.signed_block_header.message.proposer_index, header_hash, ) - .map_err(|e| GossipDataColumnError::BeaconChainError(Box::new(e.into())))?; + .map_err(BeaconChainError::from)?; Ok(Self { header, @@ -469,7 +502,7 @@ impl GossipVerifiedPartialDataColumnHeader { }) } - pub fn new_from_cached(header: PartialDataColumnHeader) -> Self { + pub fn new_from_cached(header: Arc>) -> Self { Self { header, previously_cached: true, @@ -479,6 +512,14 @@ impl GossipVerifiedPartialDataColumnHeader { pub fn was_cached(&self) -> bool { self.previously_cached } + + pub fn as_header(&self) -> &PartialDataColumnHeader { + &self.header + } + + pub fn into_header(self) -> Arc> { + self.header + } } pub type CustodyDataColumnList = @@ -531,17 +572,6 @@ impl KzgVerifiedCustodyDataColumn { } } - /// Verify a column already marked as custody column - pub fn new( - data_column: CustodyDataColumn, - kzg: &Kzg, - ) -> Result, KzgError)> { - verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; - Ok(Self { - data: data_column.data, - }) - } - pub fn reconstruct_columns( kzg: &Kzg, partial_set_of_columns: &[Self], @@ -649,10 +679,24 @@ pub fn verify_kzg_for_data_column( #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( data_column: PartialDataColumn, + unverified_cells: Vec, + header: &GossipVerifiedPartialDataColumnHeader, kzg: &Kzg, -) -> Result, (Option, KzgError)> { +) -> Result, GossipPartialDataColumnError> { + let Some(filtered_column) = data_column + .sidecar + .filter(|idx| unverified_cells.contains(&idx)) + else { + return Err(GossipDataColumnError::PriorKnownUnpublished.into()); + }; + let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); - validate_partial_data_columns(kzg, iter::once(&data_column))?; + validate_partial_data_columns( + kzg, + iter::once((data_column.index, filtered_column)), + header.header.kzg_commitments.as_ref(), + ) + .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; Ok(KzgVerifiedPartialDataColumn { data: data_column }) } @@ -744,32 +788,82 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( - data_column: PartialDataColumn, + mut column: PartialDataColumn, chain: &BeaconChain, -) -> Result, GossipDataColumnError> { - let block_root = data_column.block_root; - - let filtered_column = if let Some(missing_cells) = chain - .data_availability_checker - .determine_missing_cells_partial(&block_root, &data_column) - { - if missing_cells.is_empty() { - return Err(GossipDataColumnError::PriorKnownUnpublished); +) -> PartialColumnVerificationResult { + let block_root = column.block_root; + + // Remove the header (if any) to avoid wasted memory. + let header = column.sidecar.take_header(); + + let header = if let Some(header) = header { + // Header was sent, so it is required to be valid + match chain.verify_partial_data_column_header_for_gossip(block_root, header) { + Ok(verified) => verified, + Err(err) => { + return PartialColumnVerificationResult::Err(err); + } } - - data_column - .clone_filter(|idx| missing_cells.contains(&idx)) - .ok_or_else(|| GossipDataColumnError::PriorKnownUnpublished)? } else { - data_column + // There is no header, so we check if we have a cached one to use + let Some(header) = chain + .data_availability_checker + .partial_assembler() + .get_header(&column.block_root) + else { + return PartialColumnVerificationResult::Err( + GossipPartialDataColumnError::MissingHeader, + ); + }; + GossipVerifiedPartialDataColumnHeader::new_from_cached(header) + }; + + let Some(missing_cells) = chain + .data_availability_checker + .determine_missing_cells_partial(&block_root, &column) + else { + // TODO(dknopik): This shuld be a different error + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipDataColumnError::PriorKnownUnpublished.into(), + header, + }; }; + if missing_cells.is_empty() { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipDataColumnError::PriorKnownUnpublished.into(), + header, + }; + } + // We do not have to check block related data here, as we create the verifiable column from // gossip accepted block - let kzg = &chain.kzg; - verify_kzg_for_partial_data_column(filtered_column, kzg) - .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e)) + let column = match verify_kzg_for_partial_data_column(column, missing_cells, &header, kzg) { + Ok(column) => column, + Err(err) => { + return PartialColumnVerificationResult::ErrWithValidHeader { err, header }; + } + }; + + PartialColumnVerificationResult::Ok { column, header } +} + +/// The result of a `validate_partial_data_column_sidecar_for_gossip` call. Any headers returned +/// herein were cached during this call or previously cached. +pub enum PartialColumnVerificationResult { + /// Verification succeeded fully. + Ok { + column: KzgVerifiedPartialDataColumn, + header: GossipVerifiedPartialDataColumnHeader, + }, + /// Verification of the column failed, but the header is valid. + ErrWithValidHeader { + err: GossipPartialDataColumnError, + header: GossipVerifiedPartialDataColumnHeader, + }, + /// Verification of the column or header failed, and no valid header was cached previously. + Err(GossipPartialDataColumnError), } /// Verify if the data column sidecar is valid. diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index d4f3d1addfe..8e2723e7209 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -68,7 +68,7 @@ pub enum FetchEngineBlobError { pub async fn fetch_and_process_engine_blobs( chain: Arc>, block_root: Hash256, - header: &PartialDataColumnHeader, + header: Arc>, custody_columns: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, ) -> Result, FetchEngineBlobError> { @@ -87,7 +87,7 @@ pub async fn fetch_and_process_engine_blobs( async fn fetch_and_process_engine_blobs_inner( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, - header: &PartialDataColumnHeader, + header: Arc>, custody_columns: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, ) -> Result, FetchEngineBlobError> { @@ -123,7 +123,7 @@ async fn fetch_and_process_engine_blobs_inner( fetch_and_process_blobs_v1( chain_adapter, block_root, - header, + &header, versioned_hashes, publish_fn, ) @@ -222,12 +222,13 @@ async fn fetch_and_process_blobs_v1( async fn fetch_and_process_blobs_v2_or_v3( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, - header: &PartialDataColumnHeader, + header: Arc>, versioned_hashes: Vec, custody_columns_indices: &[ColumnIndex], publish_fn: impl Fn(EngineGetBlobsOutput) + Send + 'static, ) -> Result, FetchEngineBlobError> { let num_expected_blobs = versioned_hashes.len(); + let slot = header.slot(); metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); @@ -314,7 +315,7 @@ async fn fetch_and_process_blobs_v2_or_v3( let custody_columns_to_import = compute_custody_columns_to_import( &chain_adapter, block_root, - header, + &header, blobs_and_proofs, custody_columns_indices, ) @@ -331,7 +332,7 @@ async fn fetch_and_process_blobs_v2_or_v3( // Initialize the partial assembler with the columns from the engine let assembler = chain_adapter.partial_assembler(); let merge_result = assembler - .merge_partials(block_root, custody_columns_to_import) + .merge_partials(block_root, custody_columns_to_import, header) .ok_or_else(|| { FetchEngineBlobError::InternalError( "Failed to merge partials into assembler".to_string(), @@ -351,14 +352,14 @@ async fn fetch_and_process_blobs_v2_or_v3( let availability_processing_status = if !merge_result.full_columns.is_empty() { chain_adapter .process_engine_blobs( - header.slot(), + slot, block_root, EngineGetBlobsOutput::CustodyColumns(merge_result.full_columns), ) .await? } else { // No complete columns yet, still missing components - AvailabilityProcessingStatus::MissingComponents(header.slot(), block_root) + AvailabilityProcessingStatus::MissingComponents(slot, block_root) }; Ok(Some(availability_processing_status)) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 7bf425879d5..0701c5c2054 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -43,7 +43,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &(&block).try_into().unwrap(), + Arc::new((&block).try_into().unwrap()), &custody_columns, publish_fn, ) @@ -68,7 +68,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -96,7 +96,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -129,7 +129,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -168,7 +168,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -210,7 +210,7 @@ mod get_blobs_v2 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -278,7 +278,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(&block_no_blobs).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(&block_no_blobs).unwrap()), &custody_columns, publish_fn, ) @@ -305,7 +305,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -351,7 +351,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -391,7 +391,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -439,7 +439,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) @@ -483,7 +483,7 @@ mod get_blobs_v1 { let processing_status = fetch_and_process_engine_blobs_inner( mock_adapter, block_root, - &PartialDataColumnHeader::try_from(block.as_ref()).unwrap(), + Arc::new(PartialDataColumnHeader::try_from(block.as_ref()).unwrap()), &custody_columns, publish_fn, ) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index f33bab351c1..6da996d3048 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -12,9 +12,10 @@ use types::data::{ }; use types::kzg_ext::KzgCommitments; use types::{ - Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, - DataColumnSidecarGloas, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, - SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlindedBeaconBlock, Slot, + Blob, BlobSidecar, BlobSidecarList, ChainSpec, ColumnIndex, DataColumnSidecar, + DataColumnSidecarFulu, DataColumnSidecarGloas, DataColumnSidecarList, EthSpec, Hash256, + KzgCommitment, KzgProof, PartialDataColumnSidecarRef, SignedBeaconBlock, + SignedBeaconBlockHeader, SignedBlindedBeaconBlock, Slot, }; /// Converts a blob ssz FixedVector to a reference to a fixed-size array @@ -116,17 +117,15 @@ pub fn validate_full_data_columns<'a, E: EthSpec>( /// Partial columns may have missing cells, indicated by a bitmap. We only verify present cells. pub fn validate_partial_data_columns<'a, E: EthSpec>( kzg: &Kzg, - data_column_iter: impl Iterator>, + data_column_iter: impl Iterator)>, + kzg_commitments: &[KzgCommitment], ) -> Result<(), (Option, KzgError)> { let mut cells = Vec::new(); let mut proofs = Vec::new(); let mut column_indices = Vec::new(); let mut commitments = Vec::new(); - for data_column in data_column_iter { - let col_index = data_column.index; - let sidecar = &data_column.sidecar; - + for (col_index, sidecar) in data_column_iter { if sidecar.column.is_empty() { return Err((Some(col_index), KzgError::KzgVerificationFailed)); } @@ -134,15 +133,7 @@ pub fn validate_partial_data_columns<'a, E: EthSpec>( // Partial columns have a bitmap indicating present cells // We iterate over the bitmap and only process present cells let mut present_iterator = sidecar.column.iter().zip(sidecar.kzg_proofs.iter()); - for (present, commitment) in sidecar.cells_present_bitmap.iter().zip( - data_column - .sidecar - .header - .first() - .ok_or((Some(col_index), KzgError::PartialWithoutHeader))? - .kzg_commitments - .iter(), - ) { + for (present, commitment) in sidecar.cells_present_bitmap.iter().zip(kzg_commitments) { if present { let (cell, proof) = present_iterator.next().ok_or(( Some(col_index), @@ -550,7 +541,7 @@ pub(crate) fn build_partial_data_columns( .get_mut(col) .ok_or(format!("Missing data column proofs at index {col}"))?; - column.push(Arc::new(cell)); + column.push(cell); column_proofs.push(*proof); } } diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 8932b2b1bee..af0aa2f60f3 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -3,10 +3,8 @@ use crate::data_column_verification::{ }; use lru::LruCache; use parking_lot::RwLock; -use parking_lot::lock_api::RwLockReadGuard; use std::collections::HashMap; use std::num::NonZeroUsize; -use std::ops::Deref; use std::sync::Arc; use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; use types::{DataColumnSidecarFulu, EthSpec, Hash256}; @@ -19,7 +17,7 @@ pub struct PartialDataColumnAssembler { /// Tracks partial columns being assembled for a single block struct PartialAssembly { - header: PartialDataColumnHeader, + header: Arc>, has_local_blobs: bool, /// Map of column_index -> partial column being assembled columns: HashMap>, @@ -51,20 +49,13 @@ impl PartialDataColumnAssembler { } /// Returns true unless the header was already contained or the passed argument failed to convert to a header - pub fn init(&self, block_root: Hash256, header: T) -> bool - where - T: TryInto>, - { + pub fn init(&self, block_root: Hash256, header: Arc>) -> bool { let mut assemblies = self.assemblies.write(); if assemblies.contains(&block_root) { return false; } - let Ok(header) = header.try_into() else { - return false; - }; - let assembly = PartialAssembly { header, has_local_blobs: false, @@ -76,28 +67,20 @@ impl PartialDataColumnAssembler { true } - /// Merge a received partial column into the assembly. - /// Returns the merge result indicating if column is now complete. + /// Merge one or more received partial columns into the assembly. + /// Returns the merge result indicating if the columns are now complete. pub fn merge_partials( &self, block_root: Hash256, partials: Vec>, + header: Arc>, ) -> Option> { let mut assemblies = self.assemblies.write(); - let assembly = assemblies - .try_get_or_insert_mut(block_root, || { - partials - .iter() - .filter_map(|partial| partial.as_data_column().sidecar.header.first()) - .next() - .map(|header| PartialAssembly { - header: header.clone(), - has_local_blobs: false, - columns: HashMap::new(), - }) - .ok_or(()) - }) - .ok()?; + let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { + header: header.clone(), + has_local_blobs: false, + columns: HashMap::new(), + }); let mut full_columns = Vec::new(); let mut updated_partials = Vec::new(); @@ -167,11 +150,11 @@ impl PartialDataColumnAssembler { pub fn mark_as_complete(&self, block_root: Hash256, column: &DataColumnSidecarFulu) -> bool { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { - header: PartialDataColumnHeader { + header: Arc::new(PartialDataColumnHeader { kzg_commitments: column.kzg_commitments.clone(), signed_block_header: column.signed_block_header.clone(), kzg_commitments_inclusion_proof: column.kzg_commitments_inclusion_proof.clone(), - }, + }), has_local_blobs: false, columns: Default::default(), }); @@ -201,7 +184,7 @@ impl PartialDataColumnAssembler { pub fn get_partials_and_mark_as_local_fetched( &self, block_root: Hash256, - header: &PartialDataColumnHeader, + header: &Arc>, ) -> Option>> { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { @@ -227,14 +210,11 @@ impl PartialDataColumnAssembler { } /// Get header for a block if we have an active assembly - pub fn get_header( - &self, - block_root: &Hash256, - ) -> Option>> { - RwLockReadGuard::try_map(self.assemblies.read(), |assemblies| { - assemblies.peek(block_root).map(|a| &a.header) - }) - .ok() + pub fn get_header(&self, block_root: &Hash256) -> Option>> { + self.assemblies + .read() + .peek(block_root) + .map(|a| a.header.clone()) } /// Maintenance: remove assemblies older than cutoff epoch diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b86466eebfe..f1ba81691c9 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -16,6 +16,7 @@ use eth2::types::{ use execution_layer::{ProvenancedPayload, SubmitBlindedBlockResponse}; use futures::TryFutureExt; use lighthouse_network::PubsubMessage; +use logging::crit; use network::NetworkMessage; use rand::prelude::SliceRandom; use reqwest::StatusCode; @@ -520,10 +521,16 @@ fn publish_column_sidecars( } let mut full_messages = Vec::new(); let mut partial_columns = Vec::new(); + let mut partial_header = None; for data_col in data_column_sidecars { if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { - partial_columns.push(Arc::new(fulu_data_col.to_partial())); + let mut partial = fulu_data_col.to_partial(); + let header = partial.sidecar.take_header(); + if let Some(header) = header { + partial_header = Some(header); + } + partial_columns.push(Arc::new(partial)); } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); @@ -541,13 +548,20 @@ fn publish_column_sidecars( // Publish partial messages if !partial_columns.is_empty() { - crate::utils::publish_network_message( - sender_clone, - NetworkMessage::PublishPartial { - columns: partial_columns, - }, - ) - .map_err(|_| BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish)))?; + if let Some(header) = partial_header { + crate::utils::publish_network_message( + sender_clone, + NetworkMessage::PublishPartialColumns { + columns: partial_columns, + header: Arc::new(header), + }, + ) + .map_err(|_| { + BlockError::BeaconChainError(Box::new(BeaconChainError::UnableToPublish)) + })?; + } else { + crit!("Unable to extract header from full columns") + } } Ok(()) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index f5b92b0cfee..7b8f8871a3b 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -44,7 +44,7 @@ use std::time::Duration; use tracing::{debug, error, info, trace, warn}; use types::{ ChainSpec, DataColumnSubnetId, EnrForkId, EthSpec, ForkContext, ForkName, PartialDataColumn, - Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, + PartialDataColumnHeader, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, }; use utils::{Context as ServiceContext, build_transport, strip_peer_id}; @@ -922,7 +922,11 @@ impl Network { } /// Publishes partial data column sidecars to the gossipsub network. - pub fn publish_partial(&mut self, columns: Vec>>) { + pub fn publish_partial( + &mut self, + columns: Vec>>, + header: Arc>, + ) { if !self.network_globals.config.enable_partial_columns { return; } @@ -943,10 +947,7 @@ impl Network { let header_sent_set = self .partial_column_header_tracker .get_for_block(column.block_root); - let Ok(partial_message) = OutgoingPartialColumn::new(column, header_sent_set) else { - error!("Unable to publish partial column without header"); - continue; - }; + let partial_message = OutgoingPartialColumn::new(column, &header, header_sent_set); let publish_topic: Topic = topic.clone().into(); if let Err(e) = self diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index de471cf9fc0..7084193716d 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -3,21 +3,19 @@ use itertools::Itertools; use libp2p::gossipsub::partial_messages::{Metadata, Partial, PartialAction, PartialError}; use parking_lot::Mutex; use ssz::{Decode, Encode}; -use ssz_types::VariableList; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use tracing::debug; -use types::{CellBitmap, PartialDataColumn}; -use types::{EthSpec, Hash256}; -use types::{PartialDataColumnPartsMetadata, PartialDataColumnSidecar}; +use types::{ + CellBitmap, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, + PartialDataColumnPartsMetadata, PartialDataColumnSidecar, PartialDataColumnSidecarRef, +}; const PARTIAL_COLUMNS_VERSION_BYTE: u8 = 0; pub type HeaderSentSet = Arc>>; -pub struct NoHeaderInColumnError; - #[derive(Debug, Clone)] pub struct OutgoingPartialColumn { partial_column: Arc>, @@ -29,12 +27,9 @@ pub struct OutgoingPartialColumn { impl OutgoingPartialColumn { pub fn new( partial_column: Arc>, + header: &PartialDataColumnHeader, header_sent_set: HeaderSentSet, - ) -> Result { - let Some(header) = partial_column.sidecar.header.first().cloned() else { - return Err(NoHeaderInColumnError); - }; - + ) -> Self { // For now, always request all cells let mut request = partial_column.sidecar.cells_present_bitmap.clone(); for idx in 0..request.len() { @@ -48,23 +43,23 @@ impl OutgoingPartialColumn { } .into(); - let header_message = PartialDataColumnSidecar { + let header_message = PartialDataColumnSidecarRef { cells_present_bitmap: CellBitmap::::with_capacity( partial_column.sidecar.cells_present_bitmap.len(), ) .expect("Taking length from bitmap with same bound"), - column: VariableList::empty(), - kzg_proofs: VariableList::empty(), - header: VariableList::repeat_full(header), + column: vec![], + kzg_proofs: vec![], + header: vec![header], } .as_ssz_bytes(); - Ok(OutgoingPartialColumn { + OutgoingPartialColumn { partial_column, metadata, header_message, header_sent_set, - }) + } } } @@ -211,7 +206,7 @@ impl Partial for OutgoingPartialColumn { let send = self .partial_column .sidecar - .clone_filter(|idx| want.get(idx).expect("Bound checked above")) + .filter(|idx| want.get(idx).expect("Bound checked above")) .map(|sidecar| { debug!( peer=%peer_id, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 4910037d8cd..60fffbbdba1 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -6,7 +6,9 @@ use crate::{ }; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, KzgVerifiedPartialDataColumn, + GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, + GossipVerifiedPartialDataColumnHeader, KzgVerifiedPartialDataColumn, + PartialColumnVerificationResult, }; use beacon_chain::store::Error; use beacon_chain::{ @@ -35,7 +37,6 @@ use logging::crit; use operation_pool::ReceivedPreCapella; use slot_clock::SlotClock; use ssz::Encode; -use ssz_types::VariableList; use std::fs; use std::io::Write; use std::path::PathBuf; @@ -796,10 +797,6 @@ impl NetworkBeaconProcessor { MessageAcceptance::Ignore, ); } - GossipDataColumnError::PartialHeaderMismatches - | GossipDataColumnError::PartialHeaderIncorrectRoot { .. } => { - error!(error = ?err, "Unexpected error during full column verification"); - } } } } @@ -815,69 +812,25 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - mut column: PartialDataColumn, + column: PartialDataColumn, seen_duration: Duration, topic: GossipTopic, ) { - let mut get_blobs = false; - let header = if let Some(header) = column.sidecar.header.first() { - // Header was sent, so it is required to be valid - match self - .chain - .verify_partial_data_column_header_for_gossip(column.block_root, header.clone()) - { - Ok(verified) => { - get_blobs = !verified.was_cached(); - } - Err(err) => { - self.handle_partial_verification_error( - peer_id, - err, - column.block_root, - column.index, - Some(column), - topic, - ); - return; - } - } - header.clone() - } else { - // There is no header, so we check if we have a cached one to use - let Some(header) = self - .chain - .data_availability_checker - .partial_assembler() - .get_header(&column.block_root) - else { - metrics::inc_counter( - &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_MISSING_HEADER_TOTAL, - ); - warn!(block=%column.block_root, index=%column.index, "Received partial column while not having header stored"); - return; - }; - column.sidecar.header = VariableList::repeat_full((*header).clone()); - (*header).clone() - }; - let block_root = column.block_root; let index = column.index; - let slot = header.slot(); - let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); - // Log metrics to track delay from other nodes on the network. - metrics::observe_duration( - &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, - delay, - ); - match self + + let result = self .chain - .verify_partial_data_column_sidecar_for_gossip(column) - { - Ok(gossip_verified_data_column) => { + .verify_partial_data_column_sidecar_for_gossip(column); + + let header = match result { + PartialColumnVerificationResult::Ok { header, column } => { metrics::inc_counter( &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL, ); + let slot = header.as_header().slot(); + debug!( %slot, %block_root, @@ -899,133 +852,170 @@ impl NetworkBeaconProcessor { self.process_gossip_verified_partial_data_column( peer_id, - gossip_verified_data_column, + column, + header.clone(), slot, seen_duration, ) - .await + .await; + Some(header) } - Err(err) => { + PartialColumnVerificationResult::ErrWithValidHeader { header, err } => { self.handle_partial_verification_error( peer_id, err, block_root, index, None, topic, ); + Some(header) } - } + PartialColumnVerificationResult::Err(err) => { + self.handle_partial_verification_error( + peer_id, err, block_root, index, None, topic, + ); + None + } + }; - if get_blobs { - debug!(block = %block_root, "Triggering getBlobs after receiving partial header"); - // We want to publish immediately when this finishes - let publish_blobs = true; - self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) - .await + if let Some(header) = header { + let slot = header.as_header().slot(); + let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); + // Log metrics to track delay from other nodes on the network. + metrics::observe_duration( + &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, + delay, + ); + + if !header.was_cached() { + debug!(block = %block_root, "Triggering getBlobs after receiving partial header"); + // We want to publish immediately when this finishes + let publish_blobs = true; + self.fetch_engine_blobs_and_publish(header.into_header(), block_root, publish_blobs) + .await + } } } fn handle_partial_verification_error( self: &Arc, peer_id: PeerId, - err: GossipDataColumnError, + err: GossipPartialDataColumnError, block_root: Hash256, index: ColumnIndex, partial_data_column: Option>, topic: GossipTopic, ) { match err { - GossipDataColumnError::InvalidVariant => { - // TODO(gloas) we should probably penalize the peer here - debug!( - %block_root, - %index, - "Invalid gossip partial data column variant." - ) - } - GossipDataColumnError::PriorKnownUnpublished => { - debug!( - %block_root, - %index, - "Gossip partial data column already processed via the EL." - ); - } - GossipDataColumnError::ParentUnknown { parent_root } => { - if let Some(partial_data_column) = partial_data_column { + GossipPartialDataColumnError::GossipDataColumnError(err) => match err { + GossipDataColumnError::InvalidVariant => { + // TODO(gloas) we should probably penalize the peer here + debug!( + %block_root, + %index, + "Invalid gossip partial data column variant." + ) + } + GossipDataColumnError::PriorKnownUnpublished => { debug!( - action = "requesting parent", %block_root, - %parent_root, - "Unknown parent hash for partial column" + %index, + "Gossip partial data column already processed via the EL." ); - self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( - peer_id, - partial_data_column, - )); - } else { + } + GossipDataColumnError::ParentUnknown { parent_root } => { + if let Some(partial_data_column) = partial_data_column { + debug!( + action = "requesting parent", + %block_root, + %parent_root, + "Unknown parent hash for partial column" + ); + self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( + peer_id, + partial_data_column, + )); + } else { + crit!( + %parent_root, + "Unknown parent hash for partial column - but got no partial column to reprocess" + ) + } + } + GossipDataColumnError::PubkeyCacheTimeout + | GossipDataColumnError::BeaconChainError(_) => { crit!( - %parent_root, - "Unknown parent hash for partial column - but got no partial column to reprocess" + error = ?err, + "Internal error when verifying partial column sidecar" ) } - } - GossipDataColumnError::PubkeyCacheTimeout - | GossipDataColumnError::BeaconChainError(_) => { - crit!( - error = ?err, - "Internal error when verifying partial column sidecar" - ) - } - GossipDataColumnError::ProposalSignatureInvalid - | GossipDataColumnError::UnknownValidator(_) - | GossipDataColumnError::ProposerIndexMismatch { .. } - | GossipDataColumnError::IsNotLaterThanParent { .. } - | GossipDataColumnError::InvalidSubnetId { .. } - | GossipDataColumnError::InvalidInclusionProof - | GossipDataColumnError::InvalidKzgProof { .. } - | GossipDataColumnError::UnexpectedDataColumn - | GossipDataColumnError::InvalidColumnIndex(_) - | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } - | GossipDataColumnError::InconsistentCommitmentsLength { .. } - | GossipDataColumnError::InconsistentProofsLength { .. } - | GossipDataColumnError::NotFinalizedDescendant { .. } => { - debug!( - error = ?err, - %block_root, - %index, - "Could not verify partial column for gossip. Rejecting the column sidecar" - ); - // Prevent recurring behaviour by penalizing the peer slightly. - self.gossip_penalize_peer( - peer_id, - PeerAction::LowToleranceError, - "gossip_data_column_low", + GossipDataColumnError::ProposalSignatureInvalid + | GossipDataColumnError::UnknownValidator(_) + | GossipDataColumnError::ProposerIndexMismatch { .. } + | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::InvalidSubnetId { .. } + | GossipDataColumnError::InvalidInclusionProof + | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::UnexpectedDataColumn + | GossipDataColumnError::InvalidColumnIndex(_) + | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } + | GossipDataColumnError::InconsistentCommitmentsLength { .. } + | GossipDataColumnError::InconsistentProofsLength { .. } + | GossipDataColumnError::NotFinalizedDescendant { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify partial column for gossip. Rejecting the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_partial_data_column_low", + ); + self.propagate_partial_validation_failure(peer_id, topic); + } + GossipDataColumnError::PriorKnown { .. } => { + // Data column is available via either the EL or reconstruction. + // Do not penalise the peer. + // Gossip filter should filter any duplicates received after this. + debug!( + %block_root, + %index, + "Received already available column sidecar. Ignoring the partial column sidecar" + ) + } + GossipDataColumnError::FutureSlot { .. } + | GossipDataColumnError::PastFinalizedSlot { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify column sidecar for gossip. Ignoring the partial column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::HighToleranceError, + "gossip_partial_data_column_high", + ); + } + }, + GossipPartialDataColumnError::MissingHeader => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_MISSING_HEADER_TOTAL, ); - self.propagate_partial_validation_failure(peer_id, topic); - } - GossipDataColumnError::PriorKnown { .. } => { - // Data column is available via either the EL or reconstruction. - // Do not penalise the peer. - // Gossip filter should filter any duplicates received after this. - debug!( - %block_root, - %index, - "Received already available column sidecar. Ignoring the partial column sidecar" - ) - } - GossipDataColumnError::FutureSlot { .. } - | GossipDataColumnError::PastFinalizedSlot { .. } => { - debug!( + warn!( error = ?err, %block_root, %index, - "Could not verify column sidecar for gossip. Ignoring the partial column sidecar" + "Received partial column while not having header stored" ); - // Prevent recurring behaviour by penalizing the peer slightly. self.gossip_penalize_peer( peer_id, PeerAction::HighToleranceError, - "gossip_data_column_high", + "gossip_partial_data_column_high", ); } - GossipDataColumnError::PartialHeaderMismatches - | GossipDataColumnError::PartialHeaderIncorrectRoot { .. } => { + GossipPartialDataColumnError::HeaderMismatches + | GossipPartialDataColumnError::HeaderIncorrectRoot { .. } => { debug!( error = ?err, %block_root, @@ -1035,7 +1025,7 @@ impl NetworkBeaconProcessor { self.gossip_penalize_peer( peer_id, PeerAction::LowToleranceError, - "gossip_data_column_high", + "gossip_partial_data_column_high", ); } } @@ -1313,9 +1303,16 @@ impl NetworkBeaconProcessor { &[&data_column_index.to_string()], ); - self.send_network_message(NetworkMessage::PublishPartial { - columns: vec![Arc::new(col.to_partial())], - }); + let mut column = col.to_partial(); + let header = column.sidecar.take_header(); + if let Some(header) = header { + self.send_network_message(NetworkMessage::PublishPartialColumns { + columns: vec![Arc::new(column)], + header: Arc::new(header), + }); + } else { + crit!("Converting from full to partial yielded headerless partial") + }; } let result = self @@ -1387,6 +1384,7 @@ impl NetworkBeaconProcessor { self: &Arc, _peer_id: PeerId, verified_partial: KzgVerifiedPartialDataColumn, + verified_header: GossipVerifiedPartialDataColumnHeader, slot: Slot, _seen_duration: Duration, ) { @@ -1396,7 +1394,7 @@ impl NetworkBeaconProcessor { let result = self .chain - .process_gossip_partial_data_column(verified_partial, slot) + .process_gossip_partial_data_column(verified_partial, verified_header.clone(), slot) .await; // First, handle merge results (if any) @@ -1429,7 +1427,6 @@ impl NetworkBeaconProcessor { let only_send_completed_partials = merge_result.local_blobs || self.chain.config.disable_get_blobs; - debug!(block = %block_root, "Not incomplete publishing partials before getBlobs"); let columns = merge_result .updated_partials .into_iter() @@ -1440,7 +1437,16 @@ impl NetworkBeaconProcessor { .collect::>(); if !columns.is_empty() { - self.send_network_message(NetworkMessage::PublishPartial { columns }); + if only_send_completed_partials { + debug!( + block = %block_root, + "Not publishing incomplete partials before getBlobs" + ); + } + self.send_network_message(NetworkMessage::PublishPartialColumns { + columns, + header: verified_header.into_header(), + }); } Ok(avail) } @@ -1915,7 +1921,7 @@ impl NetworkBeaconProcessor { async move { if let Ok(header) = PartialDataColumnHeader::try_from(block_clone.as_ref()) { self_clone - .fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) + .fetch_engine_blobs_and_publish(Arc::new(header), block_root, publish_blobs) .await } } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index c780fb538c0..993d932dfa3 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -876,7 +876,7 @@ impl NetworkBeaconProcessor { pub async fn fetch_engine_blobs_and_publish( self: &Arc, - header: &PartialDataColumnHeader, + header: Arc>, block_root: Hash256, publish_blobs: bool, ) { @@ -908,7 +908,7 @@ impl NetworkBeaconProcessor { match fetch_and_process_engine_blobs( self.chain.clone(), block_root, - header, + header.clone(), custody_columns, publish_fn, ) @@ -958,14 +958,15 @@ impl NetworkBeaconProcessor { .chain .data_availability_checker .partial_assembler() - .get_partials_and_mark_as_local_fetched(block_root, header) + .get_partials_and_mark_as_local_fetched(block_root, &header) { debug!(block = %block_root, "Publishing all partials after getBlobs"); - self.send_network_message(NetworkMessage::PublishPartial { + self.send_network_message(NetworkMessage::PublishPartialColumns { columns: columns .into_iter() .map(|partial| partial.into_inner()) .collect(), + header, }); } } diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 0bd933e1c3f..8f89b669485 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use std::time::Duration; use store::KzgCommitment; use tracing::{debug, debug_span, error, info, instrument, warn}; -use types::data::{FixedBlobSidecarList, PartialDataColumnHeader}; +use types::data::FixedBlobSidecarList; use types::kzg_ext::format_kzg_commitments; use types::{BlockImportSource, DataColumnSidecarList, Epoch, Hash256}; @@ -218,11 +218,14 @@ impl NetworkBeaconProcessor { // Block is valid, we can now attempt fetching blobs from EL using version hashes // derived from kzg commitments from the block, without having to wait for all blobs // to be sent from the peers if we already have them. - if let Ok(header) = PartialDataColumnHeader::try_from(signed_beacon_block.as_ref()) - { + if let Ok(header) = signed_beacon_block.as_ref().try_into() { let publish_blobs = false; - self.fetch_engine_blobs_and_publish(&header, block_root, publish_blobs) - .await; + self.fetch_engine_blobs_and_publish( + Arc::new(header), + block_root, + publish_blobs, + ) + .await; } } _ => {} diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 9d5d25204cf..ce54ffc38fd 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -39,8 +39,8 @@ use tokio::time::Sleep; use tracing::{debug, error, info, trace, warn}; use typenum::Unsigned; use types::{ - EthSpec, ForkContext, PartialDataColumn, Slot, SubnetId, SyncCommitteeSubscription, - SyncSubnetId, ValidatorSubscription, + EthSpec, ForkContext, PartialDataColumn, PartialDataColumnHeader, Slot, SubnetId, + SyncCommitteeSubscription, SyncSubnetId, ValidatorSubscription, }; mod tests; @@ -84,8 +84,9 @@ pub enum NetworkMessage { /// Publish a list of messages to the gossipsub protocol. Publish { messages: Vec> }, /// Publish partial data column sidecars via the partial gossipsub protocol. - PublishPartial { + PublishPartialColumns { columns: Vec>>, + header: Arc>, }, /// Validates a received gossipsub message. This will propagate the message on the network. ValidationResult { @@ -682,8 +683,8 @@ impl NetworkService { ); self.libp2p.publish(messages); } - NetworkMessage::PublishPartial { columns } => { - self.libp2p.publish_partial(columns); + NetworkMessage::PublishPartialColumns { columns, header } => { + self.libp2p.publish_partial(columns, header); } NetworkMessage::ReportPeer { peer_id, diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 7dbe14060f1..3eb5b6e6c54 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -222,20 +222,12 @@ impl DataColumnSidecarFulu { let block_root = self.block_root(); - let column = self - .column - .iter() - .map(|col| Arc::new(col.clone())) - .collect::>() - .try_into() - .expect("our column has the same bound"); - PartialDataColumn { block_root, index: self.index, sidecar: PartialDataColumnSidecar { cells_present_bitmap: bitmap, - column, + column: self.column.clone(), kzg_proofs: self.kzg_proofs.clone(), header: VariableList::repeat_full(PartialDataColumnHeader { kzg_commitments: self.kzg_commitments.clone(), diff --git a/consensus/types/src/data/mod.rs b/consensus/types/src/data/mod.rs index c920d2c2ae2..40f25d896d8 100644 --- a/consensus/types/src/data/mod.rs +++ b/consensus/types/src/data/mod.rs @@ -20,7 +20,7 @@ pub use data_column_sidecar::{ pub use data_column_subnet_id::{DataColumnSubnetId, all_data_column_sidecar_subnets_from_spec}; pub use partial_data_column_sidecar::{ CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, - PartialDataColumnSidecar, + PartialDataColumnSidecar, PartialDataColumnSidecarRef, }; use crate::core::EthSpec; diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 073c028e653..e415013c31c 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -11,7 +11,7 @@ use ssz::{BitList, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use std::fmt::Display; -use std::sync::Arc; +use std::mem; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; @@ -28,17 +28,29 @@ pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnSidecar { pub cells_present_bitmap: CellBitmap, - pub column: VariableList>, ::MaxBlobCommitmentsPerBlock>, + pub column: VariableList, ::MaxBlobCommitmentsPerBlock>, pub kzg_proofs: VariableList, pub header: VariableList, U1>, } +/// Equivalent to `PartialDataColumnSidecar`, but containing references to the cells. This is done +/// so that we can get a part of a sidecar without expensively cloning all the contents. +#[derive(Debug, Clone, Encode)] +pub struct PartialDataColumnSidecarRef<'a, E: EthSpec> { + pub cells_present_bitmap: CellBitmap, + // It is fine to use `Vec` here as we never decode directly into this type, and only create + // this from the `PartialDataColumnSidecar` type above. This avoids a few ugly `expect` calls. + pub column: Vec<&'a Cell>, + pub kzg_proofs: Vec<&'a KzgProof>, + pub header: Vec<&'a PartialDataColumnHeader>, +} + impl PartialDataColumnSidecar { pub fn min_size() -> usize { // min size is one cell Self { cells_present_bitmap: BitList::with_capacity(1).unwrap(), - column: VariableList::new(vec![Arc::new(Cell::::default())]).unwrap(), + column: VariableList::new(vec![Cell::::default()]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), header: VariableList::new(vec![]).unwrap(), } @@ -73,16 +85,15 @@ impl PartialDataColumnSidecar { self.cells_present_bitmap.iter().all(|bit| bit) } - /// Creates a new partial data column sidecar containing only the blob indices for which the - /// passed closure returns `true` and were present in `self`. Will return `None` if there is no - /// overlap. - pub fn clone_filter(&self, filter: F) -> Option + /// Creates a reference to this sidecar containing only the blob indices for which the passed + /// closure returns `true` and is present in `self`. Will return `None` if there is no overlap. + pub fn filter(&self, filter: F) -> Option> where F: Fn(usize) -> bool, { let mut new_bitmap = self.cells_present_bitmap.clone(); - let mut new_column = VariableList::default(); - let mut new_proofs = VariableList::default(); + let mut new_column = Vec::new(); + let mut new_proofs = Vec::new(); let mut column_idx = 0; for (blob_idx, present) in self.cells_present_bitmap.iter().enumerate() { @@ -90,13 +101,9 @@ impl PartialDataColumnSidecar { if filter(blob_idx) { // Keep this cell let cell = self.column.get(column_idx)?; - new_column - .push(cell.clone()) - .expect("Has same capacity as existing column"); + new_column.push(cell); let proof = self.kzg_proofs.get(column_idx)?; - new_proofs - .push(*proof) - .expect("Has same capacity as existing column"); + new_proofs.push(proof); } else { // Mark as not present new_bitmap @@ -113,11 +120,11 @@ impl PartialDataColumnSidecar { return None; } - Some(Self { + Some(PartialDataColumnSidecarRef { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, - header: self.header.clone(), + header: self.header.iter().collect(), }) } @@ -176,6 +183,10 @@ impl PartialDataColumnSidecar { }, }) } + + pub fn take_header(&mut self) -> Option> { + Vec::from(mem::take(&mut self.header)).pop() + } } #[cfg_attr( @@ -251,17 +262,6 @@ pub struct PartialDataColumn { } impl PartialDataColumn { - pub fn clone_filter(&self, filter: F) -> Option - where - F: Fn(usize) -> bool, - { - Some(PartialDataColumn { - sidecar: self.sidecar.clone_filter(filter)?, - block_root: self.block_root, - index: self.index, - }) - } - pub fn try_clone_full(&self) -> Option> { if !self.sidecar.is_complete() { return None; @@ -272,14 +272,7 @@ impl PartialDataColumn { Some(DataColumnSidecar::Fulu(DataColumnSidecarFulu { index: self.index, - column: VariableList::new( - self.sidecar - .column - .iter() - .map(|cell| cell.as_ref().clone()) - .collect(), - ) - .expect("Same bounds"), + column: self.sidecar.column.clone(), kzg_commitments: header.kzg_commitments.clone(), kzg_proofs: self.sidecar.kzg_proofs.clone(), signed_block_header: header.signed_block_header.clone(), From 4aa00accafbfa20659e1f07ebb04fc6bc7831cf1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 14:39:11 +0100 Subject: [PATCH 100/146] improve logging around getBlobsV3 --- .../beacon_chain/src/fetch_blobs/mod.rs | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 8e2723e7209..f588d5dbd8f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -240,23 +240,12 @@ async fn fetch_and_process_blobs_v2_or_v3( let _timer = metrics::start_timer(&metrics::BEACON_ENGINE_GET_BLOBS_V3_REQUEST_DURATION_SECONDS); - let response = chain_adapter + chain_adapter .get_blobs_v3(versioned_hashes) .await .inspect_err(|_| { inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); - })?; - - if response - .as_ref() - .is_some_and(|vec| vec.iter().all(|blob| blob.is_some())) - { - inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_COMPLETE_RESPONSES_TOTAL); - } else { - inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_PARTIAL_RESPONSES_TOTAL); - } - - response + })? } else { debug!(num_expected_blobs, "Fetching all blobs from the EL"); @@ -287,21 +276,32 @@ async fn fetch_and_process_blobs_v2_or_v3( let num_fetched_blobs = blobs_and_proofs.iter().filter(|opt| opt.is_some()).count(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); - if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { - // This scenario is not supposed to happen if the EL is spec compliant. - // It should either return all requested blobs or none, but NOT partial responses. - // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. - warn!( - num_fetched_blobs, - num_expected_blobs, "The EL did not return all requested blobs" - ); - inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); - return Ok(None); + if num_fetched_blobs != num_expected_blobs { + if !get_blobs_v3 { + // This scenario is not supposed to happen if the EL is spec compliant. + // It should either return all requested blobs or none, but NOT partial responses. + // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. + warn!( + num_fetched_blobs, + num_expected_blobs, "The EL did not return all requested blobs" + ); + inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); + return Ok(None); + } else { + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_PARTIAL_RESPONSES_TOTAL); + debug!( + num_fetched_blobs, + num_expected_blobs, "Blobs partially received from the EL" + ); + } + } else { + debug!(num_fetched_blobs, "All blobs received from the EL"); + inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); + if get_blobs_v3 { + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V3_COMPLETE_RESPONSES_TOTAL); + } } - debug!(num_fetched_blobs, "Blobs received from the EL"); - inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); - if chain_adapter.fork_choice_contains_block(&block_root) { // Avoid computing columns if the block has already been imported. debug!( From 0fd7554f1dd777001efc9b064fee09baee579201 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 15:06:03 +0100 Subject: [PATCH 101/146] update libp2p --- Cargo.lock | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c9f6a1ea67..9176f0d29d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.111", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4956,7 +4956,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "bytes", "either", @@ -4987,7 +4987,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4997,7 +4997,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5007,7 +5007,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "either", "fnv", @@ -5031,7 +5031,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "async-trait", "futures", @@ -5046,7 +5046,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5076,7 +5076,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "asynchronous-codec", "either", @@ -5116,7 +5116,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "hickory-proto", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "libp2p-core", @@ -5150,7 +5150,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "asynchronous-codec", "bytes", @@ -5168,7 +5168,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "asynchronous-codec", "bytes", @@ -5190,7 +5190,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "futures-timer", @@ -5211,7 +5211,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "either", "fnv", @@ -5232,7 +5232,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "heck", "quote", @@ -5242,7 +5242,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "futures-timer", @@ -5257,7 +5257,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "futures-rustls", @@ -5275,7 +5275,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "futures-timer", @@ -5289,7 +5289,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "either", "futures", @@ -5964,7 +5964,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "bytes", "futures", @@ -6190,7 +6190,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7060,7 +7060,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "asynchronous-codec", "bytes", @@ -7119,7 +7119,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7641,7 +7641,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7747,7 +7747,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#bb44624d0dee7fecc37e266b91fd37322b45d597" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" dependencies = [ "futures", "pin-project", @@ -8699,7 +8699,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10024,7 +10024,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] From 5442ff491781e048e457a49cd4c7e599f00999a0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 15:06:34 +0100 Subject: [PATCH 102/146] mark peer as block-aware after receiving message --- beacon_node/lighthouse_network/src/types/partial.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 7084193716d..a6fba7c80b3 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -189,6 +189,9 @@ impl Partial for OutgoingPartialColumn { send: None, }), Some(metadata) => { + // The peer is apparently aware of the header, make sure we track that: + self.header_sent_set.lock().insert(peer_id); + let peer_metadata = PartialDataColumnPartsMetadata::::from_ssz_bytes(metadata) .map_err(|_| PartialError::InvalidFormat)?; let expected_len = self.partial_column.sidecar.cells_present_bitmap.len(); From 61048e6fc8c40a864687c6e3b04facebe7105c94 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 18 Mar 2026 12:15:25 +0100 Subject: [PATCH 103/146] optional partial assembler --- beacon_node/beacon_chain/src/beacon_chain.rs | 11 ++- beacon_node/beacon_chain/src/builder.rs | 2 + beacon_node/beacon_chain/src/chain_config.rs | 3 + .../src/data_availability_checker.rs | 97 +++++++++++-------- .../src/data_column_verification.rs | 4 +- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 10 +- .../beacon_chain/src/fetch_blobs/mod.rs | 4 +- .../beacon_chain/src/fetch_blobs/tests.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 1 + .../gossip_methods.rs | 2 +- .../src/network_beacon_processor/mod.rs | 2 +- beacon_node/src/config.rs | 9 +- 12 files changed, 91 insertions(+), 56 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3976c1c58e9..361af6e42ff 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2257,7 +2257,7 @@ impl BeaconChain { if let Some(cached_header) = self .data_availability_checker .partial_assembler() - .get_header(&block_root) + .and_then(|a| a.get_header(&block_root)) { return if *cached_header == data_column_header { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_DUPES); @@ -3212,10 +3212,13 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } + let Some(assembler) = self.data_availability_checker.partial_assembler() else { + // Partial messages are apparently not activated + return Ok(None); + }; + // Merge the partial into the assembler - let merge_result = self - .data_availability_checker - .partial_assembler() + let merge_result = assembler .merge_partials( block_root, vec![verified_partial], diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 59fa5ec9ec8..9972bd4b8e9 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -922,6 +922,7 @@ where let canonical_head = CanonicalHead::new(fork_choice, Arc::new(head_snapshot)); let shuffling_cache_size = self.chain_config.shuffling_cache_size; let complete_blob_backfill = self.chain_config.complete_blob_backfill; + let enable_partial_columns = self.chain_config.enable_partial_columns; // Calculate the weak subjectivity point in which to backfill blocks to. let genesis_backfill_slot = if self.chain_config.genesis_backfill { @@ -1055,6 +1056,7 @@ where self.kzg.clone(), Arc::new(custody_context), self.spec, + enable_partial_columns, ) .map_err(|e| format!("Error initializing DataAvailabilityChecker: {:?}", e))?, ), diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index e9cc4f24e99..b2c017a469d 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -121,6 +121,8 @@ pub struct ChainConfig { pub ignore_ws_check: bool, /// Disable the getBlobs optimisation to fetch blobs from the EL mempool. pub disable_get_blobs: bool, + /// Whether to enable partial data column support. + pub enable_partial_columns: bool, /// The node's custody type, determining how many data columns to custody and sample. pub node_custody_type: NodeCustodyType, } @@ -164,6 +166,7 @@ impl Default for ChainConfig { invalid_block_roots: HashSet::new(), ignore_ws_check: false, disable_get_blobs: false, + enable_partial_columns: false, node_custody_type: NodeCustodyType::Fullnode, } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 88d13f754f4..bd9abee715a 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -79,7 +79,7 @@ const OVERFLOW_LRU_CAPACITY_NON_ZERO: NonZeroUsize = new_non_zero_usize(32); pub struct DataAvailabilityChecker { complete_blob_backfill: bool, availability_cache: Arc>, - partial_assembler: Arc>, + partial_assembler: Option>>, slot_clock: T::SlotClock, kzg: Arc, custody_context: Arc>, @@ -122,15 +122,20 @@ impl DataAvailabilityChecker { kzg: Arc, custody_context: Arc>, spec: Arc, + enable_partial_columns: bool, ) -> Result { let inner = DataAvailabilityCheckerInner::new( OVERFLOW_LRU_CAPACITY_NON_ZERO, custody_context.clone(), spec.clone(), )?; - let partial_assembler = Arc::new(PartialDataColumnAssembler::new( - OVERFLOW_LRU_CAPACITY_NON_ZERO, - )); + let partial_assembler = if enable_partial_columns { + Some(Arc::new(PartialDataColumnAssembler::new( + OVERFLOW_LRU_CAPACITY_NON_ZERO, + ))) + } else { + None + }; Ok(Self { complete_blob_backfill, partial_assembler, @@ -146,8 +151,8 @@ impl DataAvailabilityChecker { &self.custody_context } - pub fn partial_assembler(&self) -> &Arc> { - &self.partial_assembler + pub fn partial_assembler(&self) -> Option<&Arc>> { + self.partial_assembler.as_ref() } /// Checks if the block root is currently in the availability cache awaiting import because @@ -206,16 +211,21 @@ impl DataAvailabilityChecker { } // Check assembler for partial columns - match self.partial_assembler.get_partial(block_root, column_index) { - Some(AssemblyColumn::Incomplete(cached_partial)) => { - // Compare: find which cells from incoming full column aren't in cached partial - compare_full_to_partial(cell_count, cached_partial.as_data_column()) - } - Some(AssemblyColumn::Complete) => Some(vec![]), - None => { - // No cached data, all cells are "missing" (new data we want) - Some((0..cell_count).collect()) + if let Some(assembler) = &self.partial_assembler { + match assembler.get_partial(block_root, column_index) { + Some(AssemblyColumn::Incomplete(cached_partial)) => { + // Compare: find which cells from incoming full column aren't in cached partial + compare_full_to_partial(cell_count, cached_partial.as_data_column()) + } + Some(AssemblyColumn::Complete) => Some(vec![]), + None => { + // No cached data, all cells are "missing" (new data we want) + Some((0..cell_count).collect()) + } } + } else { + // No assembler, all cells are missing + Some((0..cell_count).collect()) } } @@ -242,24 +252,29 @@ impl DataAvailabilityChecker { } // Check assembler for partial columns - match self.partial_assembler.get_partial(block_root, column_index) { - Some(AssemblyColumn::Incomplete(cached_partial)) => { - // Compare: find which cells from incoming full column aren't in cached partial - compare_partial_to_partial(data_column, cached_partial.as_data_column()) - } - Some(AssemblyColumn::Complete) => Some(vec![]), - None => { - // No cached data, return all present cells as "missing" (new data we want) - let incoming_cells: Vec = data_column - .sidecar - .cells_present_bitmap - .iter() - .enumerate() - .filter_map(|(idx, present)| present.then_some(idx)) - .collect(); - Some(incoming_cells) + if let Some(assembler) = &self.partial_assembler { + match assembler.get_partial(block_root, column_index) { + Some(AssemblyColumn::Incomplete(cached_partial)) => { + // Compare: find which cells from incoming full column aren't in cached partial + return compare_partial_to_partial( + data_column, + cached_partial.as_data_column(), + ); + } + Some(AssemblyColumn::Complete) => return Some(vec![]), + None => {} } } + + // No assembler or no data, return all cells as missing + let incoming_cells: Vec = data_column + .sidecar + .cells_present_bitmap + .iter() + .enumerate() + .filter_map(|(idx, present)| present.then_some(idx)) + .collect(); + Some(incoming_cells) } /// Get a blob from the availability cache. @@ -417,9 +432,10 @@ impl DataAvailabilityChecker { executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { let block = executed_block.as_block(); - if let Ok(header) = block.try_into() { - self.partial_assembler - .init(block.canonical_root(), Arc::new(header)); + if let Some(assembler) = &self.partial_assembler + && let Ok(header) = block.try_into() + { + assembler.init(block.canonical_root(), Arc::new(header)); } self.availability_cache.put_executed_block(executed_block) } @@ -432,8 +448,10 @@ impl DataAvailabilityChecker { block: Arc>, source: BlockImportSource, ) -> Result<(), Error> { - if let Ok(header) = block.as_ref().try_into() { - self.partial_assembler.init(block_root, Arc::new(header)); + if let Some(assembler) = &self.partial_assembler + && let Ok(header) = block.as_ref().try_into() + { + assembler.init(block_root, Arc::new(header)); } self.availability_cache .put_pre_execution_block(block_root, block, source) @@ -670,7 +688,7 @@ pub fn start_availability_cache_maintenance_service( async fn availability_cache_maintenance_service( chain: Arc>, overflow_cache: Arc>, - partial_assembler: Arc>, + partial_assembler: Option>>, ) { let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32; loop { @@ -722,7 +740,9 @@ async fn availability_cache_maintenance_service( if let Err(e) = overflow_cache.do_maintenance(cutoff_epoch) { error!(error = ?e,"Failed to maintain availability cache"); } - partial_assembler.do_maintenance(cutoff_epoch); + if let Some(assembler) = &partial_assembler { + assembler.do_maintenance(cutoff_epoch); + } } None => { error!("Failed to read slot clock"); @@ -1409,6 +1429,7 @@ mod test { kzg, custody_context, spec, + true, ) .expect("should initialise data availability checker") } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 6f069893301..a14d5fd58cc 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -484,7 +484,7 @@ impl GossipVerifiedPartialDataColumnHeader { let newly_cached = chain .data_availability_checker .partial_assembler() - .init(group_id, header.clone()); + .is_some_and(|a| a.init(group_id, header.clone())); chain .observed_slashable @@ -809,7 +809,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip( let Some(header) = chain .data_availability_checker .partial_assembler() - .get_header(&column.block_root) + .and_then(|a| a.get_header(&column.block_root)) else { return PartialColumnVerificationResult::Err( GossipPartialDataColumnError::MissingHeader, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index c9d27c77308..c94fb036f8b 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -1,5 +1,6 @@ use crate::fetch_blobs::{EngineGetBlobsOutput, FetchEngineBlobError}; use crate::observed_data_sidecars::ObservationKey; +use crate::partial_data_column_assembler::PartialDataColumnAssembler; use crate::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes}; use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use kzg::Kzg; @@ -35,10 +36,11 @@ impl FetchBlobsBeaconAdapter { &self.chain.task_executor } - pub(crate) fn partial_assembler( - &self, - ) -> &Arc> { - self.chain.data_availability_checker.partial_assembler() + pub(crate) fn partial_assembler(&self) -> Option>> { + self.chain + .data_availability_checker + .partial_assembler() + .cloned() } pub(crate) async fn get_blobs_v1( diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index f588d5dbd8f..87a9b40b413 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -330,7 +330,9 @@ async fn fetch_and_process_blobs_v2_or_v3( } // Initialize the partial assembler with the columns from the engine - let assembler = chain_adapter.partial_assembler(); + let Some(assembler) = chain_adapter.partial_assembler() else { + return Ok(None); + }; let merge_result = assembler .merge_partials(block_root, custody_columns_to_import, header) .ok_or_else(|| { diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index 0701c5c2054..ef282a3eaad 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -627,6 +627,6 @@ fn mock_beacon_adapter(fork_name: ForkName, get_blobs_v3: bool) -> MockFetchBlob .returning(move || Ok(get_blobs_v3)); mock_adapter .expect_partial_assembler() - .return_const(Arc::new(partial_assembler)); + .return_const(Some(Arc::new(partial_assembler))); mock_adapter } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c53c29438e9..dc6270a1122 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -239,6 +239,7 @@ pub fn test_da_checker( kzg, custody_context, spec, + true, ) .expect("should initialise data availability checker") } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 60fffbbdba1..577b4b660c1 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1296,7 +1296,7 @@ impl NetworkBeaconProcessor { .chain .data_availability_checker .partial_assembler() - .mark_as_complete(block_root, col) + .is_some_and(|a| a.mark_as_complete(block_root, col)) { metrics::inc_counter_vec( &metrics::BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 993d932dfa3..df143bae31e 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -958,7 +958,7 @@ impl NetworkBeaconProcessor { .chain .data_availability_checker .partial_assembler() - .get_partials_and_mark_as_local_fetched(block_root, &header) + .and_then(|a| a.get_partials_and_mark_as_local_fetched(block_root, &header)) { debug!(block = %block_root, "Publishing all partials after getBlobs"); self.send_network_message(NetworkMessage::PublishPartialColumns { diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 91f3060399a..b58ddb85ba8 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -110,6 +110,11 @@ pub fn get_config( set_network_config(&mut client_config.network, cli_args, &data_dir_ref)?; + if parse_flag(cli_args, "enable-partial-columns") { + client_config.network.enable_partial_columns = true; + client_config.chain.enable_partial_columns = true; + } + // Parse custody mode from CLI flags let is_supernode = parse_flag(cli_args, "supernode"); let is_semi_supernode = parse_flag(cli_args, "semi-supernode"); @@ -1487,10 +1492,6 @@ pub fn set_network_config( })?; } - if parse_flag(cli_args, "enable-partial-columns") { - config.enable_partial_columns = true; - } - Ok(()) } From 2f4e7afa00d6769e644d4033de6e9e1c1b78617f Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 16:21:17 +0100 Subject: [PATCH 104/146] add remaining verification steps --- .../src/data_column_verification.rs | 61 ++++++++++++++++++- .../gossip_methods.rs | 17 +++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index a14d5fd58cc..0185114721d 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -224,6 +224,28 @@ pub enum GossipPartialDataColumnError { group_id: Hash256, header_hash: Hash256, }, + /// The partial message has neither a header nor cells + /// + /// ## Peer scoring + /// The column sidecar is invalid and the peer is faulty + EmptyMessage, + /// The partial message has a count of proofs anc/or cells that is inconsistent with the bitmap. + /// + /// ## Peer scoring + /// The column sidecar is invalid and the peer is faulty + InconsistentPresentCount { + bitmap_popcount: usize, + cells_len: usize, + proofs_len: usize, + }, + /// The partial message has a bitmap length that is inconsistent with the number of commitments. + /// + /// ## Peer scoring + /// The column sidecar is invalid and the peer is faulty + InconsistentMaxCount { + bitmap_len: usize, + commitments_len: usize, + }, } impl From for GossipPartialDataColumnError { @@ -810,14 +832,51 @@ pub fn validate_partial_data_column_sidecar_for_gossip( .data_availability_checker .partial_assembler() .and_then(|a| a.get_header(&column.block_root)) + .map(GossipVerifiedPartialDataColumnHeader::new_from_cached) else { return PartialColumnVerificationResult::Err( GossipPartialDataColumnError::MissingHeader, ); }; - GossipVerifiedPartialDataColumnHeader::new_from_cached(header) + + // If there was no header, there must be at least one cell. + if column.sidecar.column.is_empty() { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::EmptyMessage, + header, + }; + } + + header }; + // The number of cells nad proofs must match the population count of the bitmap. + let bitmap_popcount = column.sidecar.cells_present_bitmap.num_set_bits(); + let cells_len = column.sidecar.column.len(); + let proofs_len = column.sidecar.kzg_proofs.len(); + if bitmap_popcount != cells_len || bitmap_popcount != proofs_len { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::InconsistentPresentCount { + bitmap_popcount, + cells_len, + proofs_len, + }, + header, + }; + } + + let bitmap_len = column.sidecar.cells_present_bitmap.len(); + let commitments_len = header.as_header().kzg_commitments.len(); + if bitmap_len != commitments_len { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::InconsistentMaxCount { + bitmap_len, + commitments_len, + }, + header, + }; + } + let Some(missing_cells) = chain .data_availability_checker .determine_missing_cells_partial(&block_root, &column) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 577b4b660c1..06a1a230b8e 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1025,7 +1025,22 @@ impl NetworkBeaconProcessor { self.gossip_penalize_peer( peer_id, PeerAction::LowToleranceError, - "gossip_partial_data_column_high", + "gossip_partial_data_column_low", + ); + } + GossipPartialDataColumnError::EmptyMessage + | GossipPartialDataColumnError::InconsistentPresentCount { .. } + | GossipPartialDataColumnError::InconsistentMaxCount { .. } => { + debug!( + error = ?err, + %block_root, + %index, + "Could not verify partial column" + ); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_partial_data_column_low", ); } } From 20226bb838a1321b33506ee9031d515acb90ce0e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 18:12:45 +0100 Subject: [PATCH 105/146] fix places relying on header in sidecar --- .../src/data_column_verification.rs | 16 ++++++-- beacon_node/beacon_chain/src/kzg_utils.rs | 6 ++- .../gossip_methods.rs | 40 +++++++------------ beacon_node/network/src/sync/manager.rs | 14 ++----- .../src/data/partial_data_column_sidecar.rs | 8 ++-- 5 files changed, 39 insertions(+), 45 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 0185114721d..83c6e8d26d7 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -117,6 +117,7 @@ pub enum GossipDataColumnError { /// We cannot process the columns without validating its parent, the peer isn't necessarily faulty. ParentUnknown { parent_root: Hash256, + slot: Slot, }, /// The column conflicts with finalization, no need to propagate. /// @@ -495,6 +496,7 @@ impl GossipVerifiedPartialDataColumnHeader { verify_partial_column_header_inclusion_proof(&header)?; let parent_block = verify_parent_block_and_finalized_descendant( header.signed_block_header.message.parent_root, + column_slot, chain, )?; verify_slot_higher_than_parent(&parent_block, column_slot)?; @@ -673,7 +675,10 @@ impl KzgVerifiedCustodyPartialDataColumn { }) } - pub fn try_clone_full(&self) -> Option> { + pub fn try_clone_full( + &self, + header: &PartialDataColumnHeader, + ) -> Option> { self.data .try_clone_full() .map(|data| KzgVerifiedCustodyDataColumn { @@ -779,8 +784,11 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( block_parent_root: Hash256, + slot: Slot, chain: &BeaconChain, ) -> Result { let fork_choice = chain.canonical_head.fork_choice_read_lock(); @@ -1042,6 +1051,7 @@ fn verify_parent_block_and_finalized_descendant( let Some(parent_block) = fork_choice.get_block(&block_parent_root) else { return Err(GossipDataColumnError::ParentUnknown { parent_root: block_parent_root, + slot, }); }; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 6da996d3048..99df3989844 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -546,13 +546,15 @@ pub(crate) fn build_partial_data_columns( } } + let block_root = header.signed_block_header.message.canonical_root(); + let sidecars: Result>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() .map(|(index, (col, proofs))| { let column = PartialDataColumn { - block_root: header.signed_block_header.message.canonical_root(), + block_root, index: index as u64, sidecar: types::data::PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), @@ -560,7 +562,7 @@ pub(crate) fn build_partial_data_columns( .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - header: VariableList::repeat_full(header.clone()), + header: VariableList::empty(), }, }; Ok(column) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 06a1a230b8e..f14976781bd 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -861,15 +861,11 @@ impl NetworkBeaconProcessor { Some(header) } PartialColumnVerificationResult::ErrWithValidHeader { header, err } => { - self.handle_partial_verification_error( - peer_id, err, block_root, index, None, topic, - ); + self.handle_partial_verification_error(peer_id, err, block_root, index, topic); Some(header) } PartialColumnVerificationResult::Err(err) => { - self.handle_partial_verification_error( - peer_id, err, block_root, index, None, topic, - ); + self.handle_partial_verification_error(peer_id, err, block_root, index, topic); None } }; @@ -899,7 +895,6 @@ impl NetworkBeaconProcessor { err: GossipPartialDataColumnError, block_root: Hash256, index: ColumnIndex, - partial_data_column: Option>, topic: GossipTopic, ) { match err { @@ -919,24 +914,19 @@ impl NetworkBeaconProcessor { "Gossip partial data column already processed via the EL." ); } - GossipDataColumnError::ParentUnknown { parent_root } => { - if let Some(partial_data_column) = partial_data_column { - debug!( - action = "requesting parent", - %block_root, - %parent_root, - "Unknown parent hash for partial column" - ); - self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( - peer_id, - partial_data_column, - )); - } else { - crit!( - %parent_root, - "Unknown parent hash for partial column - but got no partial column to reprocess" - ) - } + GossipDataColumnError::ParentUnknown { parent_root, slot } => { + debug!( + action = "requesting parent", + %block_root, + %parent_root, + "Unknown parent hash for partial column" + ); + self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( + peer_id, + block_root, + parent_root, + slot, + )); } GossipDataColumnError::PubkeyCacheTimeout | GossipDataColumnError::BeaconChainError(_) => { diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 7d51cceaba1..b9ecb436ff3 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -144,7 +144,7 @@ pub enum SyncMessage { UnknownParentDataColumn(PeerId, Arc>), /// A partial data column with an unknown parent has been received. - UnknownParentPartialDataColumn(PeerId, PartialDataColumn), + UnknownParentPartialDataColumn(PeerId, Hash256, Hash256, Slot), /// A peer has sent an attestation that references a block that is unknown. This triggers the /// manager to attempt to find the block matching the unknown hash. @@ -904,21 +904,13 @@ impl SyncManager { } } } - SyncMessage::UnknownParentPartialDataColumn(peer_id, partial_data_column) => { - let block_root = partial_data_column.block_root; - let Some(header) = partial_data_column.sidecar.header.first() else { - // This should not happen - we need to know the header to know that we don't know the parent - crit!(%block_root, "Got unknown parent partial column without header"); - return; - }; - let data_column_slot = header.slot(); - let parent_root = header.signed_block_header.message.parent_root; + SyncMessage::UnknownParentPartialDataColumn(peer_id, block_root, parent_root, slot) => { debug!(%block_root, %parent_root, "Received unknown parent partial column message"); self.handle_unknown_parent( peer_id, block_root, parent_root, - data_column_slot, + slot, BlockComponent::PartialDataColumn(DownloadResult { value: parent_root, block_root, diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index e415013c31c..448e914d8d1 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -262,13 +262,13 @@ pub struct PartialDataColumn { } impl PartialDataColumn { - pub fn try_clone_full(&self) -> Option> { + pub fn try_clone_full( + &self, + header: &PartialDataColumnHeader, + ) -> Option> { if !self.sidecar.is_complete() { return None; } - let Some(header) = self.sidecar.header.first() else { - return None; - }; Some(DataColumnSidecar::Fulu(DataColumnSidecarFulu { index: self.index, From b65d5696446ab413804f716597dbfba73d55a2e2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 23 Mar 2026 09:04:28 +0100 Subject: [PATCH 106/146] fix build --- beacon_node/beacon_chain/src/data_column_verification.rs | 2 +- beacon_node/beacon_chain/src/partial_data_column_assembler.rs | 2 +- .../network/src/network_beacon_processor/gossip_methods.rs | 2 +- beacon_node/network/src/sync/manager.rs | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 83c6e8d26d7..7ff128794ed 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -680,7 +680,7 @@ impl KzgVerifiedCustodyPartialDataColumn { header: &PartialDataColumnHeader, ) -> Option> { self.data - .try_clone_full() + .try_clone_full(header) .map(|data| KzgVerifiedCustodyDataColumn { data: Arc::new(data), }) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index af0aa2f60f3..b92fcde1154 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -125,7 +125,7 @@ impl PartialDataColumnAssembler { }; // Check if merged column is now complete by trying to convert into full - let column = if let Some(full_column) = merged.try_clone_full() { + let column = if let Some(full_column) = merged.try_clone_full(&header) { full_columns.push(full_column); AssemblyColumn::Complete } else { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index f14976781bd..16fda0e37b7 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -715,7 +715,7 @@ impl NetworkBeaconProcessor { MessageAcceptance::Accept, ); } - GossipDataColumnError::ParentUnknown { parent_root } => { + GossipDataColumnError::ParentUnknown { parent_root, .. } => { debug!( action = "requesting parent", %block_root, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index b9ecb436ff3..b3986b8999f 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -74,8 +74,7 @@ use strum::IntoStaticStr; use tokio::sync::mpsc; use tracing::{debug, error, info, trace}; use types::{ - BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, PartialDataColumn, - SignedBeaconBlock, Slot, + BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, SignedBeaconBlock, Slot, }; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync From 9fbbaf984e615bbc641e9b028fb3e58c53b272ac Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 20 Mar 2026 17:14:16 +0100 Subject: [PATCH 107/146] add unit tests --- .../src/data_column_verification.rs | 244 +++++++++++++- .../src/partial_data_column_assembler.rs | 302 +++++++++++++++++- .../lighthouse_network/src/types/partial.rs | 247 ++++++++++++++ .../src/data/partial_data_column_sidecar.rs | 198 ++++++++++++ 4 files changed, 987 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 7ff128794ed..e49b6db9d61 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -1227,9 +1227,12 @@ pub fn observe_gossip_data_column( #[cfg(test)] mod test { + use crate::ChainConfig; use crate::data_column_verification::{ - GossipDataColumnError, GossipVerifiedDataColumn, + GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, + GossipVerifiedPartialDataColumnHeader, PartialColumnVerificationResult, validate_data_column_sidecar_for_gossip_fulu, + validate_partial_data_column_sidecar_for_gossip, }; use crate::observed_data_sidecars::Observe; use crate::test_utils::{ @@ -1237,10 +1240,11 @@ mod test { }; use eth2::types::BlobsBundle; use execution_layer::test_utils::generate_blobs; + use ssz::BitList; use std::sync::Arc; use types::{ DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, ForkName, - MainnetEthSpec, + MainnetEthSpec, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, }; type E = MainnetEthSpec; @@ -1365,4 +1369,240 @@ mod test { Some(GossipDataColumnError::MaxBlobsPerBlockExceeded { .. }) )); } + + /// Build a Fulu harness with partial columns enabled. + async fn partial_column_harness() -> BeaconChainHarness> { + let spec = ForkName::Fulu.make_genesis_spec(E::default_spec()); + let chain_config = ChainConfig { + enable_partial_columns: true, + ..Default::default() + }; + let harness = BeaconChainHarness::builder(E::default()) + .spec(spec.into()) + .deterministic_keypairs(64) + .fresh_ephemeral_store() + .mock_execution_layer() + .chain_config(chain_config) + .build(); + harness.advance_slot(); + harness + } + + /// Build a partial column harness with a block containing 1 blob commitment + /// and the header pre-cached in the partial assembler. + async fn partial_column_harness_with_block() -> ( + BeaconChainHarness>, + types::Hash256, + Arc>, + ) { + let harness = partial_column_harness().await; + + // Generate a block with 1 blob so we have valid data columns. + let fork = harness + .spec + .fork_name_at_epoch(harness.get_current_slot().epoch(E::slots_per_epoch())); + let BlobsBundle:: { + commitments, + proofs: _, + blobs: _, + } = generate_blobs(1, fork).unwrap().0; + + let slot = harness.get_current_slot(); + let state = harness.get_current_state(); + let ((block, _blobs_opt), _state) = harness + .make_block_with_modifier(state, slot, |block| { + *block.body_mut().blob_kzg_commitments_mut().unwrap() = + vec![commitments[0]].try_into().unwrap(); + }) + .await; + + let block_root = block.canonical_root(); + let header: PartialDataColumnHeader = block.as_ref().try_into().unwrap(); + let header = Arc::new(header); + + // Pre-cache the header in the partial assembler so headerless partials can be verified. + harness + .chain + .data_availability_checker + .partial_assembler() + .unwrap() + .init(block_root, header.clone()); + + (harness, block_root, header) + } + + #[tokio::test] + async fn partial_empty_message_without_cells_returns_error() { + let (harness, block_root, header) = partial_column_harness_with_block().await; + + // Create a headerless partial with no cells — should trigger EmptyMessage. + let num_commitments = header.kzg_commitments.len(); + let empty_bitmap = + BitList::<::MaxBlobCommitmentsPerBlock>::with_capacity(num_commitments) + .unwrap(); + + let column = PartialDataColumn { + block_root, + index: 0, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: empty_bitmap, + column: vec![].try_into().unwrap(), + kzg_proofs: vec![].try_into().unwrap(), + header: ssz_types::VariableList::new(vec![]).unwrap(), + }, + }; + + let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + assert!( + matches!( + result, + PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::EmptyMessage, + .. + } + ), + "Expected EmptyMessage" + ); + } + + #[tokio::test] + async fn partial_inconsistent_present_count_returns_error() { + let (harness, block_root, header) = partial_column_harness_with_block().await; + + // Create a bitmap that says 2 bits are set, but only provide 1 cell/proof. + let num_commitments = header.kzg_commitments.len(); + let mut bitmap = + BitList::<::MaxBlobCommitmentsPerBlock>::with_capacity(num_commitments) + .unwrap(); + bitmap.set(0, true).unwrap(); + + let column = PartialDataColumn { + block_root, + index: 0, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: vec![types::Cell::::default()].try_into().unwrap(), + // Provide 2 proofs but only 1 cell ← mismatch with popcount=1 + kzg_proofs: vec![types::KzgProof::empty(), types::KzgProof::empty()] + .try_into() + .unwrap(), + header: ssz_types::VariableList::new(vec![]).unwrap(), + }, + }; + + let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + assert!( + matches!( + result, + PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::InconsistentPresentCount { .. }, + .. + } + ), + "Expected InconsistentPresentCount" + ); + } + + #[tokio::test] + async fn partial_inconsistent_max_count_returns_error() { + let (harness, block_root, _header) = partial_column_harness_with_block().await; + + // Create a bitmap with length different from the number of commitments in the header. + // Header has 1 commitment, but we use a bitmap with capacity 3. + let mut bitmap = + BitList::<::MaxBlobCommitmentsPerBlock>::with_capacity(3).unwrap(); + bitmap.set(0, true).unwrap(); + + let column = PartialDataColumn { + block_root, + index: 0, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: vec![types::Cell::::default()].try_into().unwrap(), + kzg_proofs: vec![types::KzgProof::empty()].try_into().unwrap(), + header: ssz_types::VariableList::new(vec![]).unwrap(), + }, + }; + + let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + assert!( + matches!( + result, + PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipPartialDataColumnError::InconsistentMaxCount { .. }, + .. + } + ), + "Expected InconsistentMaxCount" + ); + } + + #[tokio::test] + async fn partial_header_with_empty_commitments_fails() { + let harness = partial_column_harness().await; + + let slot = harness.get_current_slot(); + let state = harness.get_current_state(); + let ((block, _), _) = harness + .make_block_with_modifier(state, slot, |block| { + *block.body_mut().blob_kzg_commitments_mut().unwrap() = vec![].try_into().unwrap(); + }) + .await; + + let block_root = block.canonical_root(); + let header: PartialDataColumnHeader = block.as_ref().try_into().unwrap(); + assert!(header.kzg_commitments.is_empty()); + + let result = + GossipVerifiedPartialDataColumnHeader::new(block_root, header, &*harness.chain); + assert!( + matches!( + result, + Err(GossipPartialDataColumnError::GossipDataColumnError( + GossipDataColumnError::UnexpectedDataColumn + )) + ), + "Expected UnexpectedDataColumn, got: {result:?}" + ); + } + + #[tokio::test] + async fn partial_header_root_mismatch_fails() { + let (harness, _block_root, header) = partial_column_harness_with_block().await; + + // Use a wrong group_id (not matching the header's block root) + let wrong_root = types::Hash256::repeat_byte(0xff); + let header = PartialDataColumnHeader::clone(&header); + + let result = + GossipVerifiedPartialDataColumnHeader::new(wrong_root, header, &*harness.chain); + assert!( + matches!( + result, + Err(GossipPartialDataColumnError::HeaderIncorrectRoot { .. }) + ), + "Expected HeaderIncorrectRoot, got: {result:?}" + ); + } + + #[tokio::test] + async fn partial_header_with_invalid_inclusion_proof_fails() { + let (harness, block_root, header) = partial_column_harness_with_block().await; + + // Corrupt the inclusion proof + let mut header = PartialDataColumnHeader::clone(&header); + header.kzg_commitments_inclusion_proof[0] = types::Hash256::repeat_byte(0xaa); + + let result = + GossipVerifiedPartialDataColumnHeader::new(block_root, header, &*harness.chain); + assert!( + matches!( + result, + Err(GossipPartialDataColumnError::GossipDataColumnError( + GossipDataColumnError::InvalidInclusionProof + )) + ), + "Expected InvalidInclusionProof, got: {result:?}" + ); + } } diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index b92fcde1154..f7c070cdd06 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; -use types::{DataColumnSidecarFulu, EthSpec, Hash256}; +use types::{DataColumnSidecarFulu, Epoch, EthSpec, Hash256}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -218,7 +218,7 @@ impl PartialDataColumnAssembler { } /// Maintenance: remove assemblies older than cutoff epoch - pub fn do_maintenance(&self, cutoff_epoch: types::Epoch) { + pub fn do_maintenance(&self, cutoff_epoch: Epoch) { let mut assemblies = self.assemblies.write(); let mut to_remove = vec![]; @@ -248,3 +248,301 @@ pub struct InitResult { /// Columns that need further assembly via gossip pub incomplete_partials: Vec>>, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::data_column_verification::{ + KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + }; + use bls::{FixedBytesExtended, Signature}; + use kzg::{KzgCommitment, KzgProof}; + use ssz_types::{FixedVector, VariableList}; + use types::{ + BeaconBlockHeader, Cell, CellBitmap, EthSpec, Hash256, MinimalEthSpec, + PartialDataColumnSidecar, SignedBeaconBlockHeader, Slot, + }; + + type E = MinimalEthSpec; + + fn make_cell(marker: u8) -> Cell { + let mut cell = Cell::::default(); + cell[0] = marker; + cell + } + + fn make_header(num_commitments: usize) -> Arc> { + Arc::new(PartialDataColumnHeader { + kzg_commitments: vec![KzgCommitment([0u8; 48]); num_commitments] + .try_into() + .unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader { + slot: Slot::new(1), + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body_root: Hash256::zero(), + }, + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: FixedVector::new( + vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], + ) + .unwrap(), + }) + } + + fn make_partial( + block_root: Hash256, + column_index: ColumnIndex, + total_blobs: usize, + present_indices: &[usize], + ) -> KzgVerifiedCustodyPartialDataColumn { + make_partial_with_header(block_root, column_index, total_blobs, present_indices, true) + } + + fn make_partial_with_header( + block_root: Hash256, + column_index: ColumnIndex, + total_blobs: usize, + present_indices: &[usize], + include_header: bool, + ) -> KzgVerifiedCustodyPartialDataColumn { + let mut bitmap = CellBitmap::::with_capacity(total_blobs).unwrap(); + for &idx in present_indices { + bitmap.set(idx, true).unwrap(); + } + + let column: VariableList<_, _> = present_indices + .iter() + .map(|&idx| make_cell(idx as u8)) + .collect::>() + .try_into() + .unwrap(); + let proofs: VariableList<_, _> = present_indices + .iter() + .map(|_| KzgProof::empty()) + .collect::>() + .try_into() + .unwrap(); + + let header_list = if include_header { + VariableList::new(vec![(*make_header(total_blobs)).clone()]).unwrap() + } else { + VariableList::new(vec![]).unwrap() + }; + + let partial = PartialDataColumn { + block_root, + index: column_index, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column, + kzg_proofs: proofs, + header: header_list, + }, + }; + KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( + KzgVerifiedPartialDataColumn::__new_for_testing(partial), + ) + } + + fn make_assembler() -> PartialDataColumnAssembler { + PartialDataColumnAssembler::new(NonZeroUsize::new(16).unwrap()) + } + + // -- init and get_header tests -- + + #[test] + fn init_stores_header() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + assert!(assembler.init(root, header.clone())); + let retrieved = assembler.get_header(&root).unwrap(); + assert_eq!(*retrieved, *header); + } + + #[test] + fn init_returns_false_if_already_exists() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + assert!(assembler.init(root, header.clone())); + assert!(!assembler.init(root, header)); + } + + // -- merge_partials tests -- + + #[test] + fn merge_partials_tracks_added_cells() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + + let partial = make_partial(root, 0, 4, &[0, 1, 2]); + let result = assembler + .merge_partials(root, vec![partial], header.clone()) + .unwrap(); + assert_eq!(result.added_cells, 3); + + // Merge more cells for the same column + let partial2 = make_partial(root, 0, 4, &[2, 3]); + let result2 = assembler + .merge_partials(root, vec![partial2], header) + .unwrap(); + // Only cell 3 is new (cell 2 was already present) + assert_eq!(result2.added_cells, 1); + } + + #[test] + fn merge_partials_ignores_already_complete_column() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + + // Complete the column + let partial = make_partial(root, 0, 4, &[0, 1, 2, 3]); + let result = assembler + .merge_partials(root, vec![partial], header.clone()) + .unwrap(); + assert_eq!(result.added_cells, 4); + assert_eq!(result.full_columns.len(), 1); + + // Try to merge more — should be ignored + let partial2 = make_partial(root, 0, 4, &[0, 1]); + let result2 = assembler + .merge_partials(root, vec![partial2], header) + .unwrap(); + assert_eq!(result2.added_cells, 0); + assert!(result2.full_columns.is_empty()); + } + + #[test] + fn merge_partials_completes_column_progressively() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + + let partial1 = make_partial(root, 0, 4, &[0, 1]); + let result1 = assembler + .merge_partials(root, vec![partial1], header.clone()) + .unwrap(); + assert!(result1.full_columns.is_empty()); + + let partial2 = make_partial(root, 0, 4, &[2, 3]); + let result2 = assembler + .merge_partials(root, vec![partial2], header) + .unwrap(); + assert_eq!(result2.full_columns.len(), 1); + } + + #[test] + fn merge_partials_returns_updated_partials() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + + let partial = make_partial(root, 0, 4, &[0, 2]); + let result = assembler + .merge_partials(root, vec![partial], header) + .unwrap(); + assert_eq!(result.updated_partials.len(), 1); + assert_eq!(result.updated_partials[0].index(), 0); + } + + // -- mark_as_complete tests -- + + #[test] + fn mark_as_complete_replaces_incomplete() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + let header = make_header(4); + + // Merge an incomplete partial first + let partial = make_partial(root, 0, 4, &[0, 1]); + assembler.merge_partials(root, vec![partial], header); + + let full_column = DataColumnSidecarFulu:: { + index: 0, + column: vec![Cell::::default(); 4].try_into().unwrap(), + kzg_commitments: vec![KzgCommitment([0u8; 48]); 4].try_into().unwrap(), + kzg_proofs: vec![KzgProof::empty(); 4].try_into().unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader { + slot: Slot::new(1), + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body_root: Hash256::zero(), + }, + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: FixedVector::new( + vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], + ) + .unwrap(), + }; + assert!(assembler.mark_as_complete(root, &full_column)); + } + + #[test] + fn mark_as_complete_returns_false_if_already_complete() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + + let full_column = DataColumnSidecarFulu:: { + index: 0, + column: vec![Cell::::default(); 4].try_into().unwrap(), + kzg_commitments: vec![KzgCommitment([0u8; 48]); 4].try_into().unwrap(), + kzg_proofs: vec![KzgProof::empty(); 4].try_into().unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader { + slot: Slot::new(1), + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body_root: Hash256::zero(), + }, + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: FixedVector::new( + vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], + ) + .unwrap(), + }; + assert!(assembler.mark_as_complete(root, &full_column)); + assert!(!assembler.mark_as_complete(root, &full_column)); + } + + // -- do_maintenance tests -- + + #[test] + fn do_maintenance_removes_old_assemblies() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + // Header at slot 0 → epoch 0 + let header = make_header(4); + assembler.init(root, header); + assert!(assembler.get_header(&root).is_some()); + + // Cutoff epoch 1 removes epoch 0 + assembler.do_maintenance(Epoch::new(1)); + assert!(assembler.get_header(&root).is_none()); + } + + #[test] + fn do_maintenance_keeps_recent_assemblies() { + let assembler = make_assembler(); + let root = Hash256::repeat_byte(1); + // Header at slot 100 → epoch 100/8 = 12 for MinimalEthSpec (8 slots/epoch) + let mut header = (*make_header(4)).clone(); + header.signed_block_header.message.slot = Slot::new(100); + let header = Arc::new(header); + assembler.init(root, header); + + assembler.do_maintenance(Epoch::new(1)); + assert!(assembler.get_header(&root).is_some()); + } +} diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index a6fba7c80b3..4fbea559e69 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -249,3 +249,250 @@ impl Partial for OutgoingPartialColumn { } } } + +#[cfg(test)] +mod tests { + use super::*; + use bls::Signature; + use fixed_bytes::FixedBytesExtended; + use libp2p::identity::Keypair; + use ssz_types::{FixedVector, VariableList}; + use types::{ + BeaconBlockHeader, MinimalEthSpec, PartialDataColumnHeader, SignedBeaconBlockHeader, Slot, + }; + + type E = MinimalEthSpec; + + fn make_cell(marker: u8) -> types::Cell { + let mut cell = types::Cell::::default(); + cell[0] = marker; + cell + } + + fn make_header(num_commitments: usize) -> PartialDataColumnHeader { + PartialDataColumnHeader { + kzg_commitments: vec![types::KzgCommitment([0u8; 48]); num_commitments] + .try_into() + .unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader { + slot: Slot::new(1), + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body_root: Hash256::zero(), + }, + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: FixedVector::new( + vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], + ) + .unwrap(), + } + } + + fn make_partial_column( + block_root: Hash256, + total_blobs: usize, + present_indices: &[usize], + ) -> Arc> { + let mut bitmap = CellBitmap::::with_capacity(total_blobs).unwrap(); + for &idx in present_indices { + bitmap.set(idx, true).unwrap(); + } + + Arc::new(PartialDataColumn { + block_root, + index: 0, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: present_indices + .iter() + .map(|&idx| make_cell(idx as u8)) + .collect::>() + .try_into() + .unwrap(), + kzg_proofs: present_indices + .iter() + .map(|_| types::KzgProof::empty()) + .collect::>() + .try_into() + .unwrap(), + header: VariableList::new(vec![]).unwrap(), + }, + }) + } + + fn random_peer_id() -> PeerId { + let keypair = Keypair::generate_ed25519(); + PeerId::from(keypair.public()) + } + + // -- MaybeKnownMetadata tests -- + + #[test] + fn update_from_unknown_initializes() { + let mut meta = MaybeKnownMetadata::::Unknown; + let mut bitmap = CellBitmap::::with_capacity(4).unwrap(); + bitmap.set(0, true).unwrap(); + let received = PartialDataColumnPartsMetadata { + available: bitmap.clone(), + request: bitmap, + }; + let changed = meta.do_update(received).unwrap(); + assert!(changed); + assert!(matches!(meta, MaybeKnownMetadata::Known { .. })); + } + + #[test] + fn update_unions_bitmaps() { + let mut bitmap1 = CellBitmap::::with_capacity(4).unwrap(); + bitmap1.set(0, true).unwrap(); + let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { + available: bitmap1.clone(), + request: bitmap1, + } + .into(); + + let mut bitmap2 = CellBitmap::::with_capacity(4).unwrap(); + bitmap2.set(1, true).unwrap(); + let changed = meta + .do_update(PartialDataColumnPartsMetadata { + available: bitmap2.clone(), + request: bitmap2, + }) + .unwrap(); + assert!(changed); + + if let MaybeKnownMetadata::Known { metadata, .. } = &meta { + assert!(metadata.available.get(0).unwrap()); + assert!(metadata.available.get(1).unwrap()); + assert!(!metadata.available.get(2).unwrap()); + } else { + panic!("Expected Known metadata"); + } + } + + #[test] + fn update_returns_false_when_no_change() { + let mut bitmap = CellBitmap::::with_capacity(4).unwrap(); + bitmap.set(0, true).unwrap(); + bitmap.set(1, true).unwrap(); + let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { + available: bitmap.clone(), + request: bitmap.clone(), + } + .into(); + + // Update with a subset + let mut subset = CellBitmap::::with_capacity(4).unwrap(); + subset.set(0, true).unwrap(); + let changed = meta + .do_update(PartialDataColumnPartsMetadata { + available: subset.clone(), + request: subset, + }) + .unwrap(); + assert!(!changed); + } + + #[test] + fn update_rejects_mismatched_lengths() { + let mut bitmap4 = CellBitmap::::with_capacity(4).unwrap(); + bitmap4.set(0, true).unwrap(); + let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { + available: bitmap4.clone(), + request: bitmap4, + } + .into(); + + let mut bitmap6 = CellBitmap::::with_capacity(6).unwrap(); + bitmap6.set(0, true).unwrap(); + let result = meta.do_update(PartialDataColumnPartsMetadata { + available: bitmap6.clone(), + request: bitmap6, + }); + assert!(result.is_err()); + } + + // -- OutgoingPartialColumn::partial_action_from_metadata tests -- + + #[test] + fn no_metadata_sends_header_once() { + let root = Hash256::repeat_byte(1); + let header = make_header(4); + let partial = make_partial_column(root, 4, &[0, 1]); + let header_sent_set: HeaderSentSet = Arc::new(Mutex::new(HashSet::new())); + let outgoing = OutgoingPartialColumn::new(partial, &header, header_sent_set); + + let peer = random_peer_id(); + + // First call with no metadata → sends header + let action = outgoing.partial_action_from_metadata(peer, None).unwrap(); + assert!(action.send.is_some()); + + // Second call for same peer → no send + let action2 = outgoing.partial_action_from_metadata(peer, None).unwrap(); + assert!(action2.send.is_none()); + } + + #[test] + fn metadata_filters_cells_to_send() { + let root = Hash256::repeat_byte(1); + let header = make_header(4); + // We have cells [0, 2, 3] + let partial = make_partial_column(root, 4, &[0, 2, 3]); + let header_sent_set: HeaderSentSet = Arc::new(Mutex::new(HashSet::new())); + let outgoing = OutgoingPartialColumn::new(partial, &header, header_sent_set); + + let peer = random_peer_id(); + + // Peer has [0, 1], wants [0, 1, 2, 3] + let mut peer_available = CellBitmap::::with_capacity(4).unwrap(); + peer_available.set(0, true).unwrap(); + peer_available.set(1, true).unwrap(); + let mut peer_request = CellBitmap::::with_capacity(4).unwrap(); + for i in 0..4 { + peer_request.set(i, true).unwrap(); + } + let peer_meta = PartialDataColumnPartsMetadata:: { + available: peer_available, + request: peer_request, + }; + let encoded = peer_meta.as_ssz_bytes(); + + let action = outgoing + .partial_action_from_metadata(peer, Some(&encoded)) + .unwrap(); + // We should send cells [2, 3] (want = request - available = [2,3], and we have [0,2,3]) + assert!(action.send.is_some()); + } + + #[test] + fn metadata_sets_need_when_peer_has_unknown_cells() { + let root = Hash256::repeat_byte(1); + let header = make_header(4); + // We have cells [0] + let partial = make_partial_column(root, 4, &[0]); + let header_sent_set: HeaderSentSet = Arc::new(Mutex::new(HashSet::new())); + let outgoing = OutgoingPartialColumn::new(partial, &header, header_sent_set); + + let peer = random_peer_id(); + + // Peer has [0, 1, 2] — cells [1, 2] are unknown to us + let mut peer_available = CellBitmap::::with_capacity(4).unwrap(); + peer_available.set(0, true).unwrap(); + peer_available.set(1, true).unwrap(); + peer_available.set(2, true).unwrap(); + let peer_meta = PartialDataColumnPartsMetadata:: { + available: peer_available.clone(), + request: peer_available, + }; + let encoded = peer_meta.as_ssz_bytes(); + + let action = outgoing + .partial_action_from_metadata(peer, Some(&encoded)) + .unwrap(); + assert!(action.need); + } +} diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 448e914d8d1..6c6b488ac7b 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -280,3 +280,201 @@ impl PartialDataColumn { })) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::MinimalEthSpec; + use bls::Signature; + use fixed_bytes::FixedBytesExtended; + use kzg::KzgCommitment; + + type E = MinimalEthSpec; + + fn make_cell(marker: u8) -> Cell { + let mut cell = Cell::::default(); + cell[0] = marker; + cell + } + + fn make_sidecar_with_marker( + total_blobs: usize, + present_indices: &[usize], + marker_base: u8, + ) -> PartialDataColumnSidecar { + let mut bitmap = CellBitmap::::with_capacity(total_blobs).unwrap(); + for &idx in present_indices { + bitmap.set(idx, true).unwrap(); + } + + let column: VariableList<_, _> = present_indices + .iter() + .map(|&idx| make_cell(marker_base.wrapping_add(idx as u8))) + .collect::>() + .try_into() + .unwrap(); + let proofs: VariableList<_, _> = present_indices + .iter() + .map(|_| KzgProof::empty()) + .collect::>() + .try_into() + .unwrap(); + + PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column, + kzg_proofs: proofs, + header: VariableList::new(vec![]).unwrap(), + } + } + + fn make_sidecar(total_blobs: usize, present_indices: &[usize]) -> PartialDataColumnSidecar { + make_sidecar_with_marker(total_blobs, present_indices, 0) + } + + fn make_header(num_commitments: usize) -> PartialDataColumnHeader { + PartialDataColumnHeader { + kzg_commitments: vec![KzgCommitment([0u8; 48]); num_commitments] + .try_into() + .unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: crate::BeaconBlockHeader { + slot: Slot::new(0), + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body_root: Hash256::zero(), + }, + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: FixedVector::new( + vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], + ) + .unwrap(), + } + } + + // -- filter tests -- + + #[test] + fn filter_keeps_matching_cells() { + let sidecar = make_sidecar(6, &[0, 2, 4]); + let filtered = sidecar.filter(|idx| idx == 0 || idx == 4).unwrap(); + assert_eq!(filtered.column.len(), 2); + assert_eq!(filtered.kzg_proofs.len(), 2); + assert!(filtered.cells_present_bitmap.get(0).unwrap()); + assert!(!filtered.cells_present_bitmap.get(2).unwrap()); + assert!(filtered.cells_present_bitmap.get(4).unwrap()); + } + + #[test] + fn filter_returns_none_when_no_overlap() { + let sidecar = make_sidecar(6, &[0, 2, 4]); + assert!(sidecar.filter(|idx| idx == 1 || idx == 3).is_none()); + } + + #[test] + fn filter_preserves_all_when_all_match() { + let sidecar = make_sidecar(6, &[0, 2, 4]); + let filtered = sidecar.filter(|_| true).unwrap(); + assert_eq!(filtered.column.len(), 3); + assert_eq!(filtered.kzg_proofs.len(), 3); + assert_eq!(filtered.cells_present_bitmap, sidecar.cells_present_bitmap); + + // Also, check that the encoded version matches + assert_eq!(filtered.as_ssz_bytes(), sidecar.as_ssz_bytes()); + } + + // -- merge tests -- + + #[test] + fn merge_disjoint_partials() { + let a = make_sidecar(6, &[0, 2]); + let b = make_sidecar(6, &[1, 3]); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.column.len(), 4); + assert_eq!(merged.kzg_proofs.len(), 4); + for i in 0..4 { + assert!(merged.cells_present_bitmap.get(i).unwrap()); + } + assert!(!merged.cells_present_bitmap.get(4).unwrap()); + } + + #[test] + fn merge_overlapping_partials_prefers_self() { + let a = make_sidecar_with_marker(4, &[0, 1], 0); + let b = make_sidecar_with_marker(4, &[1, 2], 100); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.column.len(), 3); + // Cell at bitmap index 1 is the second cell in the merged column. + // It should come from `a` (marker_base=0, so marker=0+1=1), not `b` (marker=100+1=101). + assert_eq!(merged.column[1][0], 1); + } + + #[test] + fn merge_with_empty_other() { + let a = make_sidecar(4, &[0, 2]); + let b = make_sidecar(4, &[]); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.column.len(), 2); + assert_eq!(merged.cells_present_bitmap, a.cells_present_bitmap); + } + + #[test] + fn merge_preserves_header_from_self() { + let mut a = make_sidecar(4, &[0]); + a.header = VariableList::new(vec![make_header(4)]).unwrap(); + let b = make_sidecar(4, &[1]); + let merged = a.merge(&b).unwrap(); + assert!(!merged.header.is_empty()); + + // If self has no header, use other's + let c = make_sidecar(4, &[0]); + let mut d = make_sidecar(4, &[1]); + d.header = VariableList::new(vec![make_header(4)]).unwrap(); + let merged2 = c.merge(&d).unwrap(); + assert!(!merged2.header.is_empty()); + } + + // -- is_complete tests -- + + #[test] + fn is_complete_true_when_all_bits_set() { + let sidecar = make_sidecar(4, &[0, 1, 2, 3]); + assert!(sidecar.is_complete()); + } + + #[test] + fn is_complete_false_when_partial() { + let sidecar = make_sidecar(4, &[0, 2]); + assert!(!sidecar.is_complete()); + } + + // -- try_clone_full tests (on PartialDataColumn) -- + + #[test] + fn try_clone_full_succeeds_when_complete() { + let sidecar = make_sidecar(3, &[0, 1, 2]); + let header = make_header(3); + let partial = PartialDataColumn { + block_root: Hash256::zero(), + index: 5, + sidecar, + }; + let full = partial.try_clone_full(&header).unwrap(); + assert_eq!(*full.index(), 5); + assert_eq!(full.column().len(), 3); + } + + #[test] + fn try_clone_full_returns_none_when_incomplete() { + let sidecar = make_sidecar(4, &[0, 2]); + let header = make_header(4); + let partial = PartialDataColumn { + block_root: Hash256::zero(), + index: 0, + sidecar, + }; + assert!(partial.try_clone_full(&header).is_none()); + } +} From bdc1f0953afdf179c745f3a89df0781f4a0dd656 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 25 Mar 2026 16:17:24 +0100 Subject: [PATCH 108/146] update libp2p --- Cargo.lock | 219 ++++++++++++++++++++++++++--------------------------- 1 file changed, 109 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 588bdc3945b..78e03e4f96d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -408,7 +408,7 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -519,7 +519,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -535,7 +535,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "syn-solidity", "tiny-keccak", ] @@ -552,7 +552,7 @@ dependencies = [ "macro-string", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "syn-solidity", ] @@ -641,7 +641,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -824,7 +824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -862,7 +862,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -973,7 +973,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -985,7 +985,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1064,7 +1064,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1075,7 +1075,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1117,7 +1117,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1385,7 +1385,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1405,7 +1405,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1588,7 +1588,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1864,7 +1864,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1965,7 +1965,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -1985,7 +1985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ff1dbbda10d495b2c92749c002b2025e0be98f42d1741ecc9ff820d2f04dce" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2097,7 +2097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b7bf98c48ffa511b14bb3c76202c24a8742cea1efa9570391c5d41373419a09" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2331,7 +2331,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2366,7 +2366,7 @@ dependencies = [ "quote", "serde", "strsim", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2379,7 +2379,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2390,7 +2390,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2401,7 +2401,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 1.0.109", ] [[package]] @@ -2550,7 +2550,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2582,7 +2582,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2595,7 +2595,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2617,7 +2617,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.111", + "syn 2.0.117", "unicode-xid", ] @@ -2733,7 +2733,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2828,7 +2828,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3051,7 +3051,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3071,7 +3071,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3301,7 +3301,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3645,12 +3645,12 @@ dependencies = [ [[package]] name = "futures-bounded" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f328e7fb845fc832912fb6a34f40cf6d1888c92f974d1893a54e97b5ff542e" +checksum = "b604752cefc5aa3ab98992a107a8bd99465d2825c1584e0b60cb6957b21e19d7" dependencies = [ - "futures-timer", "futures-util", + "tokio", ] [[package]] @@ -3705,7 +3705,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4582,7 +4582,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4918,9 +4918,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.178" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libloading" @@ -4956,7 +4956,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "bytes", "either", @@ -4987,7 +4987,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4997,7 +4997,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5007,7 +5007,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "either", "fnv", @@ -5030,10 +5030,9 @@ dependencies = [ [[package]] name = "libp2p-dns" -version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +version = "0.45.0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ - "async-trait", "futures", "hickory-resolver", "libp2p-core", @@ -5046,7 +5045,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5076,7 +5075,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "asynchronous-codec", "either", @@ -5116,7 +5115,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "hickory-proto", @@ -5134,7 +5133,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "libp2p-core", @@ -5150,7 +5149,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "asynchronous-codec", "bytes", @@ -5168,7 +5167,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "asynchronous-codec", "bytes", @@ -5190,7 +5189,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "futures-timer", @@ -5211,7 +5210,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "either", "fnv", @@ -5232,17 +5231,17 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "heck", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "libp2p-tcp" version = "0.44.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "futures-timer", @@ -5257,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "futures-rustls", @@ -5275,7 +5274,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "futures-timer", @@ -5289,7 +5288,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "either", "futures", @@ -5616,7 +5615,7 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5644,7 +5643,7 @@ checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5731,7 +5730,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5837,7 +5836,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5849,7 +5848,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5964,7 +5963,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "bytes", "futures", @@ -6284,7 +6283,7 @@ checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6533,7 +6532,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6633,7 +6632,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6836,7 +6835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6878,7 +6877,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6945,7 +6944,7 @@ checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6975,7 +6974,7 @@ checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -6998,7 +6997,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -7060,7 +7059,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "asynchronous-codec", "bytes", @@ -7082,7 +7081,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -7117,9 +7116,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7342,14 +7341,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -7747,7 +7746,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#b542bc2d9312ff2498ca1a85f3a2c8a8751b47e0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" dependencies = [ "futures", "pin-project", @@ -7983,7 +7982,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8007,7 +8006,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8050,7 +8049,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8521,7 +8520,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8541,7 +8540,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8567,9 +8566,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -8585,7 +8584,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8605,7 +8604,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8723,7 +8722,7 @@ name = "test_random_derive" version = "0.2.0" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8752,7 +8751,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8763,7 +8762,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -8920,9 +8919,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -8944,7 +8943,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -9184,7 +9183,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -9287,7 +9286,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -9888,7 +9887,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -10024,7 +10023,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -10086,7 +10085,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -10097,7 +10096,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -10550,7 +10549,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -10571,7 +10570,7 @@ checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -10591,7 +10590,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -10613,7 +10612,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -10646,7 +10645,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] From 5eb1a1d63bec1594a1b30b41bc7b5080db910783 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 31 Mar 2026 21:05:57 +0200 Subject: [PATCH 109/146] address review --- .../src/partial_data_column_assembler.rs | 4 +- .../src/network_beacon_processor/mod.rs | 28 ++++++------- .../src/data/partial_data_column_sidecar.rs | 40 ++----------------- 3 files changed, 19 insertions(+), 53 deletions(-) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index f7c070cdd06..fff2a216660 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -185,7 +185,7 @@ impl PartialDataColumnAssembler { &self, block_root: Hash256, header: &Arc>, - ) -> Option>> { + ) -> Vec> { let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { header: header.clone(), @@ -206,7 +206,7 @@ impl PartialDataColumnAssembler { } }) .collect(); - Some(columns) + columns } /// Get header for a block if we have an active assembly diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 4ff24c2b0ec..e5cb265ad5c 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -998,20 +998,20 @@ impl NetworkBeaconProcessor { } // Publish partial columns without eager send - if let Some(columns) = self - .chain - .data_availability_checker - .partial_assembler() - .and_then(|a| a.get_partials_and_mark_as_local_fetched(block_root, &header)) - { - debug!(block = %block_root, "Publishing all partials after getBlobs"); - self.send_network_message(NetworkMessage::PublishPartialColumns { - columns: columns - .into_iter() - .map(|partial| partial.into_inner()) - .collect(), - header, - }); + if let Some(assembler) = self.chain.data_availability_checker.partial_assembler() { + let columns = assembler.get_partials_and_mark_as_local_fetched(block_root, &header); + if !columns.is_empty() { + debug!(block = %block_root, "Publishing all partials after getBlobs"); + self.send_network_message(NetworkMessage::PublishPartialColumns { + columns: columns + .into_iter() + .map(|partial| partial.into_inner()) + .collect(), + header, + }); + } else { + debug!(block = %block_root, "No partials to publish after getBlobs"); + } } } diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 6c6b488ac7b..0597efd2b8c 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -7,7 +7,7 @@ use crate::{ use educe::Educe; use kzg::KzgProof; use merkle_proof::verify_merkle_proof; -use ssz::{BitList, Encode}; +use ssz::BitList; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use std::fmt::Display; @@ -28,7 +28,7 @@ pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnSidecar { pub cells_present_bitmap: CellBitmap, - pub column: VariableList, ::MaxBlobCommitmentsPerBlock>, + pub column: VariableList, E::MaxBlobCommitmentsPerBlock>, pub kzg_proofs: VariableList, pub header: VariableList, U1>, } @@ -46,41 +46,6 @@ pub struct PartialDataColumnSidecarRef<'a, E: EthSpec> { } impl PartialDataColumnSidecar { - pub fn min_size() -> usize { - // min size is one cell - Self { - cells_present_bitmap: BitList::with_capacity(1).unwrap(), - column: VariableList::new(vec![Cell::::default()]).unwrap(), - kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), - header: VariableList::new(vec![]).unwrap(), - } - .as_ssz_bytes() - .len() - } - - pub fn size(present_blobs: usize, block_blobs: usize) -> usize { - // min size is one cell - Self { - cells_present_bitmap: BitList::with_capacity(block_blobs).unwrap(), - column: VariableList::new(vec![Default::default(); present_blobs]).unwrap(), - kzg_proofs: VariableList::new(vec![KzgProof::empty(); present_blobs]).unwrap(), - header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push - } - .as_ssz_bytes() - .len() - } - - pub fn max_size(max_blobs_per_block: usize) -> usize { - Self { - cells_present_bitmap: BitList::with_capacity(max_blobs_per_block).unwrap(), - column: VariableList::new(vec![Default::default(); max_blobs_per_block]).unwrap(), - kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), - header: VariableList::new(vec![]).unwrap(), // header is not being sent on cell push - } - .as_ssz_bytes() - .len() - } - pub fn is_complete(&self) -> bool { self.cells_present_bitmap.iter().all(|bit| bit) } @@ -288,6 +253,7 @@ mod tests { use bls::Signature; use fixed_bytes::FixedBytesExtended; use kzg::KzgCommitment; + use ssz::Encode; type E = MinimalEthSpec; From e1a1b764aacefad50f516905083adba2ea8cbe22 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 9 Apr 2026 15:32:52 +0200 Subject: [PATCH 110/146] fix lint --- .../beacon_chain/src/partial_data_column_assembler.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index fff2a216660..0595a8aca66 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -195,7 +195,7 @@ impl PartialDataColumnAssembler { assembly.has_local_blobs = true; - let columns = assembly + assembly .columns .values() .filter_map(|value| { @@ -205,8 +205,7 @@ impl PartialDataColumnAssembler { None } }) - .collect(); - columns + .collect() } /// Get header for a block if we have an active assembly From 8a1b80c7cededf2b08c211bd7cf6838def8dedd0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 9 Apr 2026 15:56:09 +0200 Subject: [PATCH 111/146] fix getBlobs when partial messages are not enabled --- .../src/data_column_verification.rs | 16 ++++++++ .../beacon_chain/src/fetch_blobs/mod.rs | 41 +++++++++++-------- .../src/data/partial_data_column_sidecar.rs | 20 +++++++++ 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index e49b6db9d61..f76ebe15754 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -685,6 +685,22 @@ impl KzgVerifiedCustodyPartialDataColumn { data: Arc::new(data), }) } + + /// Try to convert the partial data column into a full one, returning None if the conversion + /// fails. + /// May clone the column if the Arc cannot be unwrapped. + pub fn try_into_full( + self, + header: &PartialDataColumnHeader, + ) -> Option> { + match Arc::try_unwrap(self.data) { + Ok(data) => data.try_into_full(header), + Err(data) => data.try_clone_full(header), + } + .map(|data| KzgVerifiedCustodyDataColumn { + data: Arc::new(data), + }) + } } /// Complete kzg verification for a `DataColumnSidecar`. diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 87a9b40b413..b335dd3a94f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -329,34 +329,43 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); } - // Initialize the partial assembler with the columns from the engine - let Some(assembler) = chain_adapter.partial_assembler() else { - return Ok(None); + let full_columns = match chain_adapter.partial_assembler() { + Some(assembler) => { + // Initialize the partial assembler with the columns from the engine and return any full + // columns for publishing + assembler + .merge_partials(block_root, custody_columns_to_import, header) + .ok_or_else(|| { + FetchEngineBlobError::InternalError( + "Failed to merge partials into assembler".to_string(), + ) + })? + .full_columns + } + None => { + // Partial columns are disabled, so let's try to directly convert the columns we got + // from the EL into full columns. + custody_columns_to_import + .into_iter() + .filter_map(|col| col.try_into_full(&header)) + .collect() + } }; - let merge_result = assembler - .merge_partials(block_root, custody_columns_to_import, header) - .ok_or_else(|| { - FetchEngineBlobError::InternalError( - "Failed to merge partials into assembler".to_string(), - ) - })?; // Publish complete columns - if !merge_result.full_columns.is_empty() { - publish_fn(EngineGetBlobsOutput::CustodyColumns( - merge_result.full_columns.clone(), - )); + if !full_columns.is_empty() { + publish_fn(EngineGetBlobsOutput::CustodyColumns(full_columns.clone())); } // We publish all partials at the calling site, regardless of result, as previous publishs // have been blocked, waiting for the results of this call // Process complete columns through DA checker - let availability_processing_status = if !merge_result.full_columns.is_empty() { + let availability_processing_status = if !full_columns.is_empty() { chain_adapter .process_engine_blobs( slot, block_root, - EngineGetBlobsOutput::CustodyColumns(merge_result.full_columns), + EngineGetBlobsOutput::CustodyColumns(full_columns), ) .await? } else { diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 0597efd2b8c..1cadf5801b1 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -227,6 +227,8 @@ pub struct PartialDataColumn { } impl PartialDataColumn { + /// Equivalent to a call to `clone` followed by `try_into_full`, but returns early if conversion + /// is not possible. pub fn try_clone_full( &self, header: &PartialDataColumnHeader, @@ -244,6 +246,24 @@ impl PartialDataColumn { kzg_commitments_inclusion_proof: header.kzg_commitments_inclusion_proof.clone(), })) } + + pub fn try_into_full( + self, + header: &PartialDataColumnHeader, + ) -> Option> { + if !self.sidecar.is_complete() { + return None; + } + + Some(DataColumnSidecar::Fulu(DataColumnSidecarFulu { + index: self.index, + column: self.sidecar.column, + kzg_commitments: header.kzg_commitments.clone(), + kzg_proofs: self.sidecar.kzg_proofs, + signed_block_header: header.signed_block_header.clone(), + kzg_commitments_inclusion_proof: header.kzg_commitments_inclusion_proof.clone(), + })) + } } #[cfg(test)] From 3b9461502af08deab5983a8ae0fbdf7fa0fc13d5 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Thu, 9 Apr 2026 15:45:14 +0200 Subject: [PATCH 112/146] track latest cell timestamp --- beacon_node/beacon_chain/src/beacon_chain.rs | 7 ++- .../src/data_column_verification.rs | 50 ++++++++++++++++--- .../gossip_methods.rs | 8 ++- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index bee33879959..263dc5a3b7b 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2299,11 +2299,16 @@ impl BeaconChain { pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, data_column_sidecar: PartialDataColumn, + seen_timestamp: Duration, ) -> PartialColumnVerificationResult { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); - let ret = validate_partial_data_column_sidecar_for_gossip(data_column_sidecar, self); + let ret = validate_partial_data_column_sidecar_for_gossip( + data_column_sidecar, + self, + seen_timestamp, + ); if matches!(ret, PartialColumnVerificationResult::Ok { .. }) { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index f76ebe15754..151cb6dadb8 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -7,6 +7,7 @@ use crate::kzg_utils::{ use crate::observed_data_sidecars::{ Error as ObservedDataSidecarsError, ObservationKey, ObservationStrategy, Observe, }; +use crate::validator_monitor::timestamp_now; use crate::{BeaconChain, BeaconChainError, BeaconChainTypes, metrics}; use educe::Educe; use fork_choice::ProtoBlock; @@ -18,6 +19,7 @@ use ssz_types::VariableList; use std::iter; use std::marker::PhantomData; use std::sync::Arc; +use std::time::Duration; use tracing::{debug, instrument}; use tree_hash::TreeHash; use types::{ @@ -433,18 +435,25 @@ impl KzgVerifiedDataColumn { #[educe(PartialEq, Eq)] pub struct KzgVerifiedPartialDataColumn { data: PartialDataColumn, + latest_cell_timestamp: Duration, } impl KzgVerifiedPartialDataColumn { /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. pub(crate) fn __new_for_testing(data_column: PartialDataColumn) -> Self { - Self { data: data_column } + Self { + data: data_column, + latest_cell_timestamp: timestamp_now(), + } } /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. pub fn from_execution_verified(data_column: PartialDataColumn) -> Self { - Self { data: data_column } + Self { + data: data_column, + latest_cell_timestamp: timestamp_now(), + } } pub fn to_data_column(self) -> PartialDataColumn { @@ -639,6 +648,7 @@ impl KzgVerifiedCustodyDataColumn { #[educe(PartialEq, Eq)] pub struct KzgVerifiedCustodyPartialDataColumn { data: Arc>, + latest_cell_timestamp: Duration, } impl KzgVerifiedCustodyPartialDataColumn { @@ -646,6 +656,7 @@ impl KzgVerifiedCustodyPartialDataColumn { /// include this column pub fn from_asserted_custody(kzg_verified: KzgVerifiedPartialDataColumn) -> Self { Self { + latest_cell_timestamp: kzg_verified.latest_cell_timestamp, data: Arc::new(kzg_verified.to_data_column()), } } @@ -672,6 +683,7 @@ impl KzgVerifiedCustodyPartialDataColumn { index: self.data.index, sidecar, }), + latest_cell_timestamp: self.latest_cell_timestamp.max(other.latest_cell_timestamp), }) } @@ -725,6 +737,7 @@ pub fn verify_kzg_for_partial_data_column( unverified_cells: Vec, header: &GossipVerifiedPartialDataColumnHeader, kzg: &Kzg, + seen_timestamp: Duration, ) -> Result, GossipPartialDataColumnError> { let Some(filtered_column) = data_column .sidecar @@ -740,7 +753,10 @@ pub fn verify_kzg_for_partial_data_column( header.header.kzg_commitments.as_ref(), ) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; - Ok(KzgVerifiedPartialDataColumn { data: data_column }) + Ok(KzgVerifiedPartialDataColumn { + data: data_column, + latest_cell_timestamp: seen_timestamp, + }) } /// Complete kzg verification for a list of `DataColumnSidecar`s. @@ -836,6 +852,7 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( mut column: PartialDataColumn, chain: &BeaconChain, + seen_timestamp: Duration, ) -> PartialColumnVerificationResult { let block_root = column.block_root; @@ -922,7 +939,13 @@ pub fn validate_partial_data_column_sidecar_for_gossip( // We do not have to check block related data here, as we create the verifiable column from // gossip accepted block let kzg = &chain.kzg; - let column = match verify_kzg_for_partial_data_column(column, missing_cells, &header, kzg) { + let column = match verify_kzg_for_partial_data_column( + column, + missing_cells, + &header, + kzg, + seen_timestamp, + ) { Ok(column) => column, Err(err) => { return PartialColumnVerificationResult::ErrWithValidHeader { err, header }; @@ -1258,6 +1281,7 @@ mod test { use execution_layer::test_utils::generate_blobs; use ssz::BitList; use std::sync::Arc; + use std::time::UNIX_EPOCH; use types::{ DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, ForkName, MainnetEthSpec, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, @@ -1468,7 +1492,11 @@ mod test { }, }; - let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + let result = validate_partial_data_column_sidecar_for_gossip( + column, + &harness.chain, + UNIX_EPOCH.elapsed().unwrap(), + ); assert!( matches!( result, @@ -1506,7 +1534,11 @@ mod test { }, }; - let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + let result = validate_partial_data_column_sidecar_for_gossip( + column, + &harness.chain, + UNIX_EPOCH.elapsed().unwrap(), + ); assert!( matches!( result, @@ -1540,7 +1572,11 @@ mod test { }, }; - let result = validate_partial_data_column_sidecar_for_gossip(column, &harness.chain); + let result = validate_partial_data_column_sidecar_for_gossip( + column, + &harness.chain, + UNIX_EPOCH.elapsed().unwrap(), + ); assert!( matches!( result, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 16fda0e37b7..1009f23f0b7 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -821,7 +821,7 @@ impl NetworkBeaconProcessor { let result = self .chain - .verify_partial_data_column_sidecar_for_gossip(column); + .verify_partial_data_column_sidecar_for_gossip(column, seen_duration); let header = match result { PartialColumnVerificationResult::Ok { header, column } => { @@ -839,8 +839,8 @@ impl NetworkBeaconProcessor { ); // Log metrics to keep track of propagation delay times. - if let Some(duration) = SystemTime::now() - .duration_since(UNIX_EPOCH) + if let Some(duration) = UNIX_EPOCH + .elapsed() .ok() .and_then(|now| now.checked_sub(seen_duration)) { @@ -855,7 +855,6 @@ impl NetworkBeaconProcessor { column, header.clone(), slot, - seen_duration, ) .await; Some(header) @@ -1391,7 +1390,6 @@ impl NetworkBeaconProcessor { verified_partial: KzgVerifiedPartialDataColumn, verified_header: GossipVerifiedPartialDataColumnHeader, slot: Slot, - _seen_duration: Duration, ) { let processing_start_time = Instant::now(); let block_root = verified_partial.block_root(); From e4a72ada782dc83d3f846f87d031b9aef49510e1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 10 Apr 2026 15:46:16 +0200 Subject: [PATCH 113/146] update ssz_types --- Cargo.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49a96904ccf..4a242f93f15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -695,7 +695,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -706,7 +706,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1397,7 +1397,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.12.1", "log", "prettyplease", "proc-macro2", @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6196,7 +6196,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7001,7 +7001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.117", @@ -7652,7 +7652,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8385,9 +8385,9 @@ dependencies = [ [[package]] name = "ssz_types" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc20a89bab2dabeee65e9c9eb96892dc222c23254b401e1319b85efd852fa31" +checksum = "d625e4de8e0057eefe7e0b1510ba1dd7adf10cd375fad6cc7fcceac7c39623c9" dependencies = [ "arbitrary", "context_deserialize", @@ -8697,7 +8697,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10016,7 +10016,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From ad57d831e41e6ec4f0c9a67b2dae7310daebb60d Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 10 Apr 2026 15:49:40 +0200 Subject: [PATCH 114/146] Arc/Box some more This avoids some Clippy warnings in future commits and makes it more consistent with full columns --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../src/data_column_verification.rs | 22 +++++++++---------- .../beacon_chain/src/fetch_blobs/mod.rs | 4 +++- .../src/partial_data_column_assembler.rs | 2 +- .../lighthouse_network/src/service/mod.rs | 4 ++-- .../gossip_methods.rs | 2 +- .../src/network_beacon_processor/mod.rs | 2 +- beacon_node/network/src/router.rs | 2 +- 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 31a28e1a5f6..4adc3ee1bc1 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2298,7 +2298,7 @@ impl BeaconChain { #[instrument(skip_all, level = "trace")] pub fn verify_partial_data_column_sidecar_for_gossip( self: &Arc, - data_column_sidecar: PartialDataColumn, + data_column_sidecar: Box>, seen_timestamp: Duration, ) -> PartialColumnVerificationResult { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index dd6e2e320d1..a1a919db9e7 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -444,13 +444,13 @@ impl KzgVerifiedDataColumn { #[derive(Debug, Educe, Clone)] #[educe(PartialEq, Eq)] pub struct KzgVerifiedPartialDataColumn { - data: PartialDataColumn, + data: Arc>, latest_cell_timestamp: Duration, } impl KzgVerifiedPartialDataColumn { /// Create a `KzgVerifiedPartialDataColumn` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: PartialDataColumn) -> Self { + pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { Self { data: data_column, latest_cell_timestamp: timestamp_now(), @@ -459,14 +459,14 @@ impl KzgVerifiedPartialDataColumn { /// Mark a partial data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: PartialDataColumn) -> Self { + pub fn from_execution_verified(data_column: Arc>) -> Self { Self { data: data_column, latest_cell_timestamp: timestamp_now(), } } - pub fn to_data_column(self) -> PartialDataColumn { + pub fn to_data_column(self) -> Arc> { self.data } @@ -691,7 +691,7 @@ impl KzgVerifiedCustodyPartialDataColumn { pub fn from_asserted_custody(kzg_verified: KzgVerifiedPartialDataColumn) -> Self { Self { latest_cell_timestamp: kzg_verified.latest_cell_timestamp, - data: Arc::new(kzg_verified.to_data_column()), + data: kzg_verified.to_data_column(), } } @@ -773,7 +773,7 @@ pub fn verify_kzg_for_data_column( /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( - data_column: PartialDataColumn, + data_column: Box>, unverified_cells: Vec, header: &GossipVerifiedPartialDataColumnHeader, kzg: &Kzg, @@ -794,7 +794,7 @@ pub fn verify_kzg_for_partial_data_column( ) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; Ok(KzgVerifiedPartialDataColumn { - data: data_column, + data: data_column.into(), latest_cell_timestamp: seen_timestamp, }) } @@ -892,7 +892,7 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( - mut column: PartialDataColumn, + mut column: Box>, chain: &BeaconChain, seen_timestamp: Duration, ) -> PartialColumnVerificationResult { @@ -1535,7 +1535,7 @@ mod test { }; let result = validate_partial_data_column_sidecar_for_gossip( - column, + Box::new(column), &harness.chain, UNIX_EPOCH.elapsed().unwrap(), ); @@ -1577,7 +1577,7 @@ mod test { }; let result = validate_partial_data_column_sidecar_for_gossip( - column, + Box::new(column), &harness.chain, UNIX_EPOCH.elapsed().unwrap(), ); @@ -1615,7 +1615,7 @@ mod test { }; let result = validate_partial_data_column_sidecar_for_gossip( - column, + Box::new(column), &harness.chain, UNIX_EPOCH.elapsed().unwrap(), ); diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 302ae019773..f7b4b8a29e9 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -421,7 +421,9 @@ async fn compute_custody_columns_to_import( .filter(|col| custody_columns_indices.contains(&col.index)) .map(|col| { KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( - KzgVerifiedPartialDataColumn::from_execution_verified(col), + KzgVerifiedPartialDataColumn::from_execution_verified( + Arc::new(col), + ), ) }) .collect::>() diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 0595a8aca66..3a20b71a49e 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -343,7 +343,7 @@ mod tests { }, }; KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( - KzgVerifiedPartialDataColumn::__new_for_testing(partial), + KzgVerifiedPartialDataColumn::__new_for_testing(Arc::new(partial)), ) } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 5a3d509e728..86602595afc 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -108,7 +108,7 @@ pub enum NetworkEvent { /// The peer from which we received this message. source: PeerId, /// The partial column data. - column: PartialDataColumn, + column: Box>, /// The topic that this message was sent on. topic: GossipTopic, }, @@ -1434,7 +1434,7 @@ impl Network { // Notify the network return Some(NetworkEvent::PartialDataColumnSidecar { source: peer_id, - column, + column: Box::new(column), topic, }); } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 1009f23f0b7..cff8e7993f1 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -812,7 +812,7 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column: PartialDataColumn, + column: Box>, seen_duration: Duration, topic: GossipTopic, ) { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index e5cb265ad5c..448d70eb424 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -255,7 +255,7 @@ impl NetworkBeaconProcessor { pub fn send_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - column_sidecar: PartialDataColumn, + column_sidecar: Box>, seen_timestamp: Duration, topic: GossipTopic, ) -> Result<(), Error> { diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 624f87bc440..443fa51cc67 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -72,7 +72,7 @@ pub enum RouterMessage { /// by the beacon chain after successful verification. PubsubMessage(MessageId, PeerId, PubsubMessage, bool), /// A partial data column sidecar has been received via gossipsub partial protocol. - PartialDataColumnSidecar(PeerId, PartialDataColumn, GossipTopic), + PartialDataColumnSidecar(PeerId, Box>, GossipTopic), /// The peer manager has requested we re-status a peer. StatusPeer(PeerId), /// The peer has an updated custody group count from METADATA. From ca4b5f0003e3d4cff44eeaa46725caf55b9b537c Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 10 Apr 2026 15:49:40 +0200 Subject: [PATCH 115/146] Use ListEncodedOption --- .../src/data_column_verification.rs | 8 ++--- beacon_node/beacon_chain/src/kzg_utils.rs | 2 +- .../src/partial_data_column_assembler.rs | 34 ++++++++----------- beacon_node/http_api/src/publish_blocks.rs | 3 +- .../lighthouse_network/src/types/partial.rs | 6 ++-- .../gossip_methods.rs | 2 +- .../types/src/data/data_column_sidecar.rs | 5 +-- .../src/data/partial_data_column_sidecar.rs | 28 ++++++--------- 8 files changed, 39 insertions(+), 49 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index a1a919db9e7..cd6249b055a 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -899,7 +899,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip( let block_root = column.block_root; // Remove the header (if any) to avoid wasted memory. - let header = column.sidecar.take_header(); + let header = column.sidecar.header.take(); let header = if let Some(header) = header { // Header was sent, so it is required to be valid @@ -1530,7 +1530,7 @@ mod test { cells_present_bitmap: empty_bitmap, column: vec![].try_into().unwrap(), kzg_proofs: vec![].try_into().unwrap(), - header: ssz_types::VariableList::new(vec![]).unwrap(), + header: None.into(), }, }; @@ -1572,7 +1572,7 @@ mod test { kzg_proofs: vec![types::KzgProof::empty(), types::KzgProof::empty()] .try_into() .unwrap(), - header: ssz_types::VariableList::new(vec![]).unwrap(), + header: None.into(), }, }; @@ -1610,7 +1610,7 @@ mod test { cells_present_bitmap: bitmap, column: vec![types::Cell::::default()].try_into().unwrap(), kzg_proofs: vec![types::KzgProof::empty()].try_into().unwrap(), - header: ssz_types::VariableList::new(vec![]).unwrap(), + header: None.into(), }, }; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 99df3989844..f62a9c0e1a6 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -562,7 +562,7 @@ pub(crate) fn build_partial_data_columns( .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - header: VariableList::empty(), + header: None.into(), }, }; Ok(column) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 3a20b71a49e..f37ce983fd2 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -270,8 +270,8 @@ mod tests { cell } - fn make_header(num_commitments: usize) -> Arc> { - Arc::new(PartialDataColumnHeader { + fn make_header(num_commitments: usize) -> PartialDataColumnHeader { + PartialDataColumnHeader { kzg_commitments: vec![KzgCommitment([0u8; 48]); num_commitments] .try_into() .unwrap(), @@ -289,7 +289,7 @@ mod tests { vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], ) .unwrap(), - }) + } } fn make_partial( @@ -326,11 +326,7 @@ mod tests { .try_into() .unwrap(); - let header_list = if include_header { - VariableList::new(vec![(*make_header(total_blobs)).clone()]).unwrap() - } else { - VariableList::new(vec![]).unwrap() - }; + let header = include_header.then(|| make_header(total_blobs)).into(); let partial = PartialDataColumn { block_root, @@ -339,7 +335,7 @@ mod tests { cells_present_bitmap: bitmap, column, kzg_proofs: proofs, - header: header_list, + header, }, }; KzgVerifiedCustodyPartialDataColumn::from_asserted_custody( @@ -358,16 +354,16 @@ mod tests { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); let header = make_header(4); - assert!(assembler.init(root, header.clone())); + assert!(assembler.init(root, Arc::new(header.clone()))); let retrieved = assembler.get_header(&root).unwrap(); - assert_eq!(*retrieved, *header); + assert_eq!(*retrieved, header); } #[test] fn init_returns_false_if_already_exists() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); assert!(assembler.init(root, header.clone())); assert!(!assembler.init(root, header)); } @@ -378,7 +374,7 @@ mod tests { fn merge_partials_tracks_added_cells() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); let partial = make_partial(root, 0, 4, &[0, 1, 2]); let result = assembler @@ -399,7 +395,7 @@ mod tests { fn merge_partials_ignores_already_complete_column() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); // Complete the column let partial = make_partial(root, 0, 4, &[0, 1, 2, 3]); @@ -422,7 +418,7 @@ mod tests { fn merge_partials_completes_column_progressively() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); let partial1 = make_partial(root, 0, 4, &[0, 1]); let result1 = assembler @@ -441,7 +437,7 @@ mod tests { fn merge_partials_returns_updated_partials() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); let partial = make_partial(root, 0, 4, &[0, 2]); let result = assembler @@ -457,7 +453,7 @@ mod tests { fn mark_as_complete_replaces_incomplete() { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let header = make_header(4); + let header = Arc::new(make_header(4)); // Merge an incomplete partial first let partial = make_partial(root, 0, 4, &[0, 1]); @@ -522,7 +518,7 @@ mod tests { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); // Header at slot 0 → epoch 0 - let header = make_header(4); + let header = Arc::new(make_header(4)); assembler.init(root, header); assert!(assembler.get_header(&root).is_some()); @@ -536,7 +532,7 @@ mod tests { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); // Header at slot 100 → epoch 100/8 = 12 for MinimalEthSpec (8 slots/epoch) - let mut header = (*make_header(4)).clone(); + let mut header = make_header(4); header.signed_block_header.message.slot = Slot::new(100); let header = Arc::new(header); assembler.init(root, header); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 1c72cffe747..dd1e9a94bd1 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -523,8 +523,7 @@ fn publish_column_sidecars( for data_col in data_column_sidecars { if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { let mut partial = fulu_data_col.to_partial(); - let header = partial.sidecar.take_header(); - if let Some(header) = header { + if let Some(header) = partial.sidecar.header.take() { partial_header = Some(header); } partial_columns.push(Arc::new(partial)); diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 4fbea559e69..ea0a40a451e 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -50,7 +50,7 @@ impl OutgoingPartialColumn { .expect("Taking length from bitmap with same bound"), column: vec![], kzg_proofs: vec![], - header: vec![header], + header: Some(header).into(), } .as_ssz_bytes(); @@ -256,7 +256,7 @@ mod tests { use bls::Signature; use fixed_bytes::FixedBytesExtended; use libp2p::identity::Keypair; - use ssz_types::{FixedVector, VariableList}; + use ssz_types::FixedVector; use types::{ BeaconBlockHeader, MinimalEthSpec, PartialDataColumnHeader, SignedBeaconBlockHeader, Slot, }; @@ -318,7 +318,7 @@ mod tests { .collect::>() .try_into() .unwrap(), - header: VariableList::new(vec![]).unwrap(), + header: None.into(), }, }) } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index cff8e7993f1..8ca4cf9ef8c 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1308,7 +1308,7 @@ impl NetworkBeaconProcessor { ); let mut column = col.to_partial(); - let header = column.sidecar.take_header(); + let header = column.sidecar.header.take(); if let Some(header) = header { self.send_network_message(NetworkMessage::PublishPartialColumns { columns: vec![Arc::new(column)], diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index 3eb5b6e6c54..f692bd9ca41 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -229,11 +229,12 @@ impl DataColumnSidecarFulu { cells_present_bitmap: bitmap, column: self.column.clone(), kzg_proofs: self.kzg_proofs.clone(), - header: VariableList::repeat_full(PartialDataColumnHeader { + header: Some(PartialDataColumnHeader { kzg_commitments: self.kzg_commitments.clone(), signed_block_header: self.signed_block_header.clone(), kzg_commitments_inclusion_proof: self.kzg_commitments_inclusion_proof.clone(), - }), + }) + .into(), }, } } diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 1cadf5801b1..060cd223d71 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -9,13 +9,11 @@ use kzg::KzgProof; use merkle_proof::verify_merkle_proof; use ssz::BitList; use ssz_derive::{Decode, Encode}; -use ssz_types::{FixedVector, VariableList}; +use ssz_types::{FixedVector, ListEncodedOption, VariableList}; use std::fmt::Display; -use std::mem; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use typenum::U1; pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; @@ -24,13 +22,13 @@ pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; derive(arbitrary::Arbitrary), arbitrary(bound = "E: EthSpec") )] -#[derive(Debug, Clone, Encode, Decode, TreeHash, TestRandom, Educe)] +#[derive(Debug, Clone, Encode, Decode, TreeHash, Educe)] #[educe(PartialEq, Eq, Hash(bound = "E: EthSpec"))] pub struct PartialDataColumnSidecar { pub cells_present_bitmap: CellBitmap, pub column: VariableList, E::MaxBlobCommitmentsPerBlock>, pub kzg_proofs: VariableList, - pub header: VariableList, U1>, + pub header: ListEncodedOption>, } /// Equivalent to `PartialDataColumnSidecar`, but containing references to the cells. This is done @@ -42,7 +40,7 @@ pub struct PartialDataColumnSidecarRef<'a, E: EthSpec> { // this from the `PartialDataColumnSidecar` type above. This avoids a few ugly `expect` calls. pub column: Vec<&'a Cell>, pub kzg_proofs: Vec<&'a KzgProof>, - pub header: Vec<&'a PartialDataColumnHeader>, + pub header: ListEncodedOption<&'a PartialDataColumnHeader>, } impl PartialDataColumnSidecar { @@ -89,7 +87,7 @@ impl PartialDataColumnSidecar { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, - header: self.header.iter().collect(), + header: self.header.as_ref().into(), }) } @@ -141,17 +139,13 @@ impl PartialDataColumnSidecar { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, - header: if !self.header.is_empty() { + header: if !self.header.is_some() { self.header.clone() } else { other.header.clone() }, }) } - - pub fn take_header(&mut self) -> Option> { - Vec::from(mem::take(&mut self.header)).pop() - } } #[cfg_attr( @@ -310,7 +304,7 @@ mod tests { cells_present_bitmap: bitmap, column, kzg_proofs: proofs, - header: VariableList::new(vec![]).unwrap(), + header: None.into(), } } @@ -409,17 +403,17 @@ mod tests { #[test] fn merge_preserves_header_from_self() { let mut a = make_sidecar(4, &[0]); - a.header = VariableList::new(vec![make_header(4)]).unwrap(); + a.header = Some(make_header(4)).into(); let b = make_sidecar(4, &[1]); let merged = a.merge(&b).unwrap(); - assert!(!merged.header.is_empty()); + assert!(merged.header.is_some()); // If self has no header, use other's let c = make_sidecar(4, &[0]); let mut d = make_sidecar(4, &[1]); - d.header = VariableList::new(vec![make_header(4)]).unwrap(); + d.header = Some(make_header(4)).into(); let merged2 = c.merge(&d).unwrap(); - assert!(!merged2.header.is_empty()); + assert!(merged2.header.is_some()); } // -- is_complete tests -- From edfa71c6d308fc6c008ed0ed95e147866c6af5c4 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 13 Apr 2026 12:31:37 +0200 Subject: [PATCH 116/146] share harness across tests --- .../src/data_column_verification.rs | 92 +++++++++++-------- 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index cd6249b055a..94d8d396cbf 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -1317,7 +1317,8 @@ mod test { }; use crate::observed_data_sidecars::Observe; use crate::test_utils::{ - BeaconChainHarness, EphemeralHarnessType, generate_data_column_sidecars_from_block, + BeaconChainHarness, EphemeralHarnessType, fork_name_from_env, + generate_data_column_sidecars_from_block, test_spec, }; use eth2::types::BlobsBundle; use execution_layer::test_utils::generate_blobs; @@ -1452,33 +1453,48 @@ mod test { )); } - /// Build a Fulu harness with partial columns enabled. - async fn partial_column_harness() -> BeaconChainHarness> { - let spec = ForkName::Fulu.make_genesis_spec(E::default_spec()); + #[tokio::test] + async fn test_partial_message_verification_fulu() { + let spec = if fork_name_from_env().is_some() { + Arc::new(test_spec::()) + } else { + Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec())) + }; + + // Only run these tests if columns are enabled. + if !spec.is_fulu_scheduled() { + return; + } + // Gloas is not supported yet. + if spec.is_gloas_scheduled() { + return; + } + let chain_config = ChainConfig { enable_partial_columns: true, ..Default::default() }; let harness = BeaconChainHarness::builder(E::default()) - .spec(spec.into()) + .spec(spec) .deterministic_keypairs(64) .fresh_ephemeral_store() .mock_execution_layer() .chain_config(chain_config) .build(); - harness.advance_slot(); - harness - } - /// Build a partial column harness with a block containing 1 blob commitment - /// and the header pre-cached in the partial assembler. - async fn partial_column_harness_with_block() -> ( - BeaconChainHarness>, - types::Hash256, - Arc>, - ) { - let harness = partial_column_harness().await; + partial_empty_message_without_cells_returns_error(&harness).await; + partial_inconsistent_present_count_returns_error(&harness).await; + partial_inconsistent_max_count_returns_error(&harness).await; + partial_header_with_empty_commitments_fails(&harness).await; + partial_header_root_mismatch_fails(&harness).await; + partial_header_with_invalid_inclusion_proof_fails(&harness).await; + } + /// Build a block containing 1 blob and pre-cache the header in the partial assembler. + async fn add_block_and_header( + harness: &BeaconChainHarness>, + ) -> (types::Hash256, Arc>) { + harness.advance_slot(); // Generate a block with 1 blob so we have valid data columns. let fork = harness .spec @@ -1510,12 +1526,13 @@ mod test { .unwrap() .init(block_root, header.clone()); - (harness, block_root, header) + (block_root, header) } - #[tokio::test] - async fn partial_empty_message_without_cells_returns_error() { - let (harness, block_root, header) = partial_column_harness_with_block().await; + async fn partial_empty_message_without_cells_returns_error( + harness: &BeaconChainHarness>, + ) { + let (block_root, header) = add_block_and_header(harness).await; // Create a headerless partial with no cells — should trigger EmptyMessage. let num_commitments = header.kzg_commitments.len(); @@ -1551,9 +1568,10 @@ mod test { ); } - #[tokio::test] - async fn partial_inconsistent_present_count_returns_error() { - let (harness, block_root, header) = partial_column_harness_with_block().await; + async fn partial_inconsistent_present_count_returns_error( + harness: &BeaconChainHarness>, + ) { + let (block_root, header) = add_block_and_header(harness).await; // Create a bitmap that says 2 bits are set, but only provide 1 cell/proof. let num_commitments = header.kzg_commitments.len(); @@ -1593,9 +1611,10 @@ mod test { ); } - #[tokio::test] - async fn partial_inconsistent_max_count_returns_error() { - let (harness, block_root, _header) = partial_column_harness_with_block().await; + async fn partial_inconsistent_max_count_returns_error( + harness: &BeaconChainHarness>, + ) { + let (block_root, _header) = add_block_and_header(harness).await; // Create a bitmap with length different from the number of commitments in the header. // Header has 1 commitment, but we use a bitmap with capacity 3. @@ -1631,10 +1650,9 @@ mod test { ); } - #[tokio::test] - async fn partial_header_with_empty_commitments_fails() { - let harness = partial_column_harness().await; - + async fn partial_header_with_empty_commitments_fails( + harness: &BeaconChainHarness>, + ) { let slot = harness.get_current_slot(); let state = harness.get_current_state(); let ((block, _), _) = harness @@ -1660,9 +1678,10 @@ mod test { ); } - #[tokio::test] - async fn partial_header_root_mismatch_fails() { - let (harness, _block_root, header) = partial_column_harness_with_block().await; + async fn partial_header_root_mismatch_fails( + harness: &BeaconChainHarness>, + ) { + let (_block_root, header) = add_block_and_header(harness).await; // Use a wrong group_id (not matching the header's block root) let wrong_root = types::Hash256::repeat_byte(0xff); @@ -1679,9 +1698,10 @@ mod test { ); } - #[tokio::test] - async fn partial_header_with_invalid_inclusion_proof_fails() { - let (harness, block_root, header) = partial_column_harness_with_block().await; + async fn partial_header_with_invalid_inclusion_proof_fails( + harness: &BeaconChainHarness>, + ) { + let (block_root, header) = add_block_and_header(harness).await; // Corrupt the inclusion proof let mut header = PartialDataColumnHeader::clone(&header); From 00e9c83ee94a83789c7fcef551e55deb93ef956a Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 13 Apr 2026 14:47:27 +0200 Subject: [PATCH 117/146] Rename InconsistentMaxCount --- beacon_node/beacon_chain/src/data_column_verification.rs | 6 +++--- .../network/src/network_beacon_processor/gossip_methods.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 94d8d396cbf..9f01995d13c 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -244,7 +244,7 @@ pub enum GossipPartialDataColumnError { /// /// ## Peer scoring /// The column sidecar is invalid and the peer is faulty - InconsistentMaxCount { + InconsistentCommitmentsLength { bitmap_len: usize, commitments_len: usize, }, @@ -952,7 +952,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip( let commitments_len = header.as_header().kzg_commitments.len(); if bitmap_len != commitments_len { return PartialColumnVerificationResult::ErrWithValidHeader { - err: GossipPartialDataColumnError::InconsistentMaxCount { + err: GossipPartialDataColumnError::InconsistentCommitmentsLength { bitmap_len, commitments_len, }, @@ -1642,7 +1642,7 @@ mod test { matches!( result, PartialColumnVerificationResult::ErrWithValidHeader { - err: GossipPartialDataColumnError::InconsistentMaxCount { .. }, + err: GossipPartialDataColumnError::InconsistentCommitmentsLength { .. }, .. } ), diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 8ca4cf9ef8c..cda6a44b85c 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1019,7 +1019,7 @@ impl NetworkBeaconProcessor { } GossipPartialDataColumnError::EmptyMessage | GossipPartialDataColumnError::InconsistentPresentCount { .. } - | GossipPartialDataColumnError::InconsistentMaxCount { .. } => { + | GossipPartialDataColumnError::InconsistentCommitmentsLength { .. } => { debug!( error = ?err, %block_root, From 40125cdf58826d002670b2edfc3202e9110b90e7 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 13 Apr 2026 14:47:27 +0200 Subject: [PATCH 118/146] Introduce error for missing column assembler --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 +++--- .../src/data_column_verification.rs | 28 +++++++++++++------ .../gossip_methods.rs | 13 +++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 4adc3ee1bc1..8910738e7c8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2271,11 +2271,10 @@ impl BeaconChain { let _timer = metrics::start_timer( &metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_GOSSIP_VERIFICATION_TIMES, ); - if let Some(cached_header) = self - .data_availability_checker - .partial_assembler() - .and_then(|a| a.get_header(&block_root)) - { + let Some(assembler) = self.data_availability_checker.partial_assembler() else { + return Err(GossipPartialDataColumnError::PartialColumnsDisabled); + }; + if let Some(cached_header) = assembler.get_header(&block_root) { return if *cached_header == data_column_header { metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_DUPES); Ok(GossipVerifiedPartialDataColumnHeader::new_from_cached( diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 9f01995d13c..774a8dfa1d4 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -206,6 +206,12 @@ impl From for GossipDataColumnError { #[derive(Debug)] pub enum GossipPartialDataColumnError { GossipDataColumnError(GossipDataColumnError), + /// Partial messages are disabled and we can not validate them. + /// + /// ## Peer scoring + /// A peer sent us a partial message even though we did not advertize support for it, penalize + /// it + PartialColumnsDisabled, /// The partial data column does not contain a header, and we do not have it cached. /// /// ## Peer scoring @@ -226,7 +232,7 @@ pub enum GossipPartialDataColumnError { group_id: Hash256, header_hash: Hash256, }, - /// The partial message has neither a header nor cells + /// The partial message has neither a header nor cells. /// /// ## Peer scoring /// The column sidecar is invalid and the peer is faulty @@ -524,10 +530,10 @@ impl GossipVerifiedPartialDataColumnHeader { let header = Arc::new(header); // Cache the valid header - let newly_cached = chain - .data_availability_checker - .partial_assembler() - .is_some_and(|a| a.init(group_id, header.clone())); + let Some(assembler) = chain.data_availability_checker.partial_assembler() else { + return Err(GossipPartialDataColumnError::PartialColumnsDisabled); + }; + let newly_cached = assembler.init(group_id, header.clone()); chain .observed_slashable @@ -910,11 +916,15 @@ pub fn validate_partial_data_column_sidecar_for_gossip( } } } else { + let Some(assembler) = chain.data_availability_checker.partial_assembler() else { + return PartialColumnVerificationResult::Err( + GossipPartialDataColumnError::PartialColumnsDisabled, + ); + }; + // There is no header, so we check if we have a cached one to use - let Some(header) = chain - .data_availability_checker - .partial_assembler() - .and_then(|a| a.get_header(&column.block_root)) + let Some(header) = assembler + .get_header(&column.block_root) .map(GossipVerifiedPartialDataColumnHeader::new_from_cached) else { return PartialColumnVerificationResult::Err( diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index cda6a44b85c..a1c7ef76c97 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1032,6 +1032,19 @@ impl NetworkBeaconProcessor { "gossip_partial_data_column_low", ); } + GossipPartialDataColumnError::PartialColumnsDisabled => { + error!( + error = ?err, + %block_root, + %index, + "Received partial column while disabled" + ); + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_partial_data_column_low", + ); + } } } From a0ffc72e20c2d1d2f91664864f36b80fd04d9be9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 13 Apr 2026 15:14:27 +0200 Subject: [PATCH 119/146] get rid of expects --- .../src/data_column_verification.rs | 7 +- .../src/partial_data_column_assembler.rs | 9 +- .../lighthouse_network/src/types/partial.rs | 6 +- .../gossip_methods.rs | 8 ++ consensus/types/src/data/mod.rs | 2 +- .../src/data/partial_data_column_sidecar.rs | 124 +++++++++++------- 6 files changed, 101 insertions(+), 55 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 774a8dfa1d4..b6278b2dc94 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -24,7 +24,7 @@ use tree_hash::TreeHash; use types::{ BeaconStateError, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, - SignedBeaconBlockHeader, Slot, StatePayloadStatus, + PartialDataColumnSidecarError, SignedBeaconBlockHeader, Slot, StatePayloadStatus, }; /// An error occurred while validating a gossip data column. @@ -212,6 +212,8 @@ pub enum GossipPartialDataColumnError { /// A peer sent us a partial message even though we did not advertize support for it, penalize /// it PartialColumnsDisabled, + /// There was an unexpected error while performing an operation on the partial data column. + InternalError(PartialDataColumnSidecarError), /// The partial data column does not contain a header, and we do not have it cached. /// /// ## Peer scoring @@ -713,7 +715,7 @@ impl KzgVerifiedCustodyPartialDataColumn { self.data.index } - pub fn merge(&self, other: &Self) -> Option { + pub fn merge(&self, other: &Self) -> Result { self.data .sidecar .merge(&other.data.sidecar) @@ -788,6 +790,7 @@ pub fn verify_kzg_for_partial_data_column( let Some(filtered_column) = data_column .sidecar .filter(|idx| unverified_cells.contains(&idx)) + .map_err(GossipPartialDataColumnError::InternalError)? else { return Err(GossipDataColumnError::PriorKnownUnpublished.into()); }; diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index f37ce983fd2..9c1f079d89b 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -6,6 +6,7 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; +use tracing::error; use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; use types::{DataColumnSidecarFulu, Epoch, EthSpec, Hash256}; @@ -100,8 +101,12 @@ impl PartialDataColumnAssembler { let old_len = column.sidecar.column.len(); // Merge with existing partial - let Some(merged) = existing.merge(&partial) else { - continue; + let merged = match existing.merge(&partial) { + Ok(merged) => merged, + Err(err) => { + error!("Unexpected error merging partial data column: {:?}", err); + continue; + } }; let adding_cells = merged diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index ea0a40a451e..3dedc550e11 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -6,7 +6,7 @@ use ssz::{Decode, Encode}; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; -use tracing::debug; +use tracing::{debug, error}; use types::{ CellBitmap, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, PartialDataColumnSidecar, PartialDataColumnSidecarRef, @@ -210,6 +210,10 @@ impl Partial for OutgoingPartialColumn { .partial_column .sidecar .filter(|idx| want.get(idx).expect("Bound checked above")) + .map_err(|err| { + error!(?err, "Unexpected error filtering sidecar"); + PartialError::InvalidFormat + })? .map(|sidecar| { debug!( peer=%peer_id, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index a1c7ef76c97..69a94ca0b74 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1045,6 +1045,14 @@ impl NetworkBeaconProcessor { "gossip_partial_data_column_low", ); } + GossipPartialDataColumnError::InternalError(_) => { + error!( + error = ?err, + %block_root, + %index, + "Internal error while processing partial column" + ); + } } } diff --git a/consensus/types/src/data/mod.rs b/consensus/types/src/data/mod.rs index 40f25d896d8..9c7eb426260 100644 --- a/consensus/types/src/data/mod.rs +++ b/consensus/types/src/data/mod.rs @@ -20,7 +20,7 @@ pub use data_column_sidecar::{ pub use data_column_subnet_id::{DataColumnSubnetId, all_data_column_sidecar_subnets_from_spec}; pub use partial_data_column_sidecar::{ CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, - PartialDataColumnSidecar, PartialDataColumnSidecarRef, + PartialDataColumnSidecar, PartialDataColumnSidecarError, PartialDataColumnSidecarRef, }; use crate::core::EthSpec; diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 060cd223d71..4709054c202 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -43,60 +43,78 @@ pub struct PartialDataColumnSidecarRef<'a, E: EthSpec> { pub header: ListEncodedOption<&'a PartialDataColumnHeader>, } +#[derive(Debug, Clone, Copy)] +pub enum PartialDataColumnSidecarError { + UnexpectedBounds, + InternallyInconsistent, + Unmergable, +} + impl PartialDataColumnSidecar { pub fn is_complete(&self) -> bool { - self.cells_present_bitmap.iter().all(|bit| bit) + self.cells_present_bitmap.num_set_bits() == self.cells_present_bitmap.len() } /// Creates a reference to this sidecar containing only the blob indices for which the passed /// closure returns `true` and is present in `self`. Will return `None` if there is no overlap. - pub fn filter(&self, filter: F) -> Option> + pub fn filter( + &self, + filter: F, + ) -> Result>, PartialDataColumnSidecarError> where F: Fn(usize) -> bool, { + let len = self.len()?; + let mut new_bitmap = self.cells_present_bitmap.clone(); - let mut new_column = Vec::new(); - let mut new_proofs = Vec::new(); - let mut column_idx = 0; + let mut new_column = Vec::with_capacity(len); + let mut new_proofs = Vec::with_capacity(len); + let mut iter = self.column.iter().zip(self.kzg_proofs.iter()); for (blob_idx, present) in self.cells_present_bitmap.iter().enumerate() { if present { + let (cell, proof) = iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; if filter(blob_idx) { // Keep this cell - let cell = self.column.get(column_idx)?; new_column.push(cell); - let proof = self.kzg_proofs.get(column_idx)?; new_proofs.push(proof); } else { // Mark as not present new_bitmap .set(blob_idx, false) - .expect("Within bounds due to clone above"); + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?; } - column_idx = column_idx - .checked_add(1) - .expect("Will not have more cells than 2^64 - 1"); } } if new_column.is_empty() { - return None; + return Ok(None); } - Some(PartialDataColumnSidecarRef { + Ok(Some(PartialDataColumnSidecarRef { cells_present_bitmap: new_bitmap, column: new_column, kzg_proofs: new_proofs, header: self.header.as_ref().into(), - }) + })) } - pub fn merge(&self, other: &Self) -> Option { + pub fn merge(&self, other: &Self) -> Result { + // Check that each sidecar is internally consistent by checking the lengths. + self.len()?; + other.len()?; + if self.cells_present_bitmap.len() != other.cells_present_bitmap.len() { + return Err(PartialDataColumnSidecarError::Unmergable); + } + let new_bitmap = self.cells_present_bitmap.union(&other.cells_present_bitmap); - let mut new_column = VariableList::default(); - let mut new_proofs = VariableList::default(); - let mut self_cell_idx = 0usize; - let mut other_cell_idx = 0usize; + let len = new_bitmap.num_set_bits(); + let mut new_column = Vec::with_capacity(len); + let mut new_proofs = Vec::with_capacity(len); + let mut self_iter = self.column.iter().zip(self.kzg_proofs.iter()); + let mut other_iter = other.column.iter().zip(other.kzg_proofs.iter()); for presence_bits in self .cells_present_bitmap @@ -106,46 +124,50 @@ impl PartialDataColumnSidecar { match presence_bits { (false, false) => {} (true, other) => { - new_column - .push(self.column.get(self_cell_idx)?.clone()) - .expect("Has same capacity"); - new_proofs - .push(*self.kzg_proofs.get(self_cell_idx)?) - .expect("Has same capacity"); - self_cell_idx = self_cell_idx - .checked_add(1) - .expect("Will not have more cells than 2^64 - 1"); + let (cell, proof) = self_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; + new_column.push(cell.clone()); + new_proofs.push(*proof); if other { - other_cell_idx = other_cell_idx - .checked_add(1) - .expect("Will not have more cells than 2^64 - 1"); + other_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; } } (false, true) => { - new_column - .push(other.column.get(other_cell_idx)?.clone()) - .expect("Has same capacity"); - new_proofs - .push(*other.kzg_proofs.get(other_cell_idx)?) - .expect("Has same capacity"); - other_cell_idx = other_cell_idx - .checked_add(1) - .expect("Will not have more cells than 2^64 - 1"); + let (cell, proof) = other_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; + new_column.push(cell.clone()); + new_proofs.push(*proof); } } } - Some(Self { + Ok(Self { cells_present_bitmap: new_bitmap, - column: new_column, - kzg_proofs: new_proofs, - header: if !self.header.is_some() { + column: new_column + .try_into() + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, + kzg_proofs: new_proofs + .try_into() + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, + header: if self.header.is_some() { self.header.clone() } else { other.header.clone() }, }) } + + fn len(&self) -> Result { + let len = self.cells_present_bitmap.num_set_bits(); + if len != self.kzg_proofs.len() || len != self.column.len() { + return Err(PartialDataColumnSidecarError::InternallyInconsistent); + } + Ok(len) + } } #[cfg_attr( @@ -191,8 +213,7 @@ impl> TryFrom<&SignedBeaconBlock> kzg_commitments_inclusion_proof: block .message() .body() - .kzg_commitments_merkle_proof()? - .clone(), + .kzg_commitments_merkle_proof()?, }) } } @@ -339,7 +360,7 @@ mod tests { #[test] fn filter_keeps_matching_cells() { let sidecar = make_sidecar(6, &[0, 2, 4]); - let filtered = sidecar.filter(|idx| idx == 0 || idx == 4).unwrap(); + let filtered = sidecar.filter(|idx| idx == 0 || idx == 4).unwrap().unwrap(); assert_eq!(filtered.column.len(), 2); assert_eq!(filtered.kzg_proofs.len(), 2); assert!(filtered.cells_present_bitmap.get(0).unwrap()); @@ -350,13 +371,18 @@ mod tests { #[test] fn filter_returns_none_when_no_overlap() { let sidecar = make_sidecar(6, &[0, 2, 4]); - assert!(sidecar.filter(|idx| idx == 1 || idx == 3).is_none()); + assert!( + sidecar + .filter(|idx| idx == 1 || idx == 3) + .unwrap() + .is_none() + ); } #[test] fn filter_preserves_all_when_all_match() { let sidecar = make_sidecar(6, &[0, 2, 4]); - let filtered = sidecar.filter(|_| true).unwrap(); + let filtered = sidecar.filter(|_| true).unwrap().unwrap(); assert_eq!(filtered.column.len(), 3); assert_eq!(filtered.kzg_proofs.len(), 3); assert_eq!(filtered.cells_present_bitmap, sidecar.cells_present_bitmap); From 8a88f88e3469622b7edba2f634287d6a8a26b3fd Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 15:49:38 +0200 Subject: [PATCH 120/146] Metadata requests field matches spec name --- .../lighthouse_network/src/types/partial.rs | 46 +++++++++---------- .../src/data/partial_data_column_sidecar.rs | 4 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 3dedc550e11..13b327ecaf9 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -31,15 +31,15 @@ impl OutgoingPartialColumn { header_sent_set: HeaderSentSet, ) -> Self { // For now, always request all cells - let mut request = partial_column.sidecar.cells_present_bitmap.clone(); - for idx in 0..request.len() { - request + let mut requests = partial_column.sidecar.cells_present_bitmap.clone(); + for idx in 0..requests.len() { + requests .set(idx, true) .expect("Bound asserted via `len` above"); } let metadata = PartialDataColumnPartsMetadata:: { available: partial_column.sidecar.cells_present_bitmap.clone(), - request: partial_column.sidecar.cells_present_bitmap.clone(), + requests: partial_column.sidecar.cells_present_bitmap.clone(), } .into(); @@ -87,9 +87,9 @@ impl MaybeKnownMetadata { if ![ received.available.len(), - received.request.len(), + received.requests.len(), metadata.available.len(), - metadata.request.len(), + metadata.requests.len(), ] .into_iter() .all_equal() @@ -97,12 +97,12 @@ impl MaybeKnownMetadata { return Err(PartialError::OutOfRange); } let new_available = metadata.available.union(&received.available); - let new_request = metadata.request.union(&received.request); - if metadata.available == new_available && metadata.request == new_request { + let new_request = metadata.requests.union(&received.requests); + if metadata.available == new_available && metadata.requests == new_request { return Ok(false); } metadata.available = new_available; - metadata.request = new_request; + metadata.requests = new_request; *encoded = metadata.as_ssz_bytes(); Ok(true) } @@ -133,7 +133,7 @@ impl Metadata for MaybeKnownMetadata { self.do_update(PartialDataColumnPartsMetadata { available: sidecar.cells_present_bitmap.clone(), - request: sidecar.cells_present_bitmap, + requests: sidecar.cells_present_bitmap, }) .map(|_| ()) } @@ -196,7 +196,7 @@ impl Partial for OutgoingPartialColumn { .map_err(|_| PartialError::InvalidFormat)?; let expected_len = self.partial_column.sidecar.cells_present_bitmap.len(); if peer_metadata.available.len() != expected_len - || peer_metadata.request.len() != expected_len + || peer_metadata.requests.len() != expected_len { return Err(PartialError::InvalidFormat); } @@ -204,7 +204,7 @@ impl Partial for OutgoingPartialColumn { let need = !peer_metadata .available .is_subset(&self.partial_column.sidecar.cells_present_bitmap); - let want = peer_metadata.request.difference(&peer_metadata.available); + let want = peer_metadata.requests.difference(&peer_metadata.available); let send = self .partial_column @@ -230,8 +230,8 @@ impl Partial for OutgoingPartialColumn { available: peer_metadata .available .union(&sidecar.cells_present_bitmap), - request: peer_metadata - .request + requests: peer_metadata + .requests .union(&sidecar.cells_present_bitmap), }, )) as Box, @@ -341,7 +341,7 @@ mod tests { bitmap.set(0, true).unwrap(); let received = PartialDataColumnPartsMetadata { available: bitmap.clone(), - request: bitmap, + requests: bitmap, }; let changed = meta.do_update(received).unwrap(); assert!(changed); @@ -354,7 +354,7 @@ mod tests { bitmap1.set(0, true).unwrap(); let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { available: bitmap1.clone(), - request: bitmap1, + requests: bitmap1, } .into(); @@ -363,7 +363,7 @@ mod tests { let changed = meta .do_update(PartialDataColumnPartsMetadata { available: bitmap2.clone(), - request: bitmap2, + requests: bitmap2, }) .unwrap(); assert!(changed); @@ -384,7 +384,7 @@ mod tests { bitmap.set(1, true).unwrap(); let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { available: bitmap.clone(), - request: bitmap.clone(), + requests: bitmap.clone(), } .into(); @@ -394,7 +394,7 @@ mod tests { let changed = meta .do_update(PartialDataColumnPartsMetadata { available: subset.clone(), - request: subset, + requests: subset, }) .unwrap(); assert!(!changed); @@ -406,7 +406,7 @@ mod tests { bitmap4.set(0, true).unwrap(); let mut meta: MaybeKnownMetadata = PartialDataColumnPartsMetadata { available: bitmap4.clone(), - request: bitmap4, + requests: bitmap4, } .into(); @@ -414,7 +414,7 @@ mod tests { bitmap6.set(0, true).unwrap(); let result = meta.do_update(PartialDataColumnPartsMetadata { available: bitmap6.clone(), - request: bitmap6, + requests: bitmap6, }); assert!(result.is_err()); } @@ -461,7 +461,7 @@ mod tests { } let peer_meta = PartialDataColumnPartsMetadata:: { available: peer_available, - request: peer_request, + requests: peer_request, }; let encoded = peer_meta.as_ssz_bytes(); @@ -490,7 +490,7 @@ mod tests { peer_available.set(2, true).unwrap(); let peer_meta = PartialDataColumnPartsMetadata:: { available: peer_available.clone(), - request: peer_available, + requests: peer_available, }; let encoded = peer_meta.as_ssz_bytes(); diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 4709054c202..b1af1d7cb6b 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -221,7 +221,7 @@ impl> TryFrom<&SignedBeaconBlock> #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub struct PartialDataColumnPartsMetadata { pub available: CellBitmap, - pub request: CellBitmap, + pub requests: CellBitmap, } impl Display for PartialDataColumnPartsMetadata { @@ -229,7 +229,7 @@ impl Display for PartialDataColumnPartsMetadata { write!( f, "(available: {}, requested: {})", - self.available, self.request + self.available, self.requests ) } } From d55cd2d79a75592e46c3fcc14c2b03b017d1b51b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 15:53:21 +0200 Subject: [PATCH 121/146] fix horrendous bug --- beacon_node/lighthouse_network/src/types/partial.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 13b327ecaf9..42f449a26a4 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -39,7 +39,7 @@ impl OutgoingPartialColumn { } let metadata = PartialDataColumnPartsMetadata:: { available: partial_column.sidecar.cells_present_bitmap.clone(), - requests: partial_column.sidecar.cells_present_bitmap.clone(), + requests, } .into(); From 5eb7a7ae46fe3d95899e4c89093bb01331d55cb1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 15:55:12 +0200 Subject: [PATCH 122/146] add cli test --- lighthouse/tests/beacon_node.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index ded1f2b7651..0c5d9a59334 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2864,3 +2864,21 @@ fn invalid_block_roots_default_mainnet() { assert!(config.chain.invalid_block_roots.is_empty()); }) } + +#[test] +fn partial_columns() { + CommandLineTest::new() + .flag("enable-partial-columns", None) + .run_with_zero_port() + .with_config(|config| { + assert!(config.network.enable_partial_columns); + assert!(config.chain.enable_partial_columns); + }); + // And disabled by default: + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert!(!config.network.enable_partial_columns); + assert!(!config.chain.enable_partial_columns); + }) +} From 98fffcc1b1e835c866f271ad157d9c3b709cf66b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 17:15:50 +0200 Subject: [PATCH 123/146] reuse block root --- beacon_node/beacon_chain/src/data_availability_checker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index bd9abee715a..5d36c0e8a19 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -435,7 +435,7 @@ impl DataAvailabilityChecker { if let Some(assembler) = &self.partial_assembler && let Ok(header) = block.try_into() { - assembler.init(block.canonical_root(), Arc::new(header)); + assembler.init(executed_block.import_data.block_root, Arc::new(header)); } self.availability_cache.put_executed_block(executed_block) } From 90ec7b3b2eb18cfbdf43559a7283638e6c4359e1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 17:17:34 +0200 Subject: [PATCH 124/146] check for 1 to 1 mapping of subnet and column --- beacon_node/src/config.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b58ddb85ba8..8ba2c0f3214 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -111,8 +111,18 @@ pub fn get_config( set_network_config(&mut client_config.network, cli_args, &data_dir_ref)?; if parse_flag(cli_args, "enable-partial-columns") { - client_config.network.enable_partial_columns = true; - client_config.chain.enable_partial_columns = true; + // Partial messages assume that each subnet maps to exactly one column. + // Check this here to avoid weird issues on networks where this is not the case. + if spec.data_column_sidecar_subnet_count == E::number_of_columns() as u64 { + client_config.network.enable_partial_columns = true; + client_config.chain.enable_partial_columns = true; + } else { + warn!( + subnets = spec.data_column_sidecar_subnet_count, + columns = E::number_of_columns(), + "Not enabling partial columns on networks with multiple columns per subnet" + ) + } } // Parse custody mode from CLI flags From 9adf8d18bef39484025a4c87bec7fd45ddc5eef5 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 15 Apr 2026 17:36:49 +0200 Subject: [PATCH 125/146] use self.sampling_columns_for_epoch --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8910738e7c8..bf422deb9b3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3206,10 +3206,8 @@ impl BeaconChain { ); // Check if we have custody of this column - let sampling_columns = self - .data_availability_checker - .custody_context() - .sampling_columns_for_epoch(slot.epoch(T::EthSpec::slots_per_epoch()), &self.spec); + let sampling_columns = + self.sampling_columns_for_epoch(slot.epoch(T::EthSpec::slots_per_epoch())); let verified_partial = if sampling_columns.contains(&partial.index) { KzgVerifiedCustodyPartialDataColumn::from_asserted_custody(verified_partial) } else { From fea44da877fb3d6ef9860e8e7739783b482e5df3 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:07:47 +0200 Subject: [PATCH 126/146] struct variant for UnknownParentPartialDataColumn --- .../src/network_beacon_processor/gossip_methods.rs | 4 ++-- beacon_node/network/src/sync/manager.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index c474cdc8f3a..e05f1bfe76d 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -922,12 +922,12 @@ impl NetworkBeaconProcessor { %parent_root, "Unknown parent hash for partial column" ); - self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn( + self.send_sync_message(SyncMessage::UnknownParentPartialDataColumn { peer_id, block_root, parent_root, slot, - )); + }); } GossipDataColumnError::PubkeyCacheTimeout | GossipDataColumnError::BeaconChainError(_) => { diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 40b548c99dd..734295ac1d3 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -142,7 +142,12 @@ pub enum SyncMessage { UnknownParentDataColumn(PeerId, Arc>), /// A partial data column with an unknown parent has been received. - UnknownParentPartialDataColumn(PeerId, Hash256, Hash256, Slot), + UnknownParentPartialDataColumn { + peer_id: PeerId, + block_root: Hash256, + parent_root: Hash256, + slot: Slot, + }, /// A peer has sent an attestation that references a block that is unknown. This triggers the /// manager to attempt to find the block matching the unknown hash. @@ -906,7 +911,12 @@ impl SyncManager { } } } - SyncMessage::UnknownParentPartialDataColumn(peer_id, block_root, parent_root, slot) => { + SyncMessage::UnknownParentPartialDataColumn { + peer_id, + block_root, + parent_root, + slot, + } => { debug!(%block_root, %parent_root, "Received unknown parent partial column message"); self.handle_unknown_parent( peer_id, From f00e38f22b6c601d1e66a5eab63dc225c6a4f922 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:20:01 +0200 Subject: [PATCH 127/146] rename len into verify_len --- consensus/types/src/data/partial_data_column_sidecar.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index b1af1d7cb6b..d27e9099243 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -64,7 +64,7 @@ impl PartialDataColumnSidecar { where F: Fn(usize) -> bool, { - let len = self.len()?; + let len = self.verify_len()?; let mut new_bitmap = self.cells_present_bitmap.clone(); let mut new_column = Vec::with_capacity(len); @@ -103,8 +103,8 @@ impl PartialDataColumnSidecar { pub fn merge(&self, other: &Self) -> Result { // Check that each sidecar is internally consistent by checking the lengths. - self.len()?; - other.len()?; + self.verify_len()?; + other.verify_len()?; if self.cells_present_bitmap.len() != other.cells_present_bitmap.len() { return Err(PartialDataColumnSidecarError::Unmergable); } @@ -161,7 +161,7 @@ impl PartialDataColumnSidecar { }) } - fn len(&self) -> Result { + fn verify_len(&self) -> Result { let len = self.cells_present_bitmap.num_set_bits(); if len != self.kzg_proofs.len() || len != self.column.len() { return Err(PartialDataColumnSidecarError::InternallyInconsistent); From 8812fd3eab1b7b28ecf337e17e15d0d58b1c7224 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:21:23 +0200 Subject: [PATCH 128/146] fix metric definitions --- beacon_node/beacon_chain/src/metrics.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 4a0fb779c65..80cfaeea620 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1693,7 +1693,7 @@ pub static PARTIAL_DATA_COLUMN_SIDECAR_HEADER_PROCESSING_SUCCESSES: LazyLock = LazyLock::new(|| { try_create_histogram( "beacon_partial_data_column_sidecar_header_gossip_verification_seconds", - "Full runtime of partial data column sidecars gossip verification", + "Full runtime of partial data column sidecar headers gossip verification", ) }); pub static PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS: LazyLock> = From 5fc3abeb8dd856cf846a0b2263441cb1db44b086 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:23:27 +0200 Subject: [PATCH 129/146] penalize on undecodable topic --- beacon_node/lighthouse_network/src/service/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 86602595afc..c8b737eb1ee 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1408,6 +1408,9 @@ impl Network { error, "Could not decode gossipsub partial message topic" ); + // punish the peer + self.gossipsub_mut() + .report_invalid_partial(peer_id, &topic_hash); }) .ok()?; From 02ffa62ec110fa44bdaf1ebd7c79cd48e5816d9d Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:28:38 +0200 Subject: [PATCH 130/146] add GOSSIP_UNACCEPTED_MESSAGES_PER_CLIENT --- beacon_node/lighthouse_network/src/service/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index c8b737eb1ee..4c6354d1343 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1018,6 +1018,19 @@ impl Network { propagation_source: PeerId, topic: GossipTopic, ) { + if let Some(client) = self + .network_globals + .peers + .read() + .peer_info(&propagation_source) + .map(|info| info.client().kind.as_ref()) + { + metrics::inc_counter_vec( + &metrics::GOSSIP_UNACCEPTED_MESSAGES_PER_CLIENT, + &[client, "reject"], + ) + } + self.gossipsub_mut() .report_invalid_partial(propagation_source, &TopicHash::from(Topic::from(topic))); } From 5312b9d83e599ea380468561d1ee1b981c410ab9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:38:38 +0200 Subject: [PATCH 131/146] nit typo --- beacon_node/lighthouse_network/src/types/pubsub.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 66b28be7759..9875d4b0c43 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -465,7 +465,7 @@ impl PubsubMessage { } /// Decodes incoming partial data column sidecar from gossipsub partial protocol. -/// Note: Currenly, data columns are the only supported partial messages. In future this could +/// Note: Currently, data columns are the only supported partial messages. In future this could /// return an enum. pub fn decode_partial( topic: &GossipTopic, From f34d6abbe7ae317dfd4a53e5b4822dc504aefd81 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:38:38 +0200 Subject: [PATCH 132/146] fix comment on propagate_partial_validation_failure --- .../network/src/network_beacon_processor/gossip_methods.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index e05f1bfe76d..6dd83606988 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -203,8 +203,8 @@ impl NetworkBeaconProcessor { }) } - /// Send a message on `message_tx` that `peer_id` should be propagated on - /// the gossip network. + /// Send a message on `message_tx` that `peer_id` has sent an invalid partial message and should + /// be penalized. pub(crate) fn propagate_partial_validation_failure( &self, propagation_source: PeerId, From 618986e668fea511d029e2a6c9e419e2324624f1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:38:38 +0200 Subject: [PATCH 133/146] fix comment on init --- beacon_node/beacon_chain/src/partial_data_column_assembler.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 9c1f079d89b..47c79bfd968 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -49,7 +49,8 @@ impl PartialDataColumnAssembler { } } - /// Returns true unless the header was already contained or the passed argument failed to convert to a header + /// Insert a `header` for the given `block_root` into the assembler. + /// Returns true unless there already is a header for the block root. pub fn init(&self, block_root: Hash256, header: Arc>) -> bool { let mut assemblies = self.assemblies.write(); From f698fc97f697c3efa2cb3d0938875facfdab0532 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 13:38:38 +0200 Subject: [PATCH 134/146] FAILED_PARTIAL_PUBLISHES_PER_MAIN_TOPIC --- beacon_node/lighthouse_network/src/metrics.rs | 8 ++++++++ beacon_node/lighthouse_network/src/service/mod.rs | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index 623d43a7270..d5d1ed50534 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -83,6 +83,14 @@ pub static FAILED_PUBLISHES_PER_MAIN_TOPIC: LazyLock> = Lazy &["topic_hash"], ) }); +pub static FAILED_PARTIAL_PUBLISHES_PER_MAIN_TOPIC: LazyLock> = + LazyLock::new(|| { + try_create_int_gauge_vec( + "gossipsub_failed_partial_publishes_per_main_topic", + "Failed gossip partial message publishes", + &["topic_hash"], + ) + }); pub static TOTAL_RPC_ERRORS_PER_CLIENT: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "libp2p_rpc_errors_per_client", diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 4c6354d1343..f0c1567cb04 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -971,7 +971,7 @@ impl Network { // add to metrics if let Some(v) = metrics::get_int_gauge( - &metrics::FAILED_PUBLISHES_PER_MAIN_TOPIC, + &metrics::FAILED_PARTIAL_PUBLISHES_PER_MAIN_TOPIC, &[&format!("{:?}", topic.kind())], ) { v.inc() From 99c87fefe2cb890c0198e68238572fc1fe7b2a15 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 17 Apr 2026 14:08:24 +0200 Subject: [PATCH 135/146] move merge to wrapper --- .../src/data_column_verification.rs | 200 ++++++++++++++++-- .../src/data/partial_data_column_sidecar.rs | 113 +--------- 2 files changed, 185 insertions(+), 128 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index b6278b2dc94..22d795e2649 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -24,7 +24,8 @@ use tree_hash::TreeHash; use types::{ BeaconStateError, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, - PartialDataColumnSidecarError, SignedBeaconBlockHeader, Slot, StatePayloadStatus, + PartialDataColumnSidecar, PartialDataColumnSidecarError, SignedBeaconBlockHeader, Slot, + StatePayloadStatus, }; /// An error occurred while validating a gossip data column. @@ -715,18 +716,93 @@ impl KzgVerifiedCustodyPartialDataColumn { self.data.index } + /// Merge two verified partial data columns. + /// + /// Each column must be internally consistent. Additionally, the columns to be merged must have + /// the same block root and index. + /// An error is returned if the columns are internally inconsistent or incompatible for merging. + /// + /// If both columns contain the same cell, the cell from `self` is used - however, as they are + /// KZG verified, they will be the same. pub fn merge(&self, other: &Self) -> Result { - self.data - .sidecar - .merge(&other.data.sidecar) - .map(|sidecar| Self { - data: Arc::new(PartialDataColumn { - block_root: self.data.block_root, - index: self.data.index, - sidecar, - }), - latest_cell_timestamp: self.latest_cell_timestamp.max(other.latest_cell_timestamp), - }) + let self_sidecar = &self.data.sidecar; + let other_sidecar = &self.data.sidecar; + + // Check that each sidecar is internally consistent by checking the lengths. + self_sidecar.verify_len()?; + other_sidecar.verify_len()?; + if self.data.block_root != other.data.block_root + || self.data.index != other.data.index + || self_sidecar.cells_present_bitmap.len() != other_sidecar.cells_present_bitmap.len() + { + return Err(PartialDataColumnSidecarError::Unmergable); + } + + let new_bitmap = self_sidecar + .cells_present_bitmap + .union(&other_sidecar.cells_present_bitmap); + let len = new_bitmap.num_set_bits(); + let mut new_column = Vec::with_capacity(len); + let mut new_proofs = Vec::with_capacity(len); + let mut self_iter = self_sidecar + .column + .iter() + .zip(self_sidecar.kzg_proofs.iter()); + let mut other_iter = other_sidecar + .column + .iter() + .zip(other_sidecar.kzg_proofs.iter()); + + for presence_bits in self_sidecar + .cells_present_bitmap + .iter() + .zip(other_sidecar.cells_present_bitmap.iter()) + { + match presence_bits { + (false, false) => {} + (true, other) => { + let (cell, proof) = self_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; + new_column.push(cell.clone()); + new_proofs.push(*proof); + if other { + other_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; + } + } + (false, true) => { + let (cell, proof) = other_iter + .next() + .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; + new_column.push(cell.clone()); + new_proofs.push(*proof); + } + } + } + + Ok(Self { + data: Arc::new(PartialDataColumn { + block_root: self.data.block_root, + index: self.data.index, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: new_bitmap, + column: new_column + .try_into() + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, + kzg_proofs: new_proofs + .try_into() + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, + header: if self_sidecar.header.is_some() { + self_sidecar.header.clone() + } else { + other_sidecar.header.clone() + }, + }, + }), + latest_cell_timestamp: self.latest_cell_timestamp.max(other.latest_cell_timestamp), + }) } pub fn try_clone_full( @@ -1324,8 +1400,8 @@ mod test { use crate::ChainConfig; use crate::data_column_verification::{ GossipDataColumnError, GossipPartialDataColumnError, GossipVerifiedDataColumn, - GossipVerifiedPartialDataColumnHeader, PartialColumnVerificationResult, - validate_data_column_sidecar_for_gossip_fulu, + GossipVerifiedPartialDataColumnHeader, KzgVerifiedCustodyPartialDataColumn, + PartialColumnVerificationResult, validate_data_column_sidecar_for_gossip_fulu, validate_partial_data_column_sidecar_for_gossip, }; use crate::observed_data_sidecars::Observe; @@ -1335,12 +1411,15 @@ mod test { }; use eth2::types::BlobsBundle; use execution_layer::test_utils::generate_blobs; + use kzg::KzgProof; use ssz::BitList; + use ssz_types::VariableList; use std::sync::Arc; use std::time::UNIX_EPOCH; use types::{ - DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, ForkName, - MainnetEthSpec, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, + Cell, CellBitmap, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, EthSpec, + ForkName, MainnetEthSpec, PartialDataColumn, PartialDataColumnHeader, + PartialDataColumnSidecar, }; type E = MainnetEthSpec; @@ -1732,4 +1811,93 @@ mod test { "Expected InvalidInclusionProof, got: {result:?}" ); } + + // -- merge tests -- + + fn make_cell(marker: u8) -> Cell { + let mut cell = Cell::::default(); + cell[0] = marker; + cell + } + + fn make_partial_with_marker( + total_blobs: usize, + present_indices: &[usize], + marker_base: u8, + ) -> KzgVerifiedCustodyPartialDataColumn { + let mut bitmap = CellBitmap::::with_capacity(total_blobs).unwrap(); + for &idx in present_indices { + bitmap.set(idx, true).unwrap(); + } + + let column: VariableList<_, _> = present_indices + .iter() + .map(|&idx| make_cell(marker_base.wrapping_add(idx as u8))) + .collect::>() + .try_into() + .unwrap(); + let proofs: VariableList<_, _> = present_indices + .iter() + .map(|_| KzgProof::empty()) + .collect::>() + .try_into() + .unwrap(); + + KzgVerifiedCustodyPartialDataColumn { + data: Arc::new(PartialDataColumn { + block_root: Default::default(), + index: 0, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column, + kzg_proofs: proofs, + header: None.into(), + }, + }), + latest_cell_timestamp: Default::default(), + } + } + + fn make_partial( + total_blobs: usize, + present_indices: &[usize], + ) -> KzgVerifiedCustodyPartialDataColumn { + make_partial_with_marker(total_blobs, present_indices, 0) + } + + #[test] + fn merge_disjoint_partials() { + let a = make_partial(6, &[0, 2]); + let b = make_partial(6, &[1, 3]); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.data.sidecar.column.len(), 4); + assert_eq!(merged.data.sidecar.kzg_proofs.len(), 4); + for i in 0..4 { + assert!(merged.data.sidecar.cells_present_bitmap.get(i).unwrap()); + } + assert!(!merged.data.sidecar.cells_present_bitmap.get(4).unwrap()); + } + + #[test] + fn merge_overlapping_partials_prefers_self() { + let a = make_partial_with_marker(4, &[0, 1], 0); + let b = make_partial_with_marker(4, &[1, 2], 100); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.data.sidecar.column.len(), 3); + // Cell at bitmap index 1 is the second cell in the merged column. + // It should come from `a` (marker_base=0, so marker=0+1=1), not `b` (marker=100+1=101). + assert_eq!(merged.data.sidecar.column[1][0], 1); + } + + #[test] + fn merge_with_empty_other() { + let a = make_partial(4, &[0, 2]); + let b = make_partial(4, &[]); + let merged = a.merge(&b).unwrap(); + assert_eq!(merged.data.sidecar.column.len(), 2); + assert_eq!( + merged.data.sidecar.cells_present_bitmap, + a.data.sidecar.cells_present_bitmap + ); + } } diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index d27e9099243..87bd26f34bf 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -101,67 +101,7 @@ impl PartialDataColumnSidecar { })) } - pub fn merge(&self, other: &Self) -> Result { - // Check that each sidecar is internally consistent by checking the lengths. - self.verify_len()?; - other.verify_len()?; - if self.cells_present_bitmap.len() != other.cells_present_bitmap.len() { - return Err(PartialDataColumnSidecarError::Unmergable); - } - - let new_bitmap = self.cells_present_bitmap.union(&other.cells_present_bitmap); - let len = new_bitmap.num_set_bits(); - let mut new_column = Vec::with_capacity(len); - let mut new_proofs = Vec::with_capacity(len); - let mut self_iter = self.column.iter().zip(self.kzg_proofs.iter()); - let mut other_iter = other.column.iter().zip(other.kzg_proofs.iter()); - - for presence_bits in self - .cells_present_bitmap - .iter() - .zip(other.cells_present_bitmap.iter()) - { - match presence_bits { - (false, false) => {} - (true, other) => { - let (cell, proof) = self_iter - .next() - .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; - new_column.push(cell.clone()); - new_proofs.push(*proof); - if other { - other_iter - .next() - .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; - } - } - (false, true) => { - let (cell, proof) = other_iter - .next() - .ok_or(PartialDataColumnSidecarError::UnexpectedBounds)?; - new_column.push(cell.clone()); - new_proofs.push(*proof); - } - } - } - - Ok(Self { - cells_present_bitmap: new_bitmap, - column: new_column - .try_into() - .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, - kzg_proofs: new_proofs - .try_into() - .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?, - header: if self.header.is_some() { - self.header.clone() - } else { - other.header.clone() - }, - }) - } - - fn verify_len(&self) -> Result { + pub fn verify_len(&self) -> Result { let len = self.cells_present_bitmap.num_set_bits(); if len != self.kzg_proofs.len() || len != self.column.len() { return Err(PartialDataColumnSidecarError::InternallyInconsistent); @@ -391,57 +331,6 @@ mod tests { assert_eq!(filtered.as_ssz_bytes(), sidecar.as_ssz_bytes()); } - // -- merge tests -- - - #[test] - fn merge_disjoint_partials() { - let a = make_sidecar(6, &[0, 2]); - let b = make_sidecar(6, &[1, 3]); - let merged = a.merge(&b).unwrap(); - assert_eq!(merged.column.len(), 4); - assert_eq!(merged.kzg_proofs.len(), 4); - for i in 0..4 { - assert!(merged.cells_present_bitmap.get(i).unwrap()); - } - assert!(!merged.cells_present_bitmap.get(4).unwrap()); - } - - #[test] - fn merge_overlapping_partials_prefers_self() { - let a = make_sidecar_with_marker(4, &[0, 1], 0); - let b = make_sidecar_with_marker(4, &[1, 2], 100); - let merged = a.merge(&b).unwrap(); - assert_eq!(merged.column.len(), 3); - // Cell at bitmap index 1 is the second cell in the merged column. - // It should come from `a` (marker_base=0, so marker=0+1=1), not `b` (marker=100+1=101). - assert_eq!(merged.column[1][0], 1); - } - - #[test] - fn merge_with_empty_other() { - let a = make_sidecar(4, &[0, 2]); - let b = make_sidecar(4, &[]); - let merged = a.merge(&b).unwrap(); - assert_eq!(merged.column.len(), 2); - assert_eq!(merged.cells_present_bitmap, a.cells_present_bitmap); - } - - #[test] - fn merge_preserves_header_from_self() { - let mut a = make_sidecar(4, &[0]); - a.header = Some(make_header(4)).into(); - let b = make_sidecar(4, &[1]); - let merged = a.merge(&b).unwrap(); - assert!(merged.header.is_some()); - - // If self has no header, use other's - let c = make_sidecar(4, &[0]); - let mut d = make_sidecar(4, &[1]); - d.header = Some(make_header(4)).into(); - let merged2 = c.merge(&d).unwrap(); - assert!(merged2.header.is_some()); - } - // -- is_complete tests -- #[test] From 9e1e390f3b307958700727e14a3dfd1d80c6a0e8 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 20 Apr 2026 14:56:58 +0200 Subject: [PATCH 136/146] fix copy paste --- beacon_node/beacon_chain/src/data_column_verification.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 22d795e2649..633b9208bc3 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -726,7 +726,7 @@ impl KzgVerifiedCustodyPartialDataColumn { /// KZG verified, they will be the same. pub fn merge(&self, other: &Self) -> Result { let self_sidecar = &self.data.sidecar; - let other_sidecar = &self.data.sidecar; + let other_sidecar = &other.data.sidecar; // Check that each sidecar is internally consistent by checking the lengths. self_sidecar.verify_len()?; From bcb272e89a0eb25b0bce1c4d51f52a35f4dce801 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 20 Apr 2026 15:17:47 +0200 Subject: [PATCH 137/146] organize imports --- .../beacon_chain/src/data_availability_checker.rs | 5 ++--- .../beacon_chain/src/data_column_verification.rs | 10 ++++++---- beacon_node/beacon_chain/src/kzg_utils.rs | 11 +++++------ .../src/partial_data_column_assembler.rs | 14 ++++++++------ .../src/service/partial_column_header_tracker.rs | 2 +- .../lighthouse_network/src/types/partial.rs | 13 +++++++------ consensus/types/src/data/data_column_sidecar.rs | 5 +---- .../types/src/data/partial_data_column_sidecar.rs | 11 +++++++---- 8 files changed, 37 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 5d36c0e8a19..f77d813ffbf 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -18,11 +18,10 @@ use std::sync::Arc; use std::time::Duration; use task_executor::TaskExecutor; use tracing::{debug, error, instrument}; -use types::data::{BlobIdentifier, FixedBlobSidecarList}; +use types::data::{BlobIdentifier, FixedBlobSidecarList, PartialDataColumn}; use types::{ BlobSidecar, BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecar, - DataColumnSidecarList, Epoch, EthSpec, Hash256, PartialDataColumn, SignedBeaconBlock, Slot, - new_non_zero_usize, + DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot, new_non_zero_usize, }; mod error; diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 633b9208bc3..63a91f69ae8 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -21,11 +21,13 @@ use std::sync::Arc; use std::time::Duration; use tracing::{debug, instrument}; use tree_hash::TreeHash; +use types::data::{ + ColumnIndex, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, + PartialDataColumnSidecarError, +}; use types::{ - BeaconStateError, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu, - DataColumnSubnetId, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, - PartialDataColumnSidecar, PartialDataColumnSidecarError, SignedBeaconBlockHeader, Slot, - StatePayloadStatus, + BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, + EthSpec, Hash256, SignedBeaconBlockHeader, Slot, StatePayloadStatus, }; /// An error occurred while validating a gossip data column. diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index f62a9c0e1a6..9641aec47d5 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,15 +7,14 @@ use std::sync::Arc; use tracing::instrument; use tree_hash::TreeHash; use types::data::{ - Cell, CellBitmap, DataColumn, DataColumnSidecarError, PartialDataColumn, - PartialDataColumnHeader, + Cell, CellBitmap, ColumnIndex, DataColumn, DataColumnSidecarError, PartialDataColumn, + PartialDataColumnHeader, PartialDataColumnSidecarRef, }; use types::kzg_ext::KzgCommitments; use types::{ - Blob, BlobSidecar, BlobSidecarList, ChainSpec, ColumnIndex, DataColumnSidecar, - DataColumnSidecarFulu, DataColumnSidecarGloas, DataColumnSidecarList, EthSpec, Hash256, - KzgCommitment, KzgProof, PartialDataColumnSidecarRef, SignedBeaconBlock, - SignedBeaconBlockHeader, SignedBlindedBeaconBlock, Slot, + Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, + DataColumnSidecarGloas, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, + SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlindedBeaconBlock, Slot, }; /// Converts a blob ssz FixedVector to a reference to a fixed-size array diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index 47c79bfd968..ee1c1f53960 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -7,8 +7,11 @@ use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::error; -use types::data::{ColumnIndex, DataColumnSidecar, PartialDataColumn, PartialDataColumnHeader}; -use types::{DataColumnSidecarFulu, Epoch, EthSpec, Hash256}; +use types::core::{Epoch, EthSpec, Hash256}; +use types::data::{ + ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu, PartialDataColumn, + PartialDataColumnHeader, +}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -263,10 +266,9 @@ mod tests { use bls::{FixedBytesExtended, Signature}; use kzg::{KzgCommitment, KzgProof}; use ssz_types::{FixedVector, VariableList}; - use types::{ - BeaconBlockHeader, Cell, CellBitmap, EthSpec, Hash256, MinimalEthSpec, - PartialDataColumnSidecar, SignedBeaconBlockHeader, Slot, - }; + use types::block::{BeaconBlockHeader, SignedBeaconBlockHeader}; + use types::core::{EthSpec, Hash256, MinimalEthSpec, Slot}; + use types::data::{Cell, CellBitmap, PartialDataColumnSidecar}; type E = MinimalEthSpec; diff --git a/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs b/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs index 720bbf7668c..bb588fe3d84 100644 --- a/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs +++ b/beacon_node/lighthouse_network/src/service/partial_column_header_tracker.rs @@ -4,7 +4,7 @@ use parking_lot::Mutex; use std::collections::HashSet; use std::num::NonZeroUsize; use std::sync::Arc; -use types::Hash256; +use types::core::Hash256; const MAX_BLOCKS: NonZeroUsize = NonZeroUsize::new(4).unwrap(); diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 42f449a26a4..f25ce9ec36f 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -7,9 +7,10 @@ use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; use tracing::{debug, error}; -use types::{ - CellBitmap, EthSpec, Hash256, PartialDataColumn, PartialDataColumnHeader, - PartialDataColumnPartsMetadata, PartialDataColumnSidecar, PartialDataColumnSidecarRef, +use types::core::{EthSpec, Hash256}; +use types::data::{ + CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnPartsMetadata, + PartialDataColumnSidecar, PartialDataColumnSidecarRef, }; const PARTIAL_COLUMNS_VERSION_BYTE: u8 = 0; @@ -261,9 +262,9 @@ mod tests { use fixed_bytes::FixedBytesExtended; use libp2p::identity::Keypair; use ssz_types::FixedVector; - use types::{ - BeaconBlockHeader, MinimalEthSpec, PartialDataColumnHeader, SignedBeaconBlockHeader, Slot, - }; + use types::block::{BeaconBlockHeader, SignedBeaconBlockHeader}; + use types::core::{MinimalEthSpec, Slot}; + use types::data::PartialDataColumnHeader; type E = MinimalEthSpec; diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index f692bd9ca41..e6a1310818d 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -1,6 +1,3 @@ -use crate::data::partial_data_column_sidecar::CellBitmap; -use crate::data::partial_data_column_sidecar::PartialDataColumn; -use crate::data::partial_data_column_sidecar::PartialDataColumnSidecar; use std::sync::Arc; use bls::Signature; @@ -22,7 +19,7 @@ use tree_hash_derive::TreeHash; use crate::{ block::{BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlockHeader}, core::{Epoch, EthSpec, Hash256, Slot}, - data::PartialDataColumnHeader, + data::{CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar}, fork::ForkName, kzg_ext::{KzgCommitments, KzgError}, state::BeaconStateError, diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 87bd26f34bf..2a0bd96fce1 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -1,8 +1,11 @@ -use crate::test_utils::TestRandom; use crate::{ - AbstractExecPayload, BLOB_KZG_COMMITMENTS_INDEX, BeaconStateError, Cell, ColumnIndex, - DataColumnSidecar, DataColumnSidecarFulu, EthSpec, Hash256, KzgCommitments, SignedBeaconBlock, - SignedBeaconBlockHeader, Slot, + block::{BLOB_KZG_COMMITMENTS_INDEX, SignedBeaconBlock, SignedBeaconBlockHeader}, + core::{EthSpec, Hash256, Slot}, + data::{Cell, ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu}, + execution::AbstractExecPayload, + kzg_ext::KzgCommitments, + state::BeaconStateError, + test_utils::TestRandom, }; use educe::Educe; use kzg::KzgProof; From f7f861956ee5640073243c49b25c4a2dfd5e5fa0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 13 Apr 2026 11:51:37 +0200 Subject: [PATCH 138/146] better validation flow around determine_missing_cells --- .../src/data_availability_checker.rs | 205 ++++++++---------- .../src/data_column_verification.rs | 169 ++++++++------- .../src/partial_data_column_assembler.rs | 72 +++--- .../gossip_methods.rs | 4 +- .../types/src/data/data_column_sidecar.rs | 48 +++- .../src/data/partial_data_column_sidecar.rs | 9 +- crypto/kzg/src/lib.rs | 6 +- 7 files changed, 282 insertions(+), 231 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index f77d813ffbf..9d8b76aaed3 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -21,7 +21,8 @@ use tracing::{debug, error, instrument}; use types::data::{BlobIdentifier, FixedBlobSidecarList, PartialDataColumn}; use types::{ BlobSidecar, BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecar, - DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot, new_non_zero_usize, + DataColumnSidecarList, Epoch, EthSpec, Hash256, PartialDataColumnSidecarError, + PartialDataColumnSidecarRef, SignedBeaconBlock, Slot, new_non_zero_usize, }; mod error; @@ -186,94 +187,104 @@ impl DataAvailabilityChecker { }) } - /// Check if the full data column is in the availability cache. - /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells - /// (cells in the incoming column that aren't cached yet) otherwise. - pub fn determine_missing_cells_full( - &self, - block_root: &Hash256, - data_column: &DataColumnSidecar, - ) -> Option> { + /// Filter out all cells that are already cached for the given `block_root`. + /// Returns None if all cells are already cached. + /// Returns an error if any cells or proofs mismatch the cached cells. + pub fn missing_cells_for_column_sidecar<'a>( + &'_ self, + data_column: &'a DataColumnSidecar, + ) -> Result>, MissingCellsError> { + let block_root = data_column.block_root(); let column_index = *data_column.index(); - let cell_count = data_column.column().len(); - // Check DA checker cache first - if we have a full column cached, nothing is missing - if self - .availability_cache - .peek_pending_components(block_root, |components| { - components - .and_then(|c| c.get_cached_data_column(column_index)) - .is_some() - }) + // Check DA checker cache first - if we have a full column cached, nothing is missing. + // We return Some(true) from the peek if it exists and matches, Some(false) if it exists but + // does not match, and None if it doesn't exist. + if let Some(matches) = + self.availability_cache + .peek_pending_components(&block_root, |components| { + components + .and_then(|c| c.get_cached_data_column(column_index)) + .map(|cached| *cached == *data_column) + }) { - return Some(vec![]); + return if matches { + Ok(None) + } else { + Err(MissingCellsError::MismatchesCachedColumn) + }; } // Check assembler for partial columns if let Some(assembler) = &self.partial_assembler { - match assembler.get_partial(block_root, column_index) { + match assembler.get_partial(&block_root, column_index) { Some(AssemblyColumn::Incomplete(cached_partial)) => { - // Compare: find which cells from incoming full column aren't in cached partial - compare_full_to_partial(cell_count, cached_partial.as_data_column()) + return data_column.try_filter_to_partial_ref(|idx, cell, proof| { + match cached_partial.as_data_column().sidecar.get(idx) { + None => Ok(true), + Some((cached_cell, cached_proof)) => { + if cell == cached_cell && proof == cached_proof { + Ok(false) + } else { + Err(MissingCellsError::MismatchesCachedColumn) + } + } + } + }); + } + // This can happen if the column has been marked as completed already but has not + // reached the availability cache yet. + Some(AssemblyColumn::Complete(_)) => { + return Ok(None); } - Some(AssemblyColumn::Complete) => Some(vec![]), None => { // No cached data, all cells are "missing" (new data we want) - Some((0..cell_count).collect()) } } - } else { - // No assembler, all cells are missing - Some((0..cell_count).collect()) } + // No cached data, all cells are "missing" (new data we want) + data_column.try_filter_to_partial_ref(|_, _, _| Ok(true)) } - /// Check if the partial data column is in the availability cache. - /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells - /// (cells in the incoming column that aren't cached yet) otherwise. - pub fn determine_missing_cells_partial( - &self, - block_root: &Hash256, - data_column: &PartialDataColumn, - ) -> Option> { - let column_index = data_column.index; + /// Filter out all cells that are already cached for the given `block_root`. + /// Returns input for kzg verification, or None if all cells are already cached. + pub fn missing_cells_for_partial_column_sidecar<'a>( + &'_ self, + partial_data_column: &'a PartialDataColumn, + ) -> Result>, MissingCellsError> { + let column_index = partial_data_column.index; + let block_root = partial_data_column.block_root; - // Check DA checker cache first - if we have a full column cached, nothing is missing + // Check DA checker cache first - if we have a full column cached, nothing is missing. if self .availability_cache - .peek_pending_components(block_root, |components| { - components - .and_then(|c| c.get_cached_data_column(column_index)) - .is_some() + .peek_pending_components(&block_root, |components| { + components.is_some_and(|c| c.get_cached_data_column(column_index).is_some()) }) { - return Some(vec![]); + return Ok(None); } // Check assembler for partial columns if let Some(assembler) = &self.partial_assembler { - match assembler.get_partial(block_root, column_index) { + match assembler.get_partial(&block_root, column_index) { Some(AssemblyColumn::Incomplete(cached_partial)) => { - // Compare: find which cells from incoming full column aren't in cached partial - return compare_partial_to_partial( - data_column, - cached_partial.as_data_column(), - ); + return Ok(partial_data_column.sidecar.filter(|idx| { + cached_partial.as_data_column().sidecar.get(idx).is_none() + })?); + } + // This can happen if the column has been marked as completed already but has not + // reached the availability cache yet. + Some(AssemblyColumn::Complete(_)) => { + return Ok(None); + } + None => { + // No cached data, all cells are "missing" (new data we want) } - Some(AssemblyColumn::Complete) => return Some(vec![]), - None => {} } } - - // No assembler or no data, return all cells as missing - let incoming_cells: Vec = data_column - .sidecar - .cells_present_bitmap - .iter() - .enumerate() - .filter_map(|(idx, present)| present.then_some(idx)) - .collect(); - Some(incoming_cells) + // No cached data, all cells are "missing" (new data we want) + Ok(partial_data_column.sidecar.filter(|_| true)?) } /// Get a blob from the availability cache. @@ -406,6 +417,12 @@ impl DataAvailabilityChecker { .map(|c| KzgVerifiedCustodyDataColumn::from_asserted_custody(c.into_inner())) .collect::>(); + if let Some(assembler) = &self.partial_assembler { + for column in &custody_columns { + assembler.mark_as_complete(block_root, column); + } + } + self.availability_cache .put_kzg_verified_data_columns(block_root, custody_columns) } @@ -998,67 +1015,19 @@ impl MaybeAvailableBlock { } } -/// Compare an incoming full column (all cells present, indices 0..cell_count) against a cached partial. -/// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. -fn compare_full_to_partial( - incoming_cell_count: usize, - cached_partial: &PartialDataColumn, -) -> Option> { - let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; - - // Check that cell counts match - if cached_bitmap.len() != incoming_cell_count { - debug!( - cached_len = cached_bitmap.len(), - incoming_len = incoming_cell_count, - "Cell count mismatch in column comparison" - ); - return None; - } - - // Find cells that are in incoming (all of them for a full column) but not in cached - let missing_cells: Vec = (0..incoming_cell_count) - .filter(|&idx| { - !cached_bitmap - .get(idx) - .expect("idx within bounds due to length check above") - }) - .collect(); - - Some(missing_cells) +pub enum MissingCellsError { + /// The provided column is not matching with the existing cached column. + /// This is to be treated as a KZG verification failure. + MismatchesCachedColumn, + /// An error occurred while operating on the column. It is possibly malformed. + /// This is not expected to happen for columns passing basic validation. + UnexpectedError(PartialDataColumnSidecarError), } -/// Compare an incoming partial column against a cached partial. -/// Returns cells from the incoming column that aren't in the cached partial, or None on data conflict. -fn compare_partial_to_partial( - incoming: &PartialDataColumn, - cached_partial: &PartialDataColumn, -) -> Option> { - let incoming_bitmap = &incoming.sidecar.cells_present_bitmap; - let cached_bitmap = &cached_partial.sidecar.cells_present_bitmap; - - // Check that cell counts match - if incoming_bitmap.len() != cached_bitmap.len() { - debug!( - incoming_len = incoming_bitmap.len(), - cached_len = cached_bitmap.len(), - "Cell count mismatch in column comparison" - ); - return None; +impl From for MissingCellsError { + fn from(e: PartialDataColumnSidecarError) -> Self { + Self::UnexpectedError(e) } - - // Find cells that are in incoming but not in cached - let missing_cells: Vec = incoming_bitmap - .iter() - .zip(cached_bitmap.iter()) - .enumerate() - .filter_map(|(idx, (in_incoming, in_cached))| { - // Cell is "missing" (from cache) if it's in incoming but not in cached - (in_incoming && !in_cached).then_some(idx) - }) - .collect(); - - Some(missing_cells) } #[cfg(test)] diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 63a91f69ae8..29097c59b49 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -1,6 +1,7 @@ use crate::block_verification::{ BlockSlashInfo, get_validator_pubkey_cache, process_block_slash_info, }; +use crate::data_availability_checker::MissingCellsError; use crate::kzg_utils::{ reconstruct_data_columns, validate_full_data_columns, validate_partial_data_columns, }; @@ -27,7 +28,8 @@ use types::data::{ }; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSidecarFulu, DataColumnSubnetId, - EthSpec, Hash256, SignedBeaconBlockHeader, Slot, StatePayloadStatus, + EthSpec, Hash256, PartialDataColumnSidecarRef, SignedBeaconBlockHeader, Slot, + StatePayloadStatus, }; /// An error occurred while validating a gossip data column. @@ -69,6 +71,13 @@ pub enum GossipDataColumnError { /// /// The data column sidecar is invalid and the peer is faulty. InvalidKzgProof(kzg::Error), + /// The column mismatches the cached (possibly partial) column. + /// This is equivalent to failed kzg verification. + /// + /// ## Peer scoring + /// + /// The data column sidecar is invalid and the peer is faulty. + MismatchesCachedColumn, /// The column was gossiped over an incorrect subnet. /// /// ## Peer scoring @@ -332,24 +341,29 @@ impl GossipVerifiedDataColumn // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; - if chain + match chain .data_availability_checker - .determine_missing_cells_full(&column_sidecar.block_root(), column_sidecar.as_ref()) - .map(|cells| cells.is_empty()) - .unwrap_or(false) + .missing_cells_for_column_sidecar(&column_sidecar) { - // Observe this data column so we don't process it again. - if O::observe() { - observe_gossip_data_column(&column_sidecar, chain)?; + Ok(Some(_)) => Ok(Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedDataColumn::from_execution_verified(column_sidecar), + _phantom: Default::default(), + }), + Ok(None) => { + // Observe this data column so we don't process it again. + if O::observe() { + observe_gossip_data_column(&column_sidecar, chain)?; + } + Err(GossipDataColumnError::PriorKnownUnpublished) + } + Err(MissingCellsError::MismatchesCachedColumn) => { + Err(GossipDataColumnError::MismatchesCachedColumn) + } + Err(MissingCellsError::UnexpectedError(_)) => { + todo!("handle unexpected error") } - return Err(GossipDataColumnError::PriorKnownUnpublished); } - - Ok(Self { - block_root: column_sidecar.block_root(), - data_column: KzgVerifiedDataColumn::from_execution_verified(column_sidecar), - _phantom: Default::default(), - }) } /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. @@ -396,14 +410,6 @@ pub struct KzgVerifiedDataColumn { } impl KzgVerifiedDataColumn { - pub fn new( - data_column: Arc>, - kzg: &Kzg, - seen_timestamp: Duration, - ) -> Result, KzgError)> { - verify_kzg_for_data_column(data_column, kzg, seen_timestamp) - } - /// Mark a data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. pub fn from_execution_verified(data_column: Arc>) -> Self { @@ -628,19 +634,6 @@ impl KzgVerifiedCustodyDataColumn { } } - /// Verify a column already marked as custody column - pub fn new( - data_column: CustodyDataColumn, - kzg: &Kzg, - seen_timestamp: Duration, - ) -> Result, KzgError)> { - verify_kzg_for_data_column(data_column.clone_arc(), kzg, seen_timestamp)?; - Ok(Self { - data: data_column.data, - seen_timestamp, - }) - } - pub fn reconstruct_columns( kzg: &Kzg, partial_set_of_columns: &[Self], @@ -733,11 +726,14 @@ impl KzgVerifiedCustodyPartialDataColumn { // Check that each sidecar is internally consistent by checking the lengths. self_sidecar.verify_len()?; other_sidecar.verify_len()?; - if self.data.block_root != other.data.block_root - || self.data.index != other.data.index - || self_sidecar.cells_present_bitmap.len() != other_sidecar.cells_present_bitmap.len() - { - return Err(PartialDataColumnSidecarError::Unmergable); + if self.data.block_root != other.data.block_root || self.data.index != other.data.index { + return Err(PartialDataColumnSidecarError::ConflictingData); + } + if self_sidecar.cells_present_bitmap.len() != other_sidecar.cells_present_bitmap.len() { + return Err(PartialDataColumnSidecarError::DifferingLengths { + lhs_len: self_sidecar.cells_present_bitmap.len(), + rhs_len: other_sidecar.cells_present_bitmap.len(), + }); } let new_bitmap = self_sidecar @@ -843,11 +839,22 @@ impl KzgVerifiedCustodyPartialDataColumn { #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_data_column( data_column: Arc>, + cells_to_verify: PartialDataColumnSidecarRef, kzg: &Kzg, seen_timestamp: Duration, ) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); - validate_full_data_columns(kzg, iter::once(&data_column))?; + let Ok(kzg_commitments) = data_column.kzg_commitments() else { + return Err(( + Some(*data_column.index()), + KzgError::InconsistentArrayLength("todo(gloas)".to_string()), + )); + }; + validate_partial_data_columns( + kzg, + iter::once((*data_column.index(), cells_to_verify)), + kzg_commitments, + )?; Ok(KzgVerifiedDataColumn { data: data_column, seen_timestamp, @@ -859,29 +866,21 @@ pub fn verify_kzg_for_data_column( /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] pub fn verify_kzg_for_partial_data_column( - data_column: Box>, - unverified_cells: Vec, + data_column: Arc>, + cells_to_verify: PartialDataColumnSidecarRef, header: &GossipVerifiedPartialDataColumnHeader, kzg: &Kzg, seen_timestamp: Duration, ) -> Result, GossipPartialDataColumnError> { - let Some(filtered_column) = data_column - .sidecar - .filter(|idx| unverified_cells.contains(&idx)) - .map_err(GossipPartialDataColumnError::InternalError)? - else { - return Err(GossipDataColumnError::PriorKnownUnpublished.into()); - }; - let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); validate_partial_data_columns( kzg, - iter::once((data_column.index, filtered_column)), + iter::once((data_column.index, cells_to_verify)), header.header.kzg_commitments.as_ref(), ) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; Ok(KzgVerifiedPartialDataColumn { - data: data_column.into(), + data: data_column, latest_cell_timestamp: seen_timestamp, }) } @@ -930,17 +929,22 @@ pub fn validate_data_column_sidecar_for_gossip_fulu { + GossipDataColumnError::MismatchesCachedColumn + } + MissingCellsError::UnexpectedError(_) => todo!("handle unexpected error"), + })? + else { // Observe this data column so we don't process it again. if O::observe() { observe_gossip_data_column(&data_column, chain)?; } return Err(GossipDataColumnError::PriorKnownUnpublished); - } + }; verify_column_inclusion_proof(data_column_fulu)?; let parent_block = verify_parent_block_and_finalized_descendant( @@ -952,9 +956,13 @@ pub fn validate_data_column_sidecar_for_gossip_fulu( }; } - let Some(missing_cells) = chain + let column = Arc::from(column); + let cells_to_kzg_verify = match chain .data_availability_checker - .determine_missing_cells_partial(&block_root, &column) - else { - // TODO(dknopik): This shuld be a different error - return PartialColumnVerificationResult::ErrWithValidHeader { - err: GossipDataColumnError::PriorKnownUnpublished.into(), - header, - }; + .missing_cells_for_partial_column_sidecar(&column) + { + Ok(Some(cells_to_kzg_verify)) => cells_to_kzg_verify, + Ok(None) => { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipDataColumnError::PriorKnownUnpublished.into(), + header, + }; + } + Err(MissingCellsError::MismatchesCachedColumn) => { + return PartialColumnVerificationResult::ErrWithValidHeader { + err: GossipDataColumnError::MismatchesCachedColumn.into(), + header, + }; + } + Err(MissingCellsError::UnexpectedError(e)) => todo!("handle unexpected error {:?}", e), }; - if missing_cells.is_empty() { - return PartialColumnVerificationResult::ErrWithValidHeader { - err: GossipDataColumnError::PriorKnownUnpublished.into(), - header, - }; - } - // We do not have to check block related data here, as we create the verifiable column from // gossip accepted block let kzg = &chain.kzg; let column = match verify_kzg_for_partial_data_column( - column, - missing_cells, + column.clone(), + cells_to_kzg_verify, &header, kzg, seen_timestamp, diff --git a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs index ee1c1f53960..0ce754c8a0b 100644 --- a/beacon_node/beacon_chain/src/partial_data_column_assembler.rs +++ b/beacon_node/beacon_chain/src/partial_data_column_assembler.rs @@ -8,10 +8,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use tracing::error; use types::core::{Epoch, EthSpec, Hash256}; -use types::data::{ - ColumnIndex, DataColumnSidecar, DataColumnSidecarFulu, PartialDataColumn, - PartialDataColumnHeader, -}; +use types::data::{ColumnIndex, PartialDataColumnHeader}; /// Assembles partial data columns into complete columns pub struct PartialDataColumnAssembler { @@ -29,7 +26,8 @@ struct PartialAssembly { #[derive(Clone, Debug)] pub enum AssemblyColumn { - Complete, + // As the actual column is Arc'd inside, storing it redundantly here will not increase memory usage. + Complete(KzgVerifiedCustodyDataColumn), Incomplete(KzgVerifiedCustodyPartialDataColumn), } @@ -135,8 +133,8 @@ impl PartialDataColumnAssembler { // Check if merged column is now complete by trying to convert into full let column = if let Some(full_column) = merged.try_clone_full(&header) { - full_columns.push(full_column); - AssemblyColumn::Complete + full_columns.push(full_column.clone()); + AssemblyColumn::Complete(full_column) } else { AssemblyColumn::Incomplete(merged.clone()) }; @@ -156,21 +154,40 @@ impl PartialDataColumnAssembler { /// Mark a column as assembled. Returns true if the column was previously incomplete or not /// in the assembly at all. - pub fn mark_as_complete(&self, block_root: Hash256, column: &DataColumnSidecarFulu) -> bool { + pub fn mark_as_complete( + &self, + block_root: Hash256, + column: &KzgVerifiedCustodyDataColumn, + ) -> bool { + // TODO(gloas): support partial messages + let Ok(fulu) = column.as_data_column().as_fulu() else { + return false; + }; + let mut assemblies = self.assemblies.write(); let assembly = assemblies.get_or_insert_mut(block_root, || PartialAssembly { header: Arc::new(PartialDataColumnHeader { - kzg_commitments: column.kzg_commitments.clone(), - signed_block_header: column.signed_block_header.clone(), - kzg_commitments_inclusion_proof: column.kzg_commitments_inclusion_proof.clone(), + kzg_commitments: fulu.kzg_commitments.clone(), + signed_block_header: fulu.signed_block_header.clone(), + kzg_commitments_inclusion_proof: fulu.kzg_commitments_inclusion_proof.clone(), }), has_local_blobs: false, columns: Default::default(), }); let prev = assembly .columns - .insert(column.index, AssemblyColumn::Complete); - !matches!(prev, Some(AssemblyColumn::Complete)) + .insert(column.index(), AssemblyColumn::Complete(column.clone())); + !matches!(prev, Some(AssemblyColumn::Complete(_))) + } + + /// Returns true if the given column is complete. + pub fn is_complete(&self, block_root: Hash256, column_index: ColumnIndex) -> bool { + self.assemblies.read().peek(&block_root).is_some_and(|a| { + matches!( + a.columns.get(&column_index), + Some(AssemblyColumn::Complete(_)) + ) + }) } /// Get the current partial for a specific column if it exists in assembly @@ -249,26 +266,21 @@ impl PartialDataColumnAssembler { } } -/// Result of initializing with engine blobs -pub struct InitResult { - /// Columns that were already complete - pub complete_columns: Vec>>, - /// Columns that need further assembly via gossip - pub incomplete_partials: Vec>>, -} - #[cfg(test)] mod tests { use super::*; use crate::data_column_verification::{ - KzgVerifiedCustodyPartialDataColumn, KzgVerifiedPartialDataColumn, + KzgVerifiedCustodyPartialDataColumn, KzgVerifiedDataColumn, KzgVerifiedPartialDataColumn, }; use bls::{FixedBytesExtended, Signature}; use kzg::{KzgCommitment, KzgProof}; use ssz_types::{FixedVector, VariableList}; use types::block::{BeaconBlockHeader, SignedBeaconBlockHeader}; use types::core::{EthSpec, Hash256, MinimalEthSpec, Slot}; - use types::data::{Cell, CellBitmap, PartialDataColumnSidecar}; + use types::data::{ + Cell, CellBitmap, DataColumnSidecar, DataColumnSidecarFulu, PartialDataColumn, + PartialDataColumnSidecar, + }; type E = MinimalEthSpec; @@ -351,6 +363,12 @@ mod tests { ) } + fn make_full_column(fulu: DataColumnSidecarFulu) -> KzgVerifiedCustodyDataColumn { + KzgVerifiedCustodyDataColumn::from_asserted_custody( + KzgVerifiedDataColumn::__new_for_testing(Arc::new(DataColumnSidecar::Fulu(fulu))), + ) + } + fn make_assembler() -> PartialDataColumnAssembler { PartialDataColumnAssembler::new(NonZeroUsize::new(16).unwrap()) } @@ -467,7 +485,7 @@ mod tests { let partial = make_partial(root, 0, 4, &[0, 1]); assembler.merge_partials(root, vec![partial], header); - let full_column = DataColumnSidecarFulu:: { + let full_column = make_full_column(DataColumnSidecarFulu:: { index: 0, column: vec![Cell::::default(); 4].try_into().unwrap(), kzg_commitments: vec![KzgCommitment([0u8; 48]); 4].try_into().unwrap(), @@ -486,7 +504,7 @@ mod tests { vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], ) .unwrap(), - }; + }); assert!(assembler.mark_as_complete(root, &full_column)); } @@ -495,7 +513,7 @@ mod tests { let assembler = make_assembler(); let root = Hash256::repeat_byte(1); - let full_column = DataColumnSidecarFulu:: { + let full_column = make_full_column(DataColumnSidecarFulu:: { index: 0, column: vec![Cell::::default(); 4].try_into().unwrap(), kzg_commitments: vec![KzgCommitment([0u8; 48]); 4].try_into().unwrap(), @@ -514,7 +532,7 @@ mod tests { vec![Hash256::zero(); E::kzg_commitments_inclusion_proof_depth()], ) .unwrap(), - }; + }); assert!(assembler.mark_as_complete(root, &full_column)); assert!(!assembler.mark_as_complete(root, &full_column)); } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 6dd83606988..ea1a2286a0f 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -743,6 +743,7 @@ impl NetworkBeaconProcessor { | GossipDataColumnError::InvalidSubnetId { .. } | GossipDataColumnError::InvalidInclusionProof | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::MismatchesCachedColumn | GossipDataColumnError::UnexpectedDataColumn | GossipDataColumnError::InvalidColumnIndex(_) | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } @@ -943,6 +944,7 @@ impl NetworkBeaconProcessor { | GossipDataColumnError::InvalidSubnetId { .. } | GossipDataColumnError::InvalidInclusionProof | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::MismatchesCachedColumn | GossipDataColumnError::UnexpectedDataColumn | GossipDataColumnError::InvalidColumnIndex(_) | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } @@ -1323,7 +1325,7 @@ impl NetworkBeaconProcessor { .chain .data_availability_checker .partial_assembler() - .is_some_and(|a| a.mark_as_complete(block_root, col)) + .is_some_and(|a| !a.is_complete(block_root, verified_data_column.index())) { metrics::inc_counter_vec( &metrics::BEACON_USEFUL_FULL_COLUMNS_RECEIVED_TOTAL, diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index e6a1310818d..109c9472a59 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -19,7 +19,10 @@ use tree_hash_derive::TreeHash; use crate::{ block::{BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, SignedBeaconBlockHeader}, core::{Epoch, EthSpec, Hash256, Slot}, - data::{CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar}, + data::{ + CellBitmap, PartialDataColumn, PartialDataColumnHeader, PartialDataColumnSidecar, + PartialDataColumnSidecarError, PartialDataColumnSidecarRef, + }, fork::ForkName, kzg_ext::{KzgCommitments, KzgError}, state::BeaconStateError, @@ -137,6 +140,49 @@ impl DataColumnSidecar { )), } } + + /// Convert this full data column into a partial data column reference for KZG verification. + /// The header will NOT be set. + /// + /// Uses the supplied filter to determine which cells to include in the partial sidecar. + pub fn try_filter_to_partial_ref( + &self, + filter: F, + ) -> Result>, Err> + where + F: Fn(usize, &Cell, &KzgProof) -> Result, + Err: From, + { + let len = self.column().len(); + let mut new_bitmap = CellBitmap::::with_capacity(len) + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?; + let mut new_column = Vec::with_capacity(len); + let mut new_proofs = Vec::with_capacity(len); + let iter = self.column().iter().zip(self.kzg_proofs().iter()); + + for (blob_idx, (cell, proof)) in iter.enumerate() { + if filter(blob_idx, cell, proof)? { + // Keep this cell + new_column.push(cell); + new_proofs.push(proof); + // Mark as present + new_bitmap + .set(blob_idx, true) + .map_err(|_| PartialDataColumnSidecarError::UnexpectedBounds)?; + } + } + + if new_column.is_empty() { + return Ok(None); + } + + Ok(Some(PartialDataColumnSidecarRef { + cells_present_bitmap: new_bitmap, + column: new_column, + kzg_proofs: new_proofs, + header: None.into(), + })) + } } impl DataColumnSidecarFulu { diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index 2a0bd96fce1..ff6b0695778 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -50,7 +50,8 @@ pub struct PartialDataColumnSidecarRef<'a, E: EthSpec> { pub enum PartialDataColumnSidecarError { UnexpectedBounds, InternallyInconsistent, - Unmergable, + DifferingLengths { lhs_len: usize, rhs_len: usize }, + ConflictingData, } impl PartialDataColumnSidecar { @@ -58,6 +59,12 @@ impl PartialDataColumnSidecar { self.cells_present_bitmap.num_set_bits() == self.cells_present_bitmap.len() } + pub fn get(&self, idx: usize) -> Option<(&Cell, &KzgProof)> { + self.column + .get(idx) + .and_then(|cell| self.kzg_proofs.get(idx).map(|proof| (cell, proof))) + } + /// Creates a reference to this sidecar containing only the blob indices for which the passed /// closure returns `true` and is present in `self`. Will return `None` if there is no overlap. pub fn filter( diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 65f00798c9e..201ed0ee531 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -21,10 +21,10 @@ pub const BYTES_PER_PROOF: usize = 48; use crate::trusted_setup::load_trusted_setup; use rayon::prelude::*; pub use rust_eth_kzg::{ - constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, + constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, }; -use tracing::{instrument, Span}; +use tracing::{Span, instrument}; // Note: Both `NUMBER_OF_COLUMNS` and `CELLS_PER_EXT_BLOB` are preset values - however this // is a constant in the KZG library - be aware that overriding `NUMBER_OF_COLUMNS` will break KZG @@ -50,8 +50,6 @@ pub enum Error { ReconstructFailed(String), /// Kzg was not initialized with PeerDAS enabled. DASContextUninitialized, - /// A partial column was provided without necessary data - PartialWithoutHeader, } impl From for Error { From cb7749d4c07716440ae218481fe301faf7c093b2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 22 Apr 2026 16:38:56 +0200 Subject: [PATCH 139/146] update libp2p and move to master branch --- Cargo.lock | 289 ++++++++++++---------- Cargo.toml | 4 +- beacon_node/lighthouse_network/Cargo.toml | 5 +- 3 files changed, 158 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 668536aa6c5..d400e1ea467 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -3109,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3737,6 +3737,10 @@ name = "futures-timer" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +dependencies = [ + "gloo-timers", + "send_wrapper", +] [[package]] name = "futures-util" @@ -3832,6 +3836,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "graffiti_file" version = "0.1.0" @@ -4364,7 +4380,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -4382,7 +4398,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -4504,35 +4520,35 @@ dependencies = [ [[package]] name = "if-addrs" -version = "0.10.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a" +checksum = "bf39cc0423ee66021dc5eccface85580e4a001e0c5288bae8bea7ecb69225e90" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "if-addrs" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf39cc0423ee66021dc5eccface85580e4a001e0c5288bae8bea7ecb69225e90" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "if-watch" -version = "3.2.1" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9d64cfcf380606e64f9a0bcf493616b65331199f984151a6fa11a7b3cde38" +checksum = "71c02a5161c313f0cbdbadc511611893584a10a7b6153cb554bdf83ddce99ec2" dependencies = [ "async-io", "core-foundation 0.9.4", "fnv", "futures", - "if-addrs 0.10.2", + "if-addrs 0.15.0", "ipnet", "log", "netlink-packet-core", @@ -4919,9 +4935,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.183" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libloading" @@ -4956,8 +4972,8 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.57.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "bytes", "either", @@ -4987,8 +5003,8 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" -version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.7.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4997,8 +5013,8 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" -version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.7.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5007,8 +5023,8 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.43.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.44.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "either", "fnv", @@ -5032,7 +5048,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.45.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "hickory-resolver", @@ -5046,7 +5062,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "async-channel 2.5.0", "asynchronous-codec", @@ -5075,8 +5091,8 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.48.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "asynchronous-codec", "either", @@ -5115,8 +5131,8 @@ dependencies = [ [[package]] name = "libp2p-mdns" -version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.49.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "hickory-proto", @@ -5126,15 +5142,15 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "smallvec", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "tracing", ] [[package]] name = "libp2p-metrics" -version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.18.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "libp2p-core", @@ -5149,8 +5165,8 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.44.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "asynchronous-codec", "bytes", @@ -5167,8 +5183,8 @@ dependencies = [ [[package]] name = "libp2p-noise" -version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.47.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "asynchronous-codec", "bytes", @@ -5189,8 +5205,8 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.14.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "futures-timer", @@ -5202,7 +5218,7 @@ dependencies = [ "rand 0.8.5", "ring", "rustls 0.23.35", - "socket2 0.6.1", + "socket2 0.6.3", "thiserror 2.0.17", "tokio", "tracing", @@ -5210,13 +5226,14 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.47.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.48.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "either", "fnv", "futures", "futures-timer", + "getrandom 0.2.16", "hashlink 0.11.0", "libp2p-core", "libp2p-identity", @@ -5226,13 +5243,14 @@ dependencies = [ "smallvec", "tokio", "tracing", + "wasm-bindgen-futures", "web-time", ] [[package]] name = "libp2p-swarm-derive" -version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.36.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "heck", "quote", @@ -5241,23 +5259,23 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.44.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.45.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "tracing", ] [[package]] name = "libp2p-tls" -version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.7.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "futures-rustls", @@ -5274,8 +5292,8 @@ dependencies = [ [[package]] name = "libp2p-upnp" -version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.7.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "futures-timer", @@ -5288,8 +5306,8 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.48.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "either", "futures", @@ -5969,8 +5987,8 @@ dependencies = [ [[package]] name = "multistream-select" -version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.14.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "bytes", "futures", @@ -5982,46 +6000,30 @@ dependencies = [ [[package]] name = "netlink-packet-core" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4" +checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" dependencies = [ - "anyhow", - "byteorder", - "netlink-packet-utils", + "paste", ] [[package]] name = "netlink-packet-route" -version = "0.17.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" +checksum = "4ce3636fa715e988114552619582b530481fd5ef176a1e5c1bf024077c2c9445" dependencies = [ - "anyhow", - "bitflags 1.3.2", - "byteorder", + "bitflags 2.10.0", "libc", + "log", "netlink-packet-core", - "netlink-packet-utils", -] - -[[package]] -name = "netlink-packet-utils" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" -dependencies = [ - "anyhow", - "byteorder", - "paste", - "thiserror 1.0.69", ] [[package]] name = "netlink-proto" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60" +checksum = "b65d130ee111430e47eed7896ea43ca693c387f097dd97376bffafbf25812128" dependencies = [ "bytes", "futures", @@ -6033,12 +6035,12 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" dependencies = [ "bytes", - "futures", + "futures-util", "libc", "log", "tokio", @@ -6124,17 +6126,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - [[package]] name = "nix" version = "0.30.1" @@ -6196,7 +6187,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6624,18 +6615,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", @@ -7067,8 +7058,8 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" -version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.4.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "asynchronous-codec", "bytes", @@ -7091,7 +7082,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.6.1", + "socket2 0.6.3", "thiserror 2.0.17", "tokio", "tracing", @@ -7128,7 +7119,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -7514,18 +7505,18 @@ dependencies = [ [[package]] name = "rtnetlink" -version = "0.13.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a552eb82d19f38c3beed3f786bd23aa434ceb9ac43ab44419ca6d67a7e186c0" +checksum = "4b960d5d873a75b5be9761b1e73b146f52dddcd27bac75263f40fba686d4d7b5" dependencies = [ - "futures", + "futures-channel", + "futures-util", "log", "netlink-packet-core", "netlink-packet-route", - "netlink-packet-utils", "netlink-proto", "netlink-sys", - "nix 0.26.4", + "nix 0.30.1", "thiserror 1.0.69", "tokio", ] @@ -7652,7 +7643,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7757,8 +7748,8 @@ dependencies = [ [[package]] name = "rw-stream-sink" -version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#968a0a3064ce58f1a3e31ef74d35a2a0a7c00fef" +version = "0.5.0" +source = "git+https://github.com/libp2p/rust-libp2p.git#f4cf4bf79b710c7502969eeab8343191ec63c956" dependencies = [ "futures", "pin-project", @@ -7947,6 +7938,12 @@ dependencies = [ "pest", ] +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "sensitive_url" version = "0.1.0" @@ -8347,9 +8344,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys 0.60.2", @@ -8623,9 +8620,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags 2.10.0", "core-foundation 0.9.4", @@ -8697,7 +8694,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8928,7 +8925,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -9152,9 +9149,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -9187,9 +9184,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -10016,7 +10013,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.60.2", ] [[package]] @@ -10027,12 +10024,14 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.53.0" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc5cf48f83140dcaab716eeaea345f9e93d0018fb81162753a3f76c3397b538" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-core 0.53.0", - "windows-targets 0.52.6", + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", ] [[package]] @@ -10048,13 +10047,12 @@ dependencies = [ ] [[package]] -name = "windows-core" -version = "0.53.0" +name = "windows-collections" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-core", ] [[package]] @@ -10066,10 +10064,21 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result 0.4.1", + "windows-result", "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -10099,12 +10108,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-result" -version = "0.1.2" +name = "windows-numerics" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-targets 0.52.6", + "windows-core", + "windows-link", ] [[package]] @@ -10218,6 +10228,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index d3463d881e4..2e62422f1ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,8 +152,6 @@ fork_choice = { path = "consensus/fork_choice" } fs2 = "0.4" futures = "0.3" genesis = { path = "beacon_node/genesis" } -# This is tracking the sigp-gossipsub branch on sigp/rust-libp2p commit: Aug 20 2025 -gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages", features = ["partial_messages", "metrics"] } graffiti_file = { path = "validator_client/graffiti_file" } hashlink = "0.9.0" health_metrics = { path = "common/health_metrics" } @@ -164,7 +162,7 @@ initialized_validators = { path = "validator_client/initialized_validators" } int_to_bytes = { path = "consensus/int_to_bytes" } itertools = "0.14" kzg = { path = "crypto/kzg" } -libp2p = { git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages", default-features = false, features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "secp256k1", "macros", "metrics", "quic", "upnp", "gossipsub"] } +libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", default-features = false, features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "secp256k1", "macros", "metrics", "quic", "upnp", "gossipsub"] } libsecp256k1 = "0.7" lighthouse_network = { path = "beacon_node/lighthouse_network" } lighthouse_validator_store = { path = "validator_client/lighthouse_validator_store" } diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index d5a28449852..44af8d70064 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -21,12 +21,13 @@ ethereum_ssz_derive = { workspace = true } fixed_bytes = { workspace = true } fnv = { workspace = true } futures = { workspace = true } -gossipsub = { workspace = true } +# Enable partial messages feature +gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/libp2p/rust-libp2p.git", features = ["partial_messages"] } hex = { workspace = true } if-addrs = "0.14" itertools = { workspace = true } libp2p = { workspace = true } -libp2p-mplex = { git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages" } +libp2p-mplex = { git = "https://github.com/libp2p/rust-libp2p.git" } lighthouse_version = { workspace = true } logging = { workspace = true } lru = { workspace = true } From 948aa892ec64aa4cc745c45ff82384c3a42e5ba1 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 22 Apr 2026 16:50:19 +0200 Subject: [PATCH 140/146] fmt From f52f0b399997b1a3a8dd863344c93f5c9dd285b7 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 22 Apr 2026 16:52:20 +0200 Subject: [PATCH 141/146] fmt --- crypto/kzg/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 201ed0ee531..6ee352b0dbb 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -21,10 +21,10 @@ pub const BYTES_PER_PROOF: usize = 48; use crate::trusted_setup::load_trusted_setup; use rayon::prelude::*; pub use rust_eth_kzg::{ - Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, constants::{BYTES_PER_CELL, CELLS_PER_EXT_BLOB}, + Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, }; -use tracing::{Span, instrument}; +use tracing::{instrument, Span}; // Note: Both `NUMBER_OF_COLUMNS` and `CELLS_PER_EXT_BLOB` are preset values - however this // is a constant in the KZG library - be aware that overriding `NUMBER_OF_COLUMNS` will break KZG From 784cb95609de99c72f96761ea05ec3fde43aaddb Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 22 Apr 2026 17:11:16 +0200 Subject: [PATCH 142/146] avoid creating sidecar and header on publish if disabled --- beacon_node/http_api/src/publish_blocks.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index dd1e9a94bd1..20c7ba8e52b 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -521,12 +521,14 @@ fn publish_column_sidecars( let mut partial_header = None; for data_col in data_column_sidecars { - if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { - let mut partial = fulu_data_col.to_partial(); - if let Some(header) = partial.sidecar.header.take() { - partial_header = Some(header); + if chain.config.enable_partial_columns { + if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { + let mut partial = fulu_data_col.to_partial(); + if let Some(header) = partial.sidecar.header.take() { + partial_header = Some(header); + } + partial_columns.push(Arc::new(partial)); } - partial_columns.push(Arc::new(partial)); } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); From d3a4145a28697a8b9178fc17937593b05082a84d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 23 Apr 2026 15:51:37 +0200 Subject: [PATCH 143/146] Fix fmt --- beacon_node/http_api/src/publish_blocks.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 20c7ba8e52b..6a5dabb714f 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -521,15 +521,14 @@ fn publish_column_sidecars( let mut partial_header = None; for data_col in data_column_sidecars { - if chain.config.enable_partial_columns { - if let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { + if chain.config.enable_partial_columns + && let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { let mut partial = fulu_data_col.to_partial(); if let Some(header) = partial.sidecar.header.take() { partial_header = Some(header); } partial_columns.push(Arc::new(partial)); } - } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); full_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( From 806ecbbcdee6a718dbd564c226d6718e5dd5e33a Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 23 Apr 2026 18:58:36 +0200 Subject: [PATCH 144/146] Fix fmt --- beacon_node/http_api/src/publish_blocks.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 6a5dabb714f..6b65995a736 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -522,13 +522,14 @@ fn publish_column_sidecars( for data_col in data_column_sidecars { if chain.config.enable_partial_columns - && let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() { - let mut partial = fulu_data_col.to_partial(); - if let Some(header) = partial.sidecar.header.take() { - partial_header = Some(header); - } - partial_columns.push(Arc::new(partial)); + && let DataColumnSidecar::Fulu(fulu_data_col) = data_col.as_ref() + { + let mut partial = fulu_data_col.to_partial(); + if let Some(header) = partial.sidecar.header.take() { + partial_header = Some(header); } + partial_columns.push(Arc::new(partial)); + } let subnet = DataColumnSubnetId::from_column_index(*data_col.index(), &chain.spec); full_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( From 0e075c542c46eb34c7bd018aa98454c4ef034941 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 23 Apr 2026 19:08:11 +0200 Subject: [PATCH 145/146] Fix `PartialDataColumnSidecar::get()` blob index mapping. --- .../src/data/partial_data_column_sidecar.rs | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/consensus/types/src/data/partial_data_column_sidecar.rs b/consensus/types/src/data/partial_data_column_sidecar.rs index ff6b0695778..df65be1ae36 100644 --- a/consensus/types/src/data/partial_data_column_sidecar.rs +++ b/consensus/types/src/data/partial_data_column_sidecar.rs @@ -60,9 +60,18 @@ impl PartialDataColumnSidecar { } pub fn get(&self, idx: usize) -> Option<(&Cell, &KzgProof)> { + if !self.cells_present_bitmap.get(idx).unwrap_or(false) { + return None; + } + let storage_idx = self + .cells_present_bitmap + .iter() + .take(idx) + .filter(|b| *b) + .count(); self.column - .get(idx) - .and_then(|cell| self.kzg_proofs.get(idx).map(|proof| (cell, proof))) + .get(storage_idx) + .and_then(|cell| self.kzg_proofs.get(storage_idx).map(|proof| (cell, proof))) } /// Creates a reference to this sidecar containing only the blob indices for which the passed @@ -382,4 +391,39 @@ mod tests { }; assert!(partial.try_clone_full(&header).is_none()); } + + // -- get tests -- + + #[test] + fn get_sparse_bitmap_maps_to_correct_storage_position() { + // bitmap: [false, true, false, true] → column: [cell_1, cell_3] + let sidecar = make_sidecar_with_marker(4, &[1, 3], 0); + let (cell, _) = sidecar.get(1).expect("cell at blob index 1 should exist"); + assert_eq!(cell[0], 1); + let (cell, _) = sidecar.get(3).expect("cell at blob index 3 should exist"); + assert_eq!(cell[0], 3); + } + + #[test] + fn get_absent_blob_index_returns_none() { + let sidecar = make_sidecar(4, &[1, 3]); + assert!(sidecar.get(0).is_none()); + assert!(sidecar.get(2).is_none()); + } + + #[test] + fn get_out_of_range_returns_none() { + let sidecar = make_sidecar(4, &[0, 2]); + assert!(sidecar.get(4).is_none()); + assert!(sidecar.get(100).is_none()); + } + + #[test] + fn get_dense_bitmap_matches_direct_index() { + let sidecar = make_sidecar_with_marker(4, &[0, 1, 2, 3], 10); + for i in 0..4 { + let (cell, _) = sidecar.get(i).expect("all cells should be present"); + assert_eq!(cell[0], 10 + i as u8); + } + } } From 4cd67db8b72d6a317b9cfd097dfe0abcf50fb6d2 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 23 Apr 2026 19:44:33 +0200 Subject: [PATCH 146/146] Fix syn duplicate in Cargo.lock (see 9f3873f2bf) and remove comment code. --- Cargo.lock | 2 +- Cargo.toml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f8f3a48ae8..aefd51a9501 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2e62422f1ef..1f58c322f19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -276,7 +276,3 @@ debug = true [patch.crates-io] quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "87c4ccb9bb2af494de375f5f6c62850badd26304" } -# This is intentionally unset as we use jxs fork in this PR. TODO(dknopik): restore/remove -#[patch."https://github.com/libp2p/rust-libp2p.git"] -#libp2p = { git = "https://github.com/sigp/rust-libp2p.git", rev = "defcaf1a78cf5b70a723b3fee0e0be051c1dbd88" } -#libp2p-mplex = { git = "https://github.com/sigp/rust-libp2p.git", rev = "defcaf1a78cf5b70a723b3fee0e0be051c1dbd88" }