From 9bb0dc4d003d8be8191d55c53677c0254eeb2f48 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Sun, 25 Oct 2026 20:07:12 -0700 Subject: [PATCH 01/20] store: flush Class C from the live slots The dirty-epoch mark is gone. A flush records the unstored index and the length, pwrites those live slots in fixed scratch chunks, and clears the mark only after that persist succeeds. A store that starts in that window is refused; the next flush writes it. A same-length rewrite still publishes the length. A failed write leaves the range dirty. --- crates/rbitcoin-store/src/array_table.rs | 275 +++++++++--------- crates/rbitcoin-store/src/chain.rs | 147 ++++++---- crates/rbitcoin-store/tests/array_table.rs | 211 ++++++++++---- .../tests/chain/strong_tests.rs | 127 +++++++- crates/rbitcoin-store/tests/dev_full.rs | 37 +++ crates/rbitcoin-store/tests/suite.rs | 1 + 6 files changed, 523 insertions(+), 275 deletions(-) create mode 100644 crates/rbitcoin-store/tests/dev_full.rs diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index cbba102c4d..9fa32fb7d8 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -7,51 +7,40 @@ //! Compact **array** tables (`confirmed`, `header_txs_*`; body ≤ //! [`class_c_inram_max_bytes`]) load fully into process RAM on open. //! `strong_tx` is always L2 (see [`crate::chain::StrongTxTable`]). -//! Mutates update the `Vec` only and mark dirty. Disk is updated on -//! `Self::flush` / barrier: +//! Mutates update the `Vec` only and record the lowest unstored index. Disk +//! is updated on `Self::flush` / barrier by reading those live slots: //! -//! - **Append-only** (dirty only at indices ≥ last flushed len): write the new -//! **suffix** only, then extend HWM — never overwrites published prefix bytes. -//! - **Truncate**: shrink HWM only (complete-or-fail length publish). -//! - **In-prefix mutate** (rare for tip Class C): full body rewrite (residual -//! mid-pwrite tear risk on same-size published range — mitigated by tip-last -//! barrier so `confirmed` is not advanced until strong/height are durable). +//! - **Append** (unstored index ≥ last flushed len): pwrite the new suffix. +//! - **Truncate**: shrink the published length only. +//! - **In-prefix mutate**: pwrite from the first unstored index through the +//! length recorded when the flush started, then publish that length even +//! when it equals the previous disk length. //! -//! Tables over the cap stay pure fd L0 (per-slot write-through). +//! Each pwrite is at most [`FLUSH_SCRATCH`] bytes. The unstored mark clears +//! only after that persist succeeds. A store that starts after the flush has +//! recorded its range is refused until the flush returns. +//! +//! Tables over the cap stay pure fd L0 (per-slot write-through). The flush +//! does not read that path. use crate::error::StoreError; use crate::file::{TableFile, FILE_HEADER_LEN}; use rbitcoin_primitives::TableKind; use std::path::Path; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Mutex, RwLock}; -/// Packed dirty generation: `0` is clean. `set` bumps; flush CASes back to `0`. -pub fn dirty_epoch_bump(epoch: &AtomicU64) { - let _ = epoch.try_update(Ordering::Release, Ordering::Relaxed, |e| { - Some(e.wrapping_add(1).max(1)) - }); -} - -/// Mark clean after a persist that sampled generation `e0`. `lo` resets -/// first: a `set` that lands here bumps the generation, the clean fails, and -/// its index survives for the next flush. -pub fn dirty_try_clean(epoch: &AtomicU64, lo: &AtomicU64, e0: u64, between: impl FnOnce()) { - let prev = lo.swap(u64::MAX, Ordering::AcqRel); - between(); - if epoch - .compare_exchange(e0, 0, Ordering::Release, Ordering::Relaxed) - .is_err() - { - lo.fetch_min(prev, Ordering::AcqRel); - } -} - #[doc(hidden)] pub const ELEM: u64 = 8; -/// Where a flush calls its test hook: after the snapshot, before the write; -/// or inside the clean step. +/// Bytes of the fixed pwrite buffer. A flush does not allocate a second image. +#[doc(hidden)] +pub const FLUSH_SCRATCH: usize = 4096; + +const CLEAN: u64 = u64::MAX; + +/// Where a flush calls its test hook: after the range is recorded, before the +/// write; or after the persist, before the unstored mark clears. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[doc(hidden)] pub enum FlushStep { @@ -75,33 +64,67 @@ pub fn class_c_inram_max_bytes() -> u64 { } } +/// Pwrite `len` bytes at `base` in [`FLUSH_SCRATCH`] chunks. `fill` loads one +/// chunk from the live image. Returns the largest chunk written. +pub(crate) fn pwrite_scratched( + mut write: impl FnMut(u64, &[u8]) -> Result<(), StoreError>, + base: u64, + len: u64, + mut fill: impl FnMut(&mut [u8], u64) -> Result<(), StoreError>, +) -> Result { + let mut scratch = [0u8; FLUSH_SCRATCH]; + let mut off = 0u64; + let mut max_chunk = 0usize; + while off < len { + let n = ((len - off) as usize).min(FLUSH_SCRATCH); + fill(&mut scratch[..n], off)?; + max_chunk = max_chunk.max(n); + write(base + off, &scratch[..n])?; + off += n as u64; + } + Ok(max_chunk) +} + pub struct ArrayTable { pub file: TableFile, pub len: AtomicU64, /// `Some` = L2 authoritative image; `None` = L0 fd-only path. pub data: RwLock>>, - /// `0` = clean. Nonzero generation: a `set` raced persist if CAS fails. - pub dirty_epoch: AtomicU64, - /// Min index mutated since last flush (`u64::MAX` = none). - pub dirty_lo: AtomicU64, + /// Lowest index not yet persisted. [`CLEAN`] means none. + pub unstored_from: AtomicU64, + /// True from the moment a flush records its range until it returns. + pub flushing: AtomicBool, /// Element count last successfully flushed to disk (L2). pub disk_len: AtomicU64, - /// One flusher at a time: a second snapshot never lands under an older one. + /// One flusher at a time. pub flush_lock: Mutex<()>, + /// Largest chunk the last flush passed to `write_at`. + #[doc(hidden)] + pub flush_chunk_max: AtomicUsize, +} + +/// Clears `flushing` on drop. On success, also publishes `disk_len` and +/// clears the unstored mark, in that order, while stores are still refused. +#[must_use] +struct EndFlush<'a> { + table: &'a ArrayTable, + persisted: Option, +} + +impl Drop for EndFlush<'_> { + fn drop(&mut self) { + if let Some(n) = self.persisted { + self.table.disk_len.store(n, Ordering::Release); + self.table.unstored_from.store(CLEAN, Ordering::Release); + } + self.table.flushing.store(false, Ordering::Release); + } } impl ArrayTable { pub fn create(path: impl AsRef, kind: TableKind) -> Result { let file = TableFile::create(path.as_ref(), kind)?; - Ok(Self { - file, - len: AtomicU64::new(0), - data: RwLock::new(Some(Vec::new())), - dirty_epoch: AtomicU64::new(0), - dirty_lo: AtomicU64::new(u64::MAX), - disk_len: AtomicU64::new(0), - flush_lock: Mutex::new(()), - }) + Ok(Self::new(file, Some(Vec::new()), 0)) } pub fn open(path: impl AsRef, kind: TableKind) -> Result { @@ -124,15 +147,20 @@ impl ArrayTable { } else { None }; - Ok(Self { + Ok(Self::new(file, data, n)) + } + + fn new(file: TableFile, data: Option>, n: u64) -> Self { + Self { file, len: AtomicU64::new(n), data: RwLock::new(data), - dirty_epoch: AtomicU64::new(0), - dirty_lo: AtomicU64::new(u64::MAX), + unstored_from: AtomicU64::new(CLEAN), + flushing: AtomicBool::new(false), disk_len: AtomicU64::new(n), flush_lock: Mutex::new(()), - }) + flush_chunk_max: AtomicUsize::new(0), + } } pub fn len(&self) -> u64 { @@ -148,23 +176,18 @@ impl ArrayTable { FILE_HEADER_LEN as u64 + index * ELEM } - #[doc(hidden)] - pub fn mark_dirty_index(&self, index: u64) { - dirty_epoch_bump(&self.dirty_epoch); - let mut cur = self.dirty_lo.load(Ordering::Relaxed); - while index < cur { - match self.dirty_lo.compare_exchange_weak( - cur, - index, - Ordering::Release, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(c) => cur = c, - } + fn refuse_while_flushing(&self) -> Result<(), StoreError> { + if self.flushing.load(Ordering::Acquire) { + Err(StoreError::Rejected("class C store during flush")) + } else { + Ok(()) } } + fn mark_unstored(&self, index: u64) { + let _ = self.unstored_from.fetch_min(index, Ordering::AcqRel); + } + pub fn get(&self, index: u64) -> Result { let len = self.len.load(Ordering::Acquire); if index >= len { @@ -183,6 +206,7 @@ impl ArrayTable { pub fn set(&self, index: u64, value: u64) -> Result<(), StoreError> { let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; if let Some(ref mut v) = *guard { let need = index as usize + 1; if v.len() < need { @@ -190,7 +214,7 @@ impl ArrayTable { } v[index as usize] = value; self.len.store(v.len() as u64, Ordering::Release); - self.mark_dirty_index(index); + self.mark_unstored(index); return Ok(()); } drop(guard); @@ -216,6 +240,7 @@ impl ArrayTable { let max_idx = pairs.iter().map(|(i, _)| *i).max().unwrap(); let min_idx = pairs.iter().map(|(i, _)| *i).min().unwrap(); let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; if let Some(ref mut v) = *guard { let need = max_idx as usize + 1; if v.len() < need { @@ -225,7 +250,7 @@ impl ArrayTable { v[index as usize] = value; } self.len.store(v.len() as u64, Ordering::Release); - self.mark_dirty_index(min_idx); + self.mark_unstored(min_idx); return Ok(()); } drop(guard); @@ -258,12 +283,12 @@ impl ArrayTable { return Err(StoreError::Corrupt("array truncate grows")); } let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; if let Some(ref mut v) = *guard { v.truncate(new_len as usize); self.len.store(new_len, Ordering::Release); - // Shrink is a length-only publish; mark dirty from 0 so flush takes - // the truncate path (set_logical_len), not an append. - self.mark_dirty_index(0); + // Length-only publish. Mark the start so flush takes the shrink path. + self.mark_unstored(0); return Ok(()); } drop(guard); @@ -273,8 +298,6 @@ impl ArrayTable { Ok(()) } - /// Persist dirty L2 image. Prefers append-only suffix writes so published - /// prefix bytes are never overwritten mid-barrier. /// In-RAM L2 image bytes (0 when FdOnly). pub fn l2_resident_bytes(&self) -> u64 { let g = self.data.read().unwrap_or_else(|e| e.into_inner()); @@ -283,13 +306,6 @@ impl ArrayTable { .unwrap_or(0) } - #[doc(hidden)] - pub fn note_flush_clean(&self, e0: u64, between: &mut impl FnMut(&Self, FlushStep)) { - dirty_try_clean(&self.dirty_epoch, &self.dirty_lo, e0, || { - between(self, FlushStep::Clean) - }); - } - pub fn flush_dirty(&self) -> Result<(), StoreError> { self.flush_dirty_between(|_, _| {}) } @@ -300,68 +316,51 @@ impl ArrayTable { mut between: impl FnMut(&Self, FlushStep), ) -> Result<(), StoreError> { let _one = self.flush_lock.lock().unwrap_or_else(|e| e.into_inner()); - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let Some(ref v) = *guard else { - return Ok(()); - }; - let e0 = self.dirty_epoch.load(Ordering::Acquire); - if e0 == 0 { - return Ok(()); - } - let n = v.len() as u64; - let disk = self.disk_len.load(Ordering::Acquire); - let dirty_lo = self.dirty_lo.load(Ordering::Acquire); - - if n < disk { - drop(guard); - between(self, FlushStep::Write); - let logical = FILE_HEADER_LEN as u64 + n * ELEM; - self.file.set_logical_len(logical)?; - self.disk_len.store(n, Ordering::Release); - self.note_flush_clean(e0, &mut between); - return Ok(()); - } - - if n == disk && dirty_lo >= disk { - drop(guard); - between(self, FlushStep::Write); - self.note_flush_clean(e0, &mut between); - return Ok(()); - } - - if dirty_lo >= disk && n > disk { - let start = disk as usize; - let mut bytes = vec![0u8; ((n - disk) as usize) * 8]; - for (i, &val) in v[start..].iter().enumerate() { - bytes[i * 8..(i + 1) * 8].copy_from_slice(&val.to_le_bytes()); + let (from, n, disk) = { + let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); + let Some(ref v) = *guard else { + return Ok(()); + }; + let from = self.unstored_from.load(Ordering::Acquire); + if from == CLEAN { + return Ok(()); } - drop(guard); - between(self, FlushStep::Write); - self.file.write_at(Self::offset(disk), &bytes)?; - let logical = FILE_HEADER_LEN as u64 + n * ELEM; - debug_assert_eq!(self.file.logical_len(), logical); - let _ = logical; - self.disk_len.store(n, Ordering::Release); - self.note_flush_clean(e0, &mut between); - return Ok(()); - } - - let from = dirty_lo.min(n); - let mut bytes = vec![0u8; ((n - from) as usize) * 8]; - for (i, &val) in v[from as usize..].iter().enumerate() { - bytes[i * 8..(i + 1) * 8].copy_from_slice(&val.to_le_bytes()); - } - drop(guard); + self.flushing.store(true, Ordering::Release); + (from, v.len() as u64, self.disk_len.load(Ordering::Acquire)) + }; + let mut end = EndFlush { + table: self, + persisted: None, + }; between(self, FlushStep::Write); - if !bytes.is_empty() { - self.file.write_at(Self::offset(from), &bytes)?; - } - if n != disk { - let logical = FILE_HEADER_LEN as u64 + n * ELEM; - self.file.set_logical_len(logical)?; + // Disconnect shrinks `confirmed` once per block. Rewriting the kept + // prefix would copy the whole image on that path. + if n >= disk { + let start = from.min(disk); + if start < n { + let max = pwrite_scratched( + |at, chunk| self.file.write_at(at, chunk), + Self::offset(start), + (n - start) * ELEM, + |dst, off| { + let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); + let v = guard + .as_ref() + .ok_or(StoreError::Corrupt("class C flush lost its image"))?; + let base = start as usize + (off as usize) / 8; + for (i, slot) in dst.chunks_exact_mut(8).enumerate() { + slot.copy_from_slice(&v[base + i].to_le_bytes()); + } + Ok(()) + }, + )?; + self.flush_chunk_max.store(max, Ordering::Relaxed); + } } - self.disk_len.store(n, Ordering::Release); - self.note_flush_clean(e0, &mut between); + let logical = FILE_HEADER_LEN as u64 + n * ELEM; + self.file.set_logical_len(logical)?; + between(self, FlushStep::Clean); + end.persisted = Some(n); Ok(()) } diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index 072c70c0fc..334cdf932a 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -140,12 +140,32 @@ pub struct StrongTxTable { pub n_bits: std::sync::atomic::AtomicU64, /// L2 byte image. `strong_tx` stays in this `Vec`; open and create never go FdOnly. pub data: std::sync::RwLock>, - /// `0` = clean. Nonzero generation: a bit `set` raced persist if CAS fails. - pub dirty_epoch: std::sync::atomic::AtomicU64, - /// Min bit index mutated since last flush (`u64::MAX` = none). - pub dirty_lo_bit: std::sync::atomic::AtomicU64, + /// Lowest bit not yet persisted (`u64::MAX` = clean). + pub unstored_bit: std::sync::atomic::AtomicU64, + /// True from the moment a flush records its range until it returns. + pub flushing: std::sync::atomic::AtomicBool, /// Body byte length last flushed to disk (L2). pub disk_bytes: std::sync::atomic::AtomicU64, + /// Largest chunk the last flush passed to `write_at`. + #[doc(hidden)] + pub flush_chunk_max: std::sync::atomic::AtomicUsize, +} + +#[must_use] +struct EndStrongFlush<'a> { + table: &'a StrongTxTable, + persisted: Option, +} + +impl Drop for EndStrongFlush<'_> { + fn drop(&mut self) { + use std::sync::atomic::Ordering; + if let Some(n) = self.persisted { + self.table.disk_bytes.store(n, Ordering::Release); + self.table.unstored_bit.store(u64::MAX, Ordering::Release); + } + self.table.flushing.store(false, Ordering::Release); + } } impl StrongTxTable { @@ -154,9 +174,10 @@ impl StrongTxTable { bits: crate::file::TableFile::create(dir.join("strong_tx.body"), TableKind::StrongTx)?, n_bits: std::sync::atomic::AtomicU64::new(0), data: std::sync::RwLock::new(Vec::new()), - dirty_epoch: std::sync::atomic::AtomicU64::new(0), - dirty_lo_bit: std::sync::atomic::AtomicU64::new(u64::MAX), + unstored_bit: std::sync::atomic::AtomicU64::new(u64::MAX), + flushing: std::sync::atomic::AtomicBool::new(false), disk_bytes: std::sync::atomic::AtomicU64::new(0), + flush_chunk_max: std::sync::atomic::AtomicUsize::new(0), }) } @@ -174,28 +195,26 @@ impl StrongTxTable { bits, n_bits: std::sync::atomic::AtomicU64::new(n_bits), data: std::sync::RwLock::new(v), - dirty_epoch: std::sync::atomic::AtomicU64::new(0), - dirty_lo_bit: std::sync::atomic::AtomicU64::new(u64::MAX), + unstored_bit: std::sync::atomic::AtomicU64::new(u64::MAX), + flushing: std::sync::atomic::AtomicBool::new(false), disk_bytes: std::sync::atomic::AtomicU64::new(body), + flush_chunk_max: std::sync::atomic::AtomicUsize::new(0), }) } + fn refuse_while_flushing(&self) -> Result<(), StoreError> { + use std::sync::atomic::Ordering; + if self.flushing.load(Ordering::Acquire) { + Err(StoreError::Rejected("class C store during flush")) + } else { + Ok(()) + } + } + #[doc(hidden)] pub fn mark_dirty_bit(&self, bit: u64) { use std::sync::atomic::Ordering; - crate::array_table::dirty_epoch_bump(&self.dirty_epoch); - let mut cur = self.dirty_lo_bit.load(Ordering::Relaxed); - while bit < cur { - match self.dirty_lo_bit.compare_exchange_weak( - cur, - bit, - Ordering::Release, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(c) => cur = c, - } - } + let _ = self.unstored_bit.fetch_min(bit, Ordering::AcqRel); } #[doc(hidden)] @@ -207,6 +226,7 @@ impl StrongTxTable { } let need_bytes = need_bits.div_ceil(8); let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; if need_bytes as usize > guard.len() { guard.resize(need_bytes as usize, 0); } @@ -231,6 +251,7 @@ impl StrongTxTable { pub fn set_bit(&self, bit: u64, on: bool) -> Result<(), StoreError> { self.ensure_bits(bit + 1)?; let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; let bi = (bit / 8) as usize; if on { guard[bi] |= 1 << (bit % 8); @@ -292,6 +313,7 @@ impl StrongTxTable { } self.ensure_bits(end)?; let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); + self.refuse_while_flushing()?; let mut bit = start; while bit < end { let bi = (bit / 8) as usize; @@ -407,50 +429,55 @@ impl StrongTxTable { Ok(ones == u64::from(count)) } - /// Persist dirty L2 bit image. Prefers append-only byte suffix writes. + /// Persist dirty L2 bit image by reading live bytes. The unstored mark + /// clears only after the length publish succeeds. pub fn flush_dirty(&self) -> Result<(), StoreError> { - use std::sync::atomic::Ordering; - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let v = &*guard; - let e0 = self.dirty_epoch.load(Ordering::Acquire); - if e0 == 0 { - return Ok(()); - } - let body_len = v.len() as u64; - let disk = self.disk_bytes.load(Ordering::Acquire); - let dirty_lo = self.dirty_lo_bit.load(Ordering::Acquire); + self.flush_dirty_between(|_, _| {}) + } - let dirty_byte = if dirty_lo == u64::MAX { - body_len - } else { - dirty_lo / 8 + #[doc(hidden)] + pub fn flush_dirty_between( + &self, + mut between: impl FnMut(&Self, crate::array_table::FlushStep), + ) -> Result<(), StoreError> { + use std::sync::atomic::Ordering; + let (body_len, disk, dirty_byte) = { + let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); + let bit = self.unstored_bit.load(Ordering::Acquire); + if bit == u64::MAX { + return Ok(()); + } + self.flushing.store(true, Ordering::Release); + let body_len = guard.len() as u64; + let disk = self.disk_bytes.load(Ordering::Acquire); + (body_len, disk, bit / 8) }; - - if body_len > disk && dirty_byte >= disk { - let suffix = v[disk as usize..].to_vec(); - drop(guard); - if !suffix.is_empty() { - self.bits - .write_at(crate::file::FILE_HEADER_LEN as u64 + disk, &suffix)?; + let mut end = EndStrongFlush { + table: self, + persisted: None, + }; + between(self, crate::array_table::FlushStep::Write); + if body_len >= disk { + let start = dirty_byte.min(disk); + if start < body_len { + let max = crate::array_table::pwrite_scratched( + |at, chunk| self.bits.write_at(at, chunk), + crate::file::FILE_HEADER_LEN as u64 + start, + body_len - start, + |dst, off| { + let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); + let at = start as usize + off as usize; + dst.copy_from_slice(&guard[at..at + dst.len()]); + Ok(()) + }, + )?; + self.flush_chunk_max.store(max, Ordering::Relaxed); } - self.disk_bytes.store(body_len, Ordering::Release); - crate::array_table::dirty_try_clean(&self.dirty_epoch, &self.dirty_lo_bit, e0, || {}); - return Ok(()); - } - - let from = dirty_byte.min(body_len); - let suffix = v[from as usize..].to_vec(); - drop(guard); - if !suffix.is_empty() { - self.bits - .write_at(crate::file::FILE_HEADER_LEN as u64 + from, &suffix)?; - } - if body_len != disk { - let logical = crate::file::FILE_HEADER_LEN as u64 + body_len; - self.bits.set_logical_len(logical)?; } - self.disk_bytes.store(body_len, Ordering::Release); - crate::array_table::dirty_try_clean(&self.dirty_epoch, &self.dirty_lo_bit, e0, || {}); + let logical = crate::file::FILE_HEADER_LEN as u64 + body_len; + self.bits.set_logical_len(logical)?; + between(self, crate::array_table::FlushStep::Clean); + end.persisted = Some(body_len); Ok(()) } diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index d606638bd4..c521b8364b 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -2,7 +2,7 @@ use rbitcoin_primitives::TableKind; use rbitcoin_store::array_table::*; use rbitcoin_store::error::StoreError; use rbitcoin_store::file::FILE_HEADER_LEN; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::Ordering; fn tmp_path() -> std::path::PathBuf { std::env::temp_dir().join(format!( @@ -50,71 +50,155 @@ fn get_racing_truncate_does_not_panic() { let _ = std::fs::remove_file(&path); } -/// A `set` that lands while flush marks the table clean is written by -/// the next flush. The IBD tip flush runs beside the next batch's archive. +/// Flush persists the live slots of the dirty range recorded at start. +/// A store from the write hook is refused; the bytes on disk are the slots +/// after that hook, so the flush did not keep a detached image. A store +/// after flush returns is absent here and present in the next flush. #[test] -fn set_during_flush_clean_is_flushed_next() { +fn flush_writes_live_slots_and_refuses_a_store_until_it_finishes() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); - t.set_many(&[(0, 10), (1, 11), (2, 12), (3, 13)]).unwrap(); + let n = 600u64; + let pairs: Vec<(u64, u64)> = (0..n).map(|i| (i, i + 1)).collect(); + t.set_many(&pairs).unwrap(); t.flush().unwrap(); - t.set(3, 23).unwrap(); + t.set(0, 5).unwrap(); + let mut during = Ok(()); t.flush_dirty_between(|t, step| { - if step == FlushStep::Clean { - t.set(1, 21).unwrap(); + if step == FlushStep::Write { + if let Some(v) = t.data.write().unwrap_or_else(|e| e.into_inner()).as_mut() { + v[0] = 77; + } + during = t.set(1, 9); + let _ = t.set_many(&[(2, 8)]); + let _ = t.truncate(n - 1); } }) .unwrap(); + let slot0 = { + let mut buf = [0u8; 8]; + t.file.read_at(FILE_HEADER_LEN as u64, &mut buf).unwrap(); + u64::from_le_bytes(buf) + }; + assert!( + during.is_err() && slot0 == 77, + "store during flush refused={during:?} live slot0={slot0}" + ); + assert!(n * ELEM > FLUSH_SCRATCH as u64); + let chunk = t.flush_chunk_max.load(Ordering::Relaxed); + assert!(chunk > 0 && chunk <= FLUSH_SCRATCH, "flush chunk {chunk}"); + t.set(1, 9).unwrap(); + let mut buf = [0u8; 8]; + t.file + .read_at(FILE_HEADER_LEN as u64 + 8, &mut buf) + .unwrap(); + assert_eq!( + u64::from_le_bytes(buf), + 2, + "store after flush stays off disk" + ); + t.flush().unwrap(); + t.file + .read_at(FILE_HEADER_LEN as u64 + 8, &mut buf) + .unwrap(); + assert_eq!(u64::from_le_bytes(buf), 9); + let _ = std::fs::remove_file(&path); +} + +/// A rewrite whose length equals the last flushed length still publishes +/// that length. A torn longer HWM must not survive the flush. +#[test] +fn same_length_rewrite_publishes_logical_len() { + let path = tmp_path(); + let _ = std::fs::remove_file(&path); + let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); + t.set_many(&[(0, 1), (1, 2), (2, 3), (3, 4)]).unwrap(); + t.flush().unwrap(); + let published = FILE_HEADER_LEN as u64 + 4 * ELEM; + assert_eq!(t.file.logical_len(), published); + t.file.set_logical_len(published + 64).unwrap(); + t.set(1, 99).unwrap(); + assert_eq!(t.len(), 4); + assert_eq!(t.disk_len.load(Ordering::Acquire), 4); t.flush().unwrap(); + assert_eq!(t.file.logical_len(), published); drop(t); - let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); - assert_eq!((t.get(1).unwrap(), t.get(3).unwrap()), (21, 23)); + let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); + assert_eq!(t.len(), 4); + assert_eq!(t.get(1).unwrap(), 99); let _ = std::fs::remove_file(&path); } -/// Two flushes of one table never leave the older image on disk. A body -/// repair flushes `header_txs` beside the IBD tip flush. +/// A failed `write_at` leaves the recorded range dirty. The retry writes +/// those slots and does not truncate the table. +#[cfg(unix)] #[test] -fn concurrent_flushes_keep_the_newer_image() { - use std::sync::Arc; +fn failed_write_keeps_the_range_dirty() { + let path = tmp_path(); + let _ = std::fs::remove_file(&path); + let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); + t.set_many(&[(0, 1), (1, 2), (2, 3), (3, 4)]).unwrap(); + t.flush().unwrap(); + t.set(1, 42).unwrap(); + let mut swapped = None; + let failed = t.flush_dirty_between(|t, step| { + if step == FlushStep::Write { + swapped = Some(crate::dev_full::DevFull::on(&t.file.read_file)); + } + }); + drop(swapped); + assert!( + failed.is_err(), + "write_at must fail while the fd is /dev/full" + ); + assert_eq!(t.len(), 4, "a failed persist must not truncate"); + let mut buf = [0u8; 8]; + t.file + .read_at(FILE_HEADER_LEN as u64 + 8, &mut buf) + .unwrap(); + assert_eq!( + u64::from_le_bytes(buf), + 2, + "failed persist must not publish the slot" + ); + t.flush().unwrap(); + t.file + .read_at(FILE_HEADER_LEN as u64 + 8, &mut buf) + .unwrap(); + assert_eq!(u64::from_le_bytes(buf), 42); + let _ = std::fs::remove_file(&path); +} + +/// A store during the clean step is refused. The store after flush returns +/// is what the next flush persists. +#[test] +fn set_after_flush_is_flushed_next() { let path = tmp_path(); let _ = std::fs::remove_file(&path); - let t = Arc::new(ArrayTable::create(&path, TableKind::ArrayLink).unwrap()); - t.set_many(&[(0, 10), (1, 11)]).unwrap(); + let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); + t.set_many(&[(0, 10), (1, 11), (2, 12), (3, 13)]).unwrap(); t.flush().unwrap(); - t.set(0, 20).unwrap(); - let (done, finished) = std::sync::mpsc::channel(); - let mut other = Some(Arc::clone(&t)); - let mut second = None; - t.flush_dirty_between(|_, step| { - let Some(t2) = other.take().filter(|_| step == FlushStep::Write) else { - return; - }; - let done = done.clone(); - second = Some(std::thread::spawn(move || { - t2.set(1, 21).unwrap(); - t2.flush().unwrap(); - done.send(()).unwrap(); - })); - let _ = finished.recv_timeout(std::time::Duration::from_millis(200)); + t.set(3, 23).unwrap(); + let mut during = Ok(()); + t.flush_dirty_between(|t, step| { + if step == FlushStep::Clean { + during = t.set(1, 21); + } }) .unwrap(); - second.unwrap().join().unwrap(); + assert!(during.is_err(), "store during clean must be refused"); + drop(t); + let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); + assert_eq!((t.get(1).unwrap(), t.get(3).unwrap()), (11, 23)); + t.set(1, 21).unwrap(); + t.flush().unwrap(); drop(t); let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); - assert_eq!((t.get(0).unwrap(), t.get(1).unwrap()), (20, 21)); + assert_eq!((t.get(1).unwrap(), t.get(3).unwrap()), (21, 23)); let _ = std::fs::remove_file(&path); } -#[test] -fn dirty_epoch_bump_from_max_stays_dirty() { - let e = AtomicU64::new(u64::MAX); - dirty_epoch_bump(&e); - assert_ne!(e.load(Ordering::Acquire), 0); - assert_eq!(e.load(Ordering::Acquire), 1); -} - #[test] fn array_set_get_grow_set_many_truncate_flush() { let path = tmp_path(); @@ -228,7 +312,7 @@ fn append_only_flush_extends_without_losing_prefix() { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); t.set_many(&[(0, 100), (1, 101)]).unwrap(); t.flush().unwrap(); - // Pure tip extension (dirty_lo = 2 >= disk_len = 2). + // Pure tip extension (unstored index 2, disk length 2). t.set(2, 102).unwrap(); t.flush().unwrap(); } @@ -278,40 +362,43 @@ fn set_gap_zeros_prefix_and_env_cap_and_flush_async() { let _ = std::fs::remove_file(&path); } +/// A store from another thread while flush is in its write step is refused. +/// The store after that flush joins is what the next flush persists. #[test] -fn concurrent_set_during_flush_persists_new() { - use std::sync::atomic::{AtomicBool, Ordering as AtomOrd}; +fn store_on_another_thread_during_flush_is_refused() { use std::sync::Arc; let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = Arc::new(ArrayTable::create(&path, TableKind::Confirmed).unwrap()); - let n = 32_768u64; - let pairs: Vec<(u64, u64)> = (0..n).map(|i| (i, i)).collect(); - t.set_many(&pairs).unwrap(); + t.set_many(&[(0, 1), (1, 2)]).unwrap(); t.flush().unwrap(); - let stop = Arc::new(AtomicBool::new(false)); + t.set(0, 5).unwrap(); + let (started, start_rx) = std::sync::mpsc::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); let flush_t = { let t = Arc::clone(&t); - let stop = Arc::clone(&stop); std::thread::spawn(move || { - while !stop.load(AtomOrd::Relaxed) { - t.flush_dirty().unwrap(); - } + t.flush_dirty_between(|_, step| { + if step == FlushStep::Write { + started.send(()).unwrap(); + release_rx.recv().unwrap(); + } + }) + .unwrap(); }) }; - for i in 1..2_000u64 { - t.set(0, i).unwrap(); - } - stop.store(true, AtomOrd::Relaxed); + start_rx.recv().unwrap(); + let during = t.set(0, 9); + release.send(()).unwrap(); flush_t.join().unwrap(); - let want = t.get(0).unwrap(); + assert!(during.is_err(), "store during flush refused={during:?}"); + drop(t); + let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); + assert_eq!(t.get(0).unwrap(), 5); + t.set(0, 9).unwrap(); t.flush().unwrap(); drop(t); let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); - assert_eq!( - t.get(0).unwrap(), - want, - "set that races flush must still persist" - ); + assert_eq!(t.get(0).unwrap(), 9); let _ = std::fs::remove_file(&path); } diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index a0140e1458..98ae1f5914 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -15,6 +15,96 @@ fn tmp() -> std::path::PathBuf { p } +/// Flush persists the live bit image. A bit store from the write hook is +/// refused; a store after flush returns lands in the next flush. +#[test] +fn strong_flush_writes_live_bytes_and_refuses_a_store() { + use rbitcoin_store::array_table::{FlushStep, FLUSH_SCRATCH}; + use std::sync::atomic::Ordering; + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + let bits = (FLUSH_SCRATCH as u32) * 8 + 64; + t.set_strong_range(Fk(1), bits, Fk(1)).unwrap(); + t.flush().unwrap(); + t.set_unstrong(Fk(1)).unwrap(); + let mut during = Ok(()); + t.flush_dirty_between(|t, step| { + if step == FlushStep::Write { + t.data.write().unwrap_or_else(|e| e.into_inner())[0] = 0xff; + during = t.set_unstrong(Fk(2)); + } + }) + .unwrap(); + assert!(during.is_err(), "store during flush refused={during:?}"); + let chunk = t.flush_chunk_max.load(Ordering::Relaxed); + assert!(chunk > 0 && chunk <= FLUSH_SCRATCH, "flush chunk {chunk}"); + drop(t); + let t = StrongTxTable::open(&dir).unwrap(); + assert!( + t.is_strong(Fk(1)).unwrap(), + "live byte must be the flushed image" + ); + assert!(t.is_strong(Fk(2)).unwrap(), "refused store stays off disk"); + t.set_unstrong(Fk(2)).unwrap(); + t.flush().unwrap(); + drop(t); + let t = StrongTxTable::open(&dir).unwrap(); + assert!(!t.is_strong(Fk(2)).unwrap()); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A failed bit-image `write_at` leaves that range dirty. The retry writes it. +#[cfg(unix)] +#[test] +fn strong_failed_write_keeps_the_range_dirty() { + use rbitcoin_store::array_table::FlushStep; + use rbitcoin_store::file::FILE_HEADER_LEN; + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + t.set_strong(Fk(1), Fk(1)).unwrap(); + t.flush().unwrap(); + t.set_unstrong(Fk(1)).unwrap(); + let mut swapped = None; + let failed = t.flush_dirty_between(|t, step| { + if step == FlushStep::Write { + swapped = Some(crate::dev_full::DevFull::on(&t.bits.read_file)); + } + }); + drop(swapped); + assert!( + failed.is_err(), + "write_at must fail while the fd is /dev/full" + ); + let mut byte = [0u8; 1]; + t.bits.read_at(FILE_HEADER_LEN as u64, &mut byte).unwrap(); + assert_eq!(byte[0] & 1, 1, "failed persist must not publish the clear"); + t.flush().unwrap(); + t.bits.read_at(FILE_HEADER_LEN as u64, &mut byte).unwrap(); + assert_eq!(byte[0] & 1, 0, "retry writes the same slots"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A same-length bit rewrite still publishes the recorded body length. +#[test] +fn strong_same_length_rewrite_publishes_logical_len() { + use rbitcoin_store::file::FILE_HEADER_LEN; + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + t.set_strong_range(Fk(1), 16, Fk(1)).unwrap(); + t.flush().unwrap(); + let published = t.bits.logical_len(); + assert!(published > FILE_HEADER_LEN as u64); + t.bits.set_logical_len(published + 64).unwrap(); + t.set_unstrong(Fk(3)).unwrap(); + t.flush_dirty().unwrap(); + assert_eq!(t.bits.logical_len(), published); + drop(t); + let t = StrongTxTable::open(&dir).unwrap(); + assert!(!t.is_strong(Fk(3)).unwrap()); + assert!(t.is_strong(Fk(1)).unwrap()); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn strong_bitset_range_and_clear() { let dir = tmp(); @@ -111,34 +201,41 @@ fn strong_tx_always_l2_above_class_c_cap() { } #[test] -fn concurrent_set_strong_during_flush_persists() { - use std::sync::atomic::{AtomicBool, Ordering as AtomOrd}; +fn strong_store_on_another_thread_during_flush_is_refused() { + use rbitcoin_store::array_table::FlushStep; use std::sync::Arc; let dir = tmp(); let t = Arc::new(StrongTxTable::create(&dir).unwrap()); - t.set_strong_range(Fk(1), 80_000, Fk(1)).unwrap(); + t.set_strong(Fk(1), Fk(1)).unwrap(); t.flush().unwrap(); - let stop = Arc::new(AtomicBool::new(false)); + t.set_unstrong(Fk(1)).unwrap(); + let (started, start_rx) = std::sync::mpsc::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); let flush_t = { let t = Arc::clone(&t); - let stop = Arc::clone(&stop); std::thread::spawn(move || { - while !stop.load(AtomOrd::Relaxed) { - t.flush_dirty().unwrap(); - } + t.flush_dirty_between(|_, step| { + if step == FlushStep::Write { + started.send(()).unwrap(); + release_rx.recv().unwrap(); + } + }) + .unwrap(); }) }; - for _ in 0..2_000 { - t.set_unstrong(Fk(1)).unwrap(); - t.set_strong(Fk(1), Fk(1)).unwrap(); - } - stop.store(true, AtomOrd::Relaxed); + start_rx.recv().unwrap(); + let during = t.set_strong(Fk(1), Fk(1)); + release.send(()).unwrap(); flush_t.join().unwrap(); - let want = t.is_strong(Fk(1)).unwrap(); + assert!(during.is_err(), "store during flush refused={during:?}"); + drop(t); + let t = StrongTxTable::open(&dir).unwrap(); + assert!(!t.is_strong(Fk(1)).unwrap()); + t.set_strong(Fk(1), Fk(1)).unwrap(); t.flush().unwrap(); drop(t); let t = StrongTxTable::open(&dir).unwrap(); - assert_eq!(t.is_strong(Fk(1)).unwrap(), want); + assert!(t.is_strong(Fk(1)).unwrap()); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/rbitcoin-store/tests/dev_full.rs b/crates/rbitcoin-store/tests/dev_full.rs new file mode 100644 index 0000000000..f718755b4d --- /dev/null +++ b/crates/rbitcoin-store/tests/dev_full.rs @@ -0,0 +1,37 @@ +/// Points an open file descriptor at `/dev/full` until dropped. +#[cfg(unix)] +pub struct DevFull { + fd: std::os::fd::RawFd, + saved: std::os::fd::RawFd, +} + +#[cfg(unix)] +impl DevFull { + pub fn on(file: &std::fs::File) -> Self { + use std::os::fd::AsRawFd; + let fd = file.as_raw_fd(); + // SAFETY: `fd` belongs to `file` for this call. `dup` returns a new descriptor or -1. + let saved = unsafe { libc::dup(fd) }; + assert!(saved >= 0, "dup"); + let full = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/full") + .unwrap(); + // SAFETY: `full` is open. `dup2` replaces `fd` in place; `Drop` puts the saved descriptor back. + let rc = unsafe { libc::dup2(full.as_raw_fd(), fd) }; + assert_eq!(rc, fd, "dup2"); + Self { fd, saved } + } +} + +#[cfg(unix)] +impl Drop for DevFull { + fn drop(&mut self) { + // SAFETY: `saved` is the duplicate from `on`. Restore `fd`, then close that duplicate. + unsafe { + libc::dup2(self.saved, self.fd); + libc::close(self.saved); + } + } +} diff --git a/crates/rbitcoin-store/tests/suite.rs b/crates/rbitcoin-store/tests/suite.rs index f4f4d1b844..6c7d2b93bb 100644 --- a/crates/rbitcoin-store/tests/suite.rs +++ b/crates/rbitcoin-store/tests/suite.rs @@ -10,6 +10,7 @@ mod chain; mod compact; mod create_loc; mod delta_loc; +mod dev_full; mod error; mod file; mod fuse8_filter; From b7518c9c39fa81b69192d0602f430b690c011c5d Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Sun, 25 Oct 2026 21:13:11 -0700 Subject: [PATCH 02/20] store: Class C stores take a single-writer capability ClassCWrite is not Send or Sync. Confirm-write, tip-accept, and startup mint it on the thread that stores. Archived-body clear is an item on the confirm-write channel: that thread joins the in-flight tip flush, then clears. Header range puts on that path wait for the same join, including when a later phase rejects the batch, so a retry still sees header_txs. Class A body append still overlaps the flush. The write thread polls the channel because Query keeps a sender so a clear can arrive. A blocking recv never observed disconnect at shutdown. --- changelog.d/class-c-snapshot.md | 11 + .../tests/handlers/pure_helper_tests.rs | 13 +- crates/rbitcoin-ibd/src/ibd/confirm/mod.rs | 79 +++++- crates/rbitcoin-ibd/tests/ibd/confirm.rs | 7 +- crates/rbitcoin-net/tests/chain.rs | 14 +- crates/rbitcoin-net/tests/versionbits_warn.rs | 5 +- crates/rbitcoin-query/src/archive.rs | 39 ++- crates/rbitcoin-query/src/confirm_run/mod.rs | 16 ++ .../rbitcoin-query/src/confirm_run/phases.rs | 3 +- .../rbitcoin-query/src/confirm_run/write.rs | 64 ++++- crates/rbitcoin-query/src/connect.rs | 22 +- crates/rbitcoin-query/src/connect_facade.rs | 4 +- crates/rbitcoin-query/src/lib.rs | 41 ++- crates/rbitcoin-query/tests/archive.rs | 12 +- .../rbitcoin-query/tests/bip68_store_tests.rs | 6 +- .../tests/confirm_run/bq_resolve.rs | 11 +- .../confirm_run/write_idempotent_tests.rs | 20 +- crates/rbitcoin-query/tests/root.rs | 95 ++++++- crates/rbitcoin-query/tests/spend_sync.rs | 5 +- .../tests/structure_rule_tests.rs | 47 +++- crates/rbitcoin-rpc/tests/methods/mod.rs | 12 +- crates/rbitcoin-store/src/array_table.rs | 38 ++- crates/rbitcoin-store/src/chain.rs | 146 +++++++--- crates/rbitcoin-store/src/integrity.rs | 44 +-- crates/rbitcoin-store/src/lib.rs | 1 + crates/rbitcoin-store/src/store.rs | 15 +- crates/rbitcoin-store/tests/array_table.rs | 144 +++++++--- .../tests/chain/chain_table_tests.rs | 114 ++++++-- .../tests/chain/strong_tests.rs | 117 +++++--- crates/rbitcoin-store/tests/integrity.rs | 117 ++++++-- crates/rbitcoin-store/tests/spend_durable.rs | 24 +- crates/rbitcoin-store/tests/store/mod.rs | 263 +++++++++++++----- crates/rbitcoin-test/tests/scenarios.rs | 24 +- docs/concurrency.md | 6 +- docs/crash-recovery.md | 2 +- 35 files changed, 1233 insertions(+), 348 deletions(-) create mode 100644 changelog.d/class-c-snapshot.md diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md new file mode 100644 index 0000000000..c02c719f56 --- /dev/null +++ b/changelog.d/class-c-snapshot.md @@ -0,0 +1,11 @@ +Changed + +- **Class C stores take a single-writer capability.** + `ClassCWrite` is not `Send`. Confirm-write, tip-accept, and startup mint + it on the thread that stores. Archived-body clear runs on + `ibd-confirm-write` after the in-flight tip flush joins. On that path, + header range puts wait for the same join. Class A body append still + overlaps the tip flush. A structural reject after that append still + stores the header ranges after the join, so a retry sees the body. + Class A device sync stays on `rbtc-spend-sync` about every 10 minutes. + Class C still flushes once per confirm batch. diff --git a/crates/rbitcoin-esplora/tests/handlers/pure_helper_tests.rs b/crates/rbitcoin-esplora/tests/handlers/pure_helper_tests.rs index d99a3d6819..4bb26af2ac 100644 --- a/crates/rbitcoin-esplora/tests/handlers/pure_helper_tests.rs +++ b/crates/rbitcoin-esplora/tests/handlers/pure_helper_tests.rs @@ -116,11 +116,20 @@ fn outspend_json_spent_includes_vin() { let view = q.pin_chain_view().unwrap().expect("tip"); q.store() .header_txs - .put_range(view.header_fk, create_fk, 2) + .put_range( + &rbitcoin_store::ClassCWrite::for_test(), + view.header_fk, + create_fk, + 2, + ) .unwrap(); q.store() .strong_tx - .set_strong(spend_fk, view.header_fk) + .set_strong( + &rbitcoin_store::ClassCWrite::for_test(), + spend_fk, + view.header_fk, + ) .unwrap(); q.store().rebuild_height_fence().unwrap(); let v = outspend_json(&q, None, &[0xcb; 32], 0, Some(&view)).unwrap(); diff --git a/crates/rbitcoin-ibd/src/ibd/confirm/mod.rs b/crates/rbitcoin-ibd/src/ibd/confirm/mod.rs index 7645e4b4ff..b7cdd64ad5 100644 --- a/crates/rbitcoin-ibd/src/ibd/confirm/mod.rs +++ b/crates/rbitcoin-ibd/src/ibd/confirm/mod.rs @@ -1788,16 +1788,25 @@ pub fn stamp_reject_operator_msg(err: &str, stats: &rbitcoin_query::ConfirmStats /// `on_extra` runs only for additionally drained parts (queue depth). Non-contig /// leftover is returned for the next write iteration (ordered scripts should /// never hit that path). +/// Archived-body clear waiting on the write thread. Not pushed back onto the +/// channel: `std` mpsc has no unread. +#[doc(hidden)] +pub struct ArchivedBodyClear { + pub hash: [u8; 32], + pub reply: std::sync::mpsc::Sender>, +} + #[doc(hidden)] pub fn drain_script_ok_write_queue( first: rbitcoin_query::ScriptOkBatch, - rx: &std::sync::mpsc::Receiver, + rx: &std::sync::mpsc::Receiver, max_parts: usize, mut on_extra: impl FnMut(&rbitcoin_query::ScriptOkBatch), ) -> ( rbitcoin_query::ScriptOkBatch, usize, Option, + Option, ) { let mut batch = first; let mut parts = 1usize; @@ -1807,7 +1816,10 @@ pub fn drain_script_ok_write_queue( break; } match rx.try_recv() { - Ok(more) => { + Ok(rbitcoin_query::ConfirmWriteMsg::ClearArchived { hash, reply }) => { + return (batch, parts, None, Some(ArchivedBodyClear { hash, reply })); + } + Ok(rbitcoin_query::ConfirmWriteMsg::Batch(more)) => { on_extra(&more); match batch.append_contiguous(more) { Ok(()) => { @@ -1827,7 +1839,7 @@ pub fn drain_script_ok_write_queue( leftover.len() ); } - return (batch, parts, Some(leftover)); + return (batch, parts, Some(leftover), None); } } } @@ -1835,7 +1847,23 @@ pub fn drain_script_ok_write_queue( Err(std::sync::mpsc::TryRecvError::Disconnected) => break, } } - (batch, parts, None) + (batch, parts, None, None) +} + +fn finish_archived_clear(query: &rbitcoin_query::Query, clear: ArchivedBodyClear) { + let result = (|| { + let _commit = query.lock_tip_commit()?; + query.clear_archived_body_here(&clear.hash) + })(); + let _ = clear.reply.send(result); +} + +struct ConfirmWriteDetach<'a>(&'a rbitcoin_query::Query); + +impl Drop for ConfirmWriteDetach<'_> { + fn drop(&mut self) { + self.0.detach_confirm_write(); + } } /// OS-thread occupancy for the confirm pipeline (lookup / load / scripts / write). @@ -1972,7 +2000,8 @@ pub fn spawn_confirm_engine( type ScriptsIn = (rbitcoin_query::LoadedBatch, u64); let (mat_tx, mat_rx) = std::sync::mpsc::sync_channel::(caps.script); let (write_tx, write_rx) = - std::sync::mpsc::sync_channel::(caps.write); + std::sync::mpsc::sync_channel::(caps.write); + hub.query.attach_confirm_write(write_tx.clone()); let (load_tx, load_rx) = std::sync::mpsc::sync_channel::(caps.load); // A write fault that does not re-arm lookup: plan drops reserved fks + // last_loaded so re-lookup after Class A partial commit does not drift @@ -1994,24 +2023,45 @@ pub fn spawn_confirm_engine( move || { info!("ibd: confirm write on dedicated OS thread"); let stats = hub_wb.query.confirm_stats_arc(); + let _detach = ConfirmWriteDetach(&hub_wb.query); // Non-contig leftover already note_write_recv'd; write it next iter. let mut leftover: Option = None; + let mut pending_clear: Option = None; loop { + if let Some(clear) = pending_clear.take() { + finish_archived_clear(&hub_wb.query, clear); + continue; + } let t_recv = Instant::now(); let first = match leftover.take() { Some(b) => b, - None => match write_rx.recv() { - Ok(b) => { + None => match write_rx.recv_timeout(Duration::from_millis(200)) { + Ok(rbitcoin_query::ConfirmWriteMsg::Batch(b)) => { let n = b.len(); let wire = b.approx_wire_bytes(); let parents = b.parent_count(); q_wb.note_write_recv(n, wire, parents); b } - Err(_) => break, + Ok(rbitcoin_query::ConfirmWriteMsg::ClearArchived { hash, reply }) => { + finish_archived_clear( + &hub_wb.query, + ArchivedBodyClear { hash, reply }, + ); + continue; + } + // Query keeps a sender so clear can arrive. That sender + // also keeps `recv` from seeing disconnect at shutdown. + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + if feed_wb.stopped() || hub_wb.query.confirm_cancelled() { + break; + } + continue; + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, }, }; - let (batch, parts, next_left) = drain_script_ok_write_queue( + let (batch, parts, next_left, clear) = drain_script_ok_write_queue( first, &write_rx, write_drain_max_parts(write_queue_cap()), @@ -2023,6 +2073,7 @@ pub fn spawn_confirm_engine( }, ); leftover = next_left; + pending_clear = clear; confirm_thr_stats::add_write_recv_wait(&stats, t_recv.elapsed()); if feed_wb.stopped() || hub_wb.query.confirm_cancelled() { break; @@ -2253,7 +2304,10 @@ pub fn spawn_confirm_engine( } info!("ibd: confirm write exit"); }) - .expect("spawn ibd-confirm-write"); + .unwrap_or_else(|e| { + hub.query.detach_confirm_write(); + panic!("spawn ibd-confirm-write: {e}"); + }); let hub_sc = Arc::clone(&hub); let feed_sc = Arc::clone(&feed); @@ -2293,7 +2347,10 @@ pub fn spawn_confirm_engine( let ww = outcome.batch.approx_wire_bytes(); let parents = outcome.batch.parent_count(); let t_send = Instant::now(); - if write_tx.send(outcome.batch).is_err() { + if write_tx + .send(rbitcoin_query::ConfirmWriteMsg::Batch(outcome.batch)) + .is_err() + { info!("ibd: confirm write channel closed"); return false; } diff --git a/crates/rbitcoin-ibd/tests/ibd/confirm.rs b/crates/rbitcoin-ibd/tests/ibd/confirm.rs index c375cac8a6..e52eecc271 100644 --- a/crates/rbitcoin-ibd/tests/ibd/confirm.rs +++ b/crates/rbitcoin-ibd/tests/ibd/confirm.rs @@ -2390,7 +2390,12 @@ fn uncover_parent_fence(hub: &rbitcoin_net::chain::ChainHub) { let header_fk = rbitcoin_primitives::Fk(first); store .header_txs - .put_range(header_fk, rbitcoin_primitives::Fk(first), 1) + .put_range( + &rbitcoin_store::ClassCWrite::for_test(), + header_fk, + rbitcoin_primitives::Fk(first), + 1, + ) .unwrap(); store .height_fence_extend(rbitcoin_primitives::Height(50), header_fk) diff --git a/crates/rbitcoin-net/tests/chain.rs b/crates/rbitcoin-net/tests/chain.rs index 7f437724b2..a991d3794d 100644 --- a/crates/rbitcoin-net/tests/chain.rs +++ b/crates/rbitcoin-net/tests/chain.rs @@ -2191,7 +2191,11 @@ fn chain_work_error_leaves_prefix_empty() { hub.query .store() .confirmed - .set(rbitcoin_primitives::Height(1), Fk(9_000)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(1), + Fk(9_000), + ) .unwrap(); assert!(hub.chain_work().is_err()); assert_eq!( @@ -4228,11 +4232,15 @@ fn check_block_proposal_reads_medians_from_confirmed_headers() { let missing = "missing confirmed header"; let confirmed = &hub.query.store().confirmed; - confirmed.set_raw_for_test(Height(11), 0).unwrap(); + confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(11), 0) + .unwrap(); let err = hub.check_block_proposal(&met).unwrap_err(); assert!(err.contains(missing), "coin median: {err}"); assert_eq!(hub.check_block_proposal(&empty), Ok(0)); - confirmed.set_raw_for_test(Height(110), 0).unwrap(); + confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(110), 0) + .unwrap(); hub.query.store().rebuild_height_fence().unwrap(); let err = hub.check_block_proposal(&empty).unwrap_err(); assert!(err.contains(missing), "tip median: {err}"); diff --git a/crates/rbitcoin-net/tests/versionbits_warn.rs b/crates/rbitcoin-net/tests/versionbits_warn.rs index e89ca31d41..d625364ab1 100644 --- a/crates/rbitcoin-net/tests/versionbits_warn.rs +++ b/crates/rbitcoin-net/tests/versionbits_warn.rs @@ -61,7 +61,10 @@ fn put_chain(q: &Query, n: u32, signal: impl Fn(u32) -> bool, salt: u8) { weight: 0, }; prev_fk = q.put_header(&rec).unwrap(); - q.store().confirmed.set(Height(h), prev_fk).unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), prev_fk) + .unwrap(); prev_hash = hash; } q.store().rebuild_height_fence().unwrap(); diff --git a/crates/rbitcoin-query/src/archive.rs b/crates/rbitcoin-query/src/archive.rs index 8c4d8c975a..e7fa6ce5b3 100644 --- a/crates/rbitcoin-query/src/archive.rs +++ b/crates/rbitcoin-query/src/archive.rs @@ -1373,6 +1373,9 @@ pub fn seqsigwit_rows_for_prune( Ok(rows) } +/// `committed`, Class A locators, and header ranges still to store. +pub type ArchiveHeadParents = (bool, Vec, Vec<(Fk, Fk, u32)>); + impl Query { /// Class A plan + commit from wire blocks. Does not set tip. pub fn archive_class_a_from_wire(&self, items: &[WirePlanNeed<'_>]) -> Result<(), QueryError> { @@ -1715,23 +1718,36 @@ impl Query { &self, plan: ArchiveWritePlan, ) -> Result<(bool, Vec), QueryError> { - self.archive_commit_plan_defer_head_parents(plan) + let (committed, loc, ranges) = self.archive_commit_plan_defer_head_parents(plan)?; + if !ranges.is_empty() { + let t = std::time::Instant::now(); + let write = rbitcoin_store::ClassCWrite::new(); + self.store.header_txs.put_ranges_batch(&write, &ranges)?; + self.confirm_stats() + .note_write_commit(0, 0, 0, 0, 0, t.elapsed().as_nanos() as u64, 0); + } + Ok((committed, loc)) } /// Class A commit. Fees and sizes were recorded before this call /// (confirm assemble, or [`stamp_wire_plan_txstat`] on the Class A wire /// planner). A non-empty plan whose vectors do not match `packed` is - /// corrupt. This function does not walk prevouts. + /// corrupt. This function does not walk prevouts. It does not store + /// header ranges: the returned list is Class C. [`Self::archive_commit_plan_defer_head`] + /// stores it immediately. The confirm-write path stores it after + /// `tip_commit` joins the in-flight tip flush, including when a later + /// phase of that write rejects the batch. `header_txs` is what a retry + /// uses to skip a second body append. pub fn archive_commit_plan_defer_head_parents( &self, mut plan: ArchiveWritePlan, - ) -> Result<(bool, Vec), QueryError> { + ) -> Result { use std::time::Instant; if plan.packed.is_empty() { - return Ok((false, Vec::new())); + return Ok((false, Vec::new(), Vec::new())); } if !plan.retain_headers_needing_body(|hfk| self.store.header_txs.has_body(hfk))? { - return Ok((false, Vec::new())); + return Ok((false, Vec::new(), Vec::new())); } let edge_rows = commit_input_edge_rows(&plan)?; let seqsigwit_est = seqsigwit_append_est(&plan); @@ -1803,14 +1819,7 @@ impl Query { } let head_ns = t.elapsed().as_nanos() as u64; - let t = Instant::now(); - if !plan.per_header_ranges.is_empty() { - self.store - .header_txs - .put_ranges_batch(&plan.per_header_ranges)?; - } - let htxs_ns = t.elapsed().as_nanos() as u64; - + let ranges = std::mem::take(&mut plan.per_header_ranges); let total_ns = t0.elapsed().as_nanos() as u64; self.confirm_stats().note_write_commit( total_ns, @@ -1818,10 +1827,10 @@ impl Query { body_ns, head_ns, 0, - htxs_ns, + 0, n_blocks.max(1), ); - Ok((true, loc)) + Ok((true, loc, ranges)) } /// Drain write-behind `tx.head` inserts (page-grouped). diff --git a/crates/rbitcoin-query/src/confirm_run/mod.rs b/crates/rbitcoin-query/src/confirm_run/mod.rs index 781698ec87..c9d923c6b7 100644 --- a/crates/rbitcoin-query/src/confirm_run/mod.rs +++ b/crates/rbitcoin-query/src/confirm_run/mod.rs @@ -213,6 +213,22 @@ pub struct ScriptOkBatch { pub trust_script_warns: Vec<(u32, rbitcoin_consensus::ConsensusError)>, } +/// One item on the confirm-write channel. +/// +/// [`ConfirmWriteMsg::ClearArchived`] is the archived-body clear. The write +/// thread joins the in-flight tip flush, then clears. IBD orchestration does +/// not store that image. +/// +/// `Batch` stays unboxed: this channel already moved the batch by value. +#[allow(clippy::large_enum_variant)] +pub enum ConfirmWriteMsg { + Batch(ScriptOkBatch), + ClearArchived { + hash: [u8; 32], + reply: std::sync::mpsc::Sender>, + }, +} + /// Outcome of load: batch ready for scripts + pure work wall. pub struct ConfirmLoadOutcome { pub batch: LoadedBatch, diff --git a/crates/rbitcoin-query/src/confirm_run/phases.rs b/crates/rbitcoin-query/src/confirm_run/phases.rs index 0042a7c65e..f3979d74dc 100644 --- a/crates/rbitcoin-query/src/confirm_run/phases.rs +++ b/crates/rbitcoin-query/src/confirm_run/phases.rs @@ -433,6 +433,7 @@ pub fn class_c_commit( query: &Query, prepared: &mut [Prepared], write_create_pins: &FkMap, + header_ranges: &[(rbitcoin_primitives::Fk, rbitcoin_primitives::Fk, u32)], ) -> Result, ConfirmError> { use std::sync::atomic::Ordering as QOrd; @@ -453,7 +454,7 @@ pub fn class_c_commit( Some(write_create_pins) }; let out = query - .confirm_blocks_run_with_create_pins(&items, pins) + .confirm_blocks_run_with_create_pins(&items, pins, header_ranges) .map_err(ConfirmError::from)?; let strong_d = query .confirm_stats() diff --git a/crates/rbitcoin-query/src/confirm_run/write.rs b/crates/rbitcoin-query/src/confirm_run/write.rs index 076db17815..d7ecee5ce5 100644 --- a/crates/rbitcoin-query/src/confirm_run/write.rs +++ b/crates/rbitcoin-query/src/confirm_run/write.rs @@ -144,6 +144,7 @@ pub fn finish_already_committed_write( pub struct ArchivePlanNs { pub pins: FkMap, pub class_a_wave: crate::block_pipeline::ClassAWave, + pub header_ranges: Vec<(rbitcoin_primitives::Fk, rbitcoin_primitives::Fk, u32)>, pub class_a_ns: u64, pub ensure_ns: u64, pub plan_take_ns: u64, @@ -158,6 +159,7 @@ pub fn apply_archive_plan( let mut ns = ArchivePlanNs { pins: FkMap::default(), class_a_wave: crate::block_pipeline::ClassAWave::default(), + header_ranges: Vec::new(), class_a_ns: 0, ensure_ns: 0, plan_take_ns: 0, @@ -179,13 +181,14 @@ pub fn apply_archive_plan( let packed_pins: Vec = std::mem::take(&mut plan.batch_pin); ns.plan_take_ns = t_take.elapsed().as_nanos() as u64; let t_ca = Instant::now(); - let (committed, loc) = { + let (committed, loc, header_ranges) = { let _gate = hold_spindle(query); spindle_mark("class-a"); query .archive_commit_plan_defer_head_parents(plan) .map_err(ConfirmError::from)? }; + ns.header_ranges = header_ranges; ns.class_a_ns = t_ca.elapsed().as_nanos() as u64; if !committed { return Ok(ns); @@ -201,14 +204,19 @@ pub fn apply_archive_plan( ns.create_map_ns = t_map.elapsed().as_nanos() as u64; } let t_ens = Instant::now(); - fill_planned_create_layout_after_commit( + if let Err(e) = fill_planned_create_layout_after_commit( query, &mut batch.batch_parents, &planned_fks, &loc, &packed_pins, &batch.prepared, - )?; + ) { + // Class A is already appended. `header_txs` is what a retry uses to + // skip that append. Store it even though this phase failed. + publish_archived_header_ranges(query, &ns.header_ranges)?; + return Err(e); + } ns.ensure_ns = t_ens.elapsed().as_nanos() as u64; if let Some(last) = batch.prepared.last() { query.set_class_a_hi(Some(last.height.0)); @@ -299,13 +307,21 @@ fn write_phase_inner( let ArchivePlanNs { pins: write_create_pins, class_a_wave, + header_ranges, class_a_ns, mut ensure_ns, plan_take_ns, create_map_ns, } = apply_archive_plan(query, batch)?; let t_ens = Instant::now(); - let abs_jobs = super::collect_spend_abs_after_fill(&batch.batch_parents, &batch.prepared)?; + let abs_jobs = match super::collect_spend_abs_after_fill(&batch.batch_parents, &batch.prepared) + { + Ok(jobs) => jobs, + Err(e) => { + publish_archived_header_ranges(query, &header_ranges)?; + return Err(e); + } + }; ensure_ns = ensure_ns.saturating_add(t_ens.elapsed().as_nanos() as u64); if class_a_ns > 0 { crate::note_confirm(&query.confirm_stats().class_a_ns, class_a_ns); @@ -360,7 +376,12 @@ fn write_phase_inner( .write_tip_flush_wait_ns .load(Ordering::Relaxed); let t_cc = Instant::now(); - let out = class_c_commit(query, &mut batch.prepared, &write_create_pins)?; + let out = class_c_commit( + query, + &mut batch.prepared, + &write_create_pins, + &header_ranges, + )?; super::phases::remember_connected_bip34(query, params, &batch.prepared); let class_c_wall_ns = t_cc.elapsed().as_nanos() as u64; let class_c_ns = query @@ -428,7 +449,16 @@ fn write_phase_inner( if drain_res.is_err() { query.store().txs.head_note_pending(&restore); } - let (out, n_blocks, structural_ns, struct_ph, class_c_ns, spend_ann_ns) = overlap?; + let (out, n_blocks, structural_ns, struct_ph, class_c_ns, spend_ann_ns) = match overlap { + Ok(v) => v, + Err(e) => { + // Success stores these inside `class_c_commit` after the same join. + // A reject before that still has to leave `header_txs` behind, or + // the next batch mixes a connected block with this partial append. + publish_archived_header_ranges(query, &header_ranges)?; + return Err(e); + } + }; drain_res.map_err(ConfirmError::from)?; if let Some(fk) = drain_max_fk { query.note_head_drain_fk(fk); @@ -456,6 +486,28 @@ fn write_phase_inner( Ok(out) } +/// Store header ranges after the in-flight tip flush joins. +/// +/// The success path does this inside `class_c_commit`. This is the same put +/// for a write that appended Class A and then rejected, so `has_body` matches +/// that append. It does not flush: the next successful commit still flushes +/// Class C once. +fn publish_archived_header_ranges( + query: &Query, + ranges: &[(rbitcoin_primitives::Fk, rbitcoin_primitives::Fk, u32)], +) -> Result<(), ConfirmError> { + if ranges.is_empty() { + return Ok(()); + } + let _commit = query.lock_tip_commit()?; + let write = rbitcoin_store::ClassCWrite::new(); + query + .store + .header_txs + .put_ranges_batch(&write, ranges) + .map_err(ConfirmError::from) +} + /// Replay spend annotate + `tx.head` drain after Class C already committed. /// /// Session-fault retry must not skip this: `height_of_hash` matching is not diff --git a/crates/rbitcoin-query/src/connect.rs b/crates/rbitcoin-query/src/connect.rs index 87925b9e5e..b206dbf3c6 100644 --- a/crates/rbitcoin-query/src/connect.rs +++ b/crates/rbitcoin-query/src/connect.rs @@ -463,7 +463,7 @@ impl Query { /// Class C-only path. [`Self::confirm_blocks_run`] itself does not annotate /// (the wire write stage would double-walk). pub fn confirm_blocks_run(&self, items: &[ConfirmPrepared]) -> Result, QueryError> { - self.confirm_blocks_run_with_create_pins(items, None) + self.confirm_blocks_run_with_create_pins(items, None, &[]) } /// The run must be contiguous, start at tip+1 (genesis on an empty @@ -520,6 +520,7 @@ impl Query { &self, items: &[ConfirmPrepared], create_pins: Option<&crate::FkMap>, + header_ranges: &[(Fk, Fk, u32)], ) -> Result, QueryError> { if items.is_empty() { return Ok(Vec::new()); @@ -530,6 +531,13 @@ impl Query { return Ok(vec![fk]); } + let write = rbitcoin_store::ClassCWrite::new(); + if !header_ranges.is_empty() { + self.store + .header_txs + .put_ranges_batch(&write, header_ranges)?; + } + let t_strong = std::time::Instant::now(); let mut confirmed_pairs = Vec::with_capacity(items.len()); let mut out = Vec::with_capacity(items.len()); @@ -541,6 +549,7 @@ impl Query { if contiguous { if let Some(&first) = item.tx_fks.first() { self.store.strong_tx.set_strong_range( + &write, first, item.tx_fks.len() as u32, item.header_fk, @@ -548,7 +557,9 @@ impl Query { } } else { for &tx_fk in &item.tx_fks { - self.store.strong_tx.set_strong(tx_fk, item.header_fk)?; + self.store + .strong_tx + .set_strong(&write, tx_fk, item.header_fk)?; } } confirmed_pairs.push((item.height, item.header_fk)); @@ -566,7 +577,7 @@ impl Query { self.store .height_fence_extend(item.height, item.header_fk)?; } - self.store.confirmed.set_many(&confirmed_pairs)?; + self.store.confirmed.set_many(&write, &confirmed_pairs)?; // Same commit as `confirmed[]`, before the flush. `tip_height` is // already the new tip; the orchestration loop reads this snapshot. if !items.is_empty() { @@ -1163,7 +1174,8 @@ impl Query { } self.store.scripthash.unlink_creates_batch(&groups)?; - self.store.confirmed.disconnect_tip(height)?; + let write = rbitcoin_store::ClassCWrite::new(); + self.store.confirmed.disconnect_tip(&write, height)?; self.store.height_fence_pop_tip(height); self.note_disconnect_height(height.0); let _ = self.on_load_pack(); @@ -1183,7 +1195,7 @@ impl Query { log_disconnect_tip(height.0, &hash, tx_fks.len()); for &tx_fk in &tx_fks { - self.store.strong_tx.set_unstrong(tx_fk)?; + self.store.strong_tx.set_unstrong(&write, tx_fk)?; } self.store.flush_class_c_after_disconnect_tip()?; diff --git a/crates/rbitcoin-query/src/connect_facade.rs b/crates/rbitcoin-query/src/connect_facade.rs index 7ffaef97c2..6abd5af803 100644 --- a/crates/rbitcoin-query/src/connect_facade.rs +++ b/crates/rbitcoin-query/src/connect_facade.rs @@ -23,8 +23,8 @@ pub use crate::confirm_run::{ confirm_wire_run, confirm_wire_run_preverified, confirm_write_phase, confirm_write_phase_or_return, drive_script_waves_with, finish_post_commit_hashes, parent_missing_from_chain, replay_spend_annotations, take_wave_items_for_load, BqResolveWave, - ConfirmLoadOutcome, ConfirmScriptOutcome, LoadedBatch, PlanStampOutcome, ScriptOkBatch, - ScriptPreverified, WireBlockIn, WireLoadPipeline, BQ_RESOLVE_WAVE_MAX_BLOCKS, + ConfirmLoadOutcome, ConfirmScriptOutcome, ConfirmWriteMsg, LoadedBatch, PlanStampOutcome, + ScriptOkBatch, ScriptPreverified, WireBlockIn, WireLoadPipeline, BQ_RESOLVE_WAVE_MAX_BLOCKS, BQ_RESOLVE_WAVE_MAX_INPUTS, }; diff --git a/crates/rbitcoin-query/src/lib.rs b/crates/rbitcoin-query/src/lib.rs index 9a0e235cd5..08b33679f2 100644 --- a/crates/rbitcoin-query/src/lib.rs +++ b/crates/rbitcoin-query/src/lib.rs @@ -497,6 +497,9 @@ pub struct Query { /// starting it) / disconnect flush. Rewind tries it and retries; it does not /// disconnect mid-flush. pub tip_commit: Mutex<()>, + /// Confirm-write channel, installed while `ibd-confirm-write` is alive. + /// Archived-body clear sends here instead of storing Class C itself. + pub confirm_write: Mutex>>, /// IBD tip barrier on `rbtc-tip-flush`, or the failure of one. The next /// `tip_commit` taker joins it. pub tip_flush: Mutex, @@ -665,6 +668,7 @@ impl Query { sh_runs_published: AtomicUsize::new(0), owned_sizes_published: arc_swap::ArcSwap::from_pointee(ProcessOwnedSizes::default()), tip_commit: Mutex::new(()), + confirm_write: Mutex::new(None), tip_flush: Mutex::new(connect::TipFlush::Idle), tip_flush_worker: connect::TipFlushWorker::default(), #[cfg(debug_assertions)] @@ -2437,11 +2441,46 @@ impl Query { /// /// Use when reconstruct fails header checks (merkle mismatch): peer re-getdata /// can supply a good body for the same block hash. + pub fn attach_confirm_write(&self, tx: std::sync::mpsc::SyncSender) { + *self.confirm_write.lock().unwrap_or_else(|e| e.into_inner()) = Some(tx); + } + + /// Drop the sender when `ibd-confirm-write` exits so a later clear does + /// not wait on a thread that is gone. + pub fn detach_confirm_write(&self) { + *self.confirm_write.lock().unwrap_or_else(|e| e.into_inner()) = None; + } + pub fn clear_archived_body(&self, hash: &[u8; 32]) -> Result { + let tx = self + .confirm_write + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + if let Some(tx) = tx { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + tx.send(crate::ConfirmWriteMsg::ClearArchived { + hash: *hash, + reply: reply_tx, + }) + .map_err(|_| StoreError::Cancelled("confirm write stopped"))?; + return reply_rx + .recv() + .map_err(|_| StoreError::Cancelled("confirm write stopped"))?; + } + self.clear_archived_body_here(hash) + } + + /// Clear on the calling thread. The confirm-write thread calls this after + /// it joins the tip flush. With no sender installed (unit tests, before + /// the engine) [`Self::clear_archived_body`] calls it inline. + #[doc(hidden)] + pub fn clear_archived_body_here(&self, hash: &[u8; 32]) -> Result { let Some((fk, _)) = self.get_header_by_hash(hash)? else { return Ok(false); }; - let cleared = self.store.header_txs.clear_body(fk)?; + let write = rbitcoin_store::ClassCWrite::new(); + let cleared = self.store.header_txs.clear_body(&write, fk)?; if cleared { self.store.header_txs.flush()?; } diff --git a/crates/rbitcoin-query/tests/archive.rs b/crates/rbitcoin-query/tests/archive.rs index d218df07cb..fb1ea19d31 100644 --- a/crates/rbitcoin-query/tests/archive.rs +++ b/crates/rbitcoin-query/tests/archive.rs @@ -494,11 +494,19 @@ fn load_gcs_header_plans_from_store_tip() { ); q.store() .confirmed - .set(rbitcoin_primitives::Height(0), Fk(1)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(0), + Fk(1), + ) .unwrap(); q.store() .confirmed - .set(rbitcoin_primitives::Height(1), Fk(2)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(1), + Fk(2), + ) .unwrap(); q.on_load_pack().unwrap(); assert!( diff --git a/crates/rbitcoin-query/tests/bip68_store_tests.rs b/crates/rbitcoin-query/tests/bip68_store_tests.rs index 276b38e038..6132ce3253 100644 --- a/crates/rbitcoin-query/tests/bip68_store_tests.rs +++ b/crates/rbitcoin-query/tests/bip68_store_tests.rs @@ -40,7 +40,11 @@ fn bip68_height_zero_time_lock_uses_the_median() { query .store() .confirmed - .set(rbitcoin_primitives::Height(0), header_fk) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(0), + header_fk, + ) .unwrap(); query.store().rebuild_height_fence().unwrap(); let params = Box::leak(Box::new(rbitcoin_consensus::ChainParams::regtest())); diff --git a/crates/rbitcoin-query/tests/confirm_run/bq_resolve.rs b/crates/rbitcoin-query/tests/confirm_run/bq_resolve.rs index 76f64c0d06..b04b604777 100644 --- a/crates/rbitcoin-query/tests/confirm_run/bq_resolve.rs +++ b/crates/rbitcoin-query/tests/confirm_run/bq_resolve.rs @@ -779,7 +779,11 @@ fn stamp_uses_inflight_when_confirmed_tip_leads_fence() { q.store() .confirmed - .set(Height(1), rbitcoin_primitives::Fk(2)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(1), + rbitcoin_primitives::Fk(2), + ) .unwrap(); assert_eq!(q.tip_height(), Some(Height(1))); assert_eq!( @@ -809,7 +813,10 @@ fn stamp_uses_inflight_when_confirmed_tip_leads_fence() { ); // Dummy height-1 confirmed row was only to tear tip vs fence; stamp // still connects at tip+1 from genesis. - q.store().confirmed.disconnect_tip(Height(1)).unwrap(); + q.store() + .confirmed + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(1)) + .unwrap(); assert_eq!(q.tip_height(), Some(Height(0))); let b1 = mine_with_txs( diff --git a/crates/rbitcoin-query/tests/confirm_run/write_idempotent_tests.rs b/crates/rbitcoin-query/tests/confirm_run/write_idempotent_tests.rs index 3330381a7b..1fc722337e 100644 --- a/crates/rbitcoin-query/tests/confirm_run/write_idempotent_tests.rs +++ b/crates/rbitcoin-query/tests/confirm_run/write_idempotent_tests.rs @@ -1589,8 +1589,14 @@ fn structural_scratch_second_block_does_not_replay_first_slots() { Vec::new(), ); bp.set_spent_range_only(fks[0], loc[0].spent); - q.store().header_txs.put_range(Fk(100), fks[0], 1).unwrap(); - q.store().confirmed.set(Height(0), Fk(100)).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(100), fks[0], 1) + .unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(100)) + .unwrap(); q.store().rebuild_height_fence().unwrap(); let coinbase = Transaction { version: TxVersion::ONE, @@ -3289,8 +3295,14 @@ fn structural_run_clears_reused_scratch_between_batches() { Vec::new(), ); bp.set_spent_range_only(fks[0], loc[0].spent); - q.store().header_txs.put_range(Fk(100), fks[0], 1).unwrap(); - q.store().confirmed.set(Height(0), Fk(100)).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(100), fks[0], 1) + .unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(100)) + .unwrap(); q.store().rebuild_height_fence().unwrap(); let coinbase = Transaction { diff --git a/crates/rbitcoin-query/tests/root.rs b/crates/rbitcoin-query/tests/root.rs index ece24189c7..8c2ed4d91b 100644 --- a/crates/rbitcoin-query/tests/root.rs +++ b/crates/rbitcoin-query/tests/root.rs @@ -25,10 +25,16 @@ fn query_open_clears_strong_above_tip() { h0.nonce, ); let hfk = q.put_header(&h0).unwrap(); - q.store().confirmed.set(Height(0), hfk).unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), hfk) + .unwrap(); q.store().rebuild_height_fence().unwrap(); let leftover = Fk(99); - q.store().strong_tx.set_strong(leftover, hfk).unwrap(); + q.store() + .strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), leftover, hfk) + .unwrap(); q.store().flush_class_c_tip().unwrap(); assert_eq!(q.tip_height(), Some(Height(0))); assert!(q.store().strong_tx.is_strong(leftover).unwrap()); @@ -61,7 +67,10 @@ fn uring_recover_credit_gap() { fn uring_recover_credit_once_per_window_does_not_repair() { let (_d, q) = temp_query("uring-recover"); let leftover = Fk(99); - q.store().strong_tx.set_strong(leftover, Fk(1)).unwrap(); + q.store() + .strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), leftover, Fk(1)) + .unwrap(); q.store().flush_class_c_tip().unwrap(); assert!(q.store().strong_tx.is_strong(leftover).unwrap()); assert_eq!(q.uring_recover("test"), UringRecover::Recovered); @@ -69,7 +78,10 @@ fn uring_recover_credit_once_per_window_does_not_repair() { q.store().strong_tx.is_strong(leftover).unwrap(), "in-process recover must not mutate Class C; leftover strong waits for open repair" ); - q.store().strong_tx.set_strong(leftover, Fk(1)).unwrap(); + q.store() + .strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), leftover, Fk(1)) + .unwrap(); q.store().flush_class_c_tip().unwrap(); assert_eq!(q.uring_recover("again"), UringRecover::Exhausted); } @@ -109,7 +121,10 @@ fn held_pread_fault_string_exhausts_lookup_recover_at_same_tip() { #[test] fn published_tip_is_in_the_height_index() { let (dir, q) = temp_query("pub-tip-index"); - q.store().confirmed.set_raw_for_test(Height(0), 1).unwrap(); + q.store() + .confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(0), 1) + .unwrap(); let hash = [0xab; 32]; q.publish_confirmed_tip(0, hash); let before = rbitcoin_query::chain_view::header_lookup_count(); @@ -125,7 +140,10 @@ fn published_tip_is_in_the_height_index() { #[test] fn extend_refuses_a_height_the_store_already_left() { let (dir, q) = temp_query("idx-left-height"); - q.store().confirmed.set_raw_for_test(Height(0), 1).unwrap(); + q.store() + .confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(0), 1) + .unwrap(); q.publish_confirmed_tip(0, [0x11; 32]); let stale = [0x22; 32]; assert!( @@ -133,7 +151,10 @@ fn extend_refuses_a_height_the_store_already_left() { "a reader must not index a height the store has already left" ); assert_eq!(q.height_of_hash(&stale).unwrap(), None); - q.store().confirmed.set_raw_for_test(Height(1), 2).unwrap(); + q.store() + .confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(1), 2) + .unwrap(); assert!(q.extend_height_index(&[(stale, 1)], 1, Some(0), false)); assert_eq!(q.height_of_hash(&stale).unwrap(), Some(Height(1))); let _ = std::fs::remove_dir_all(&dir); @@ -184,7 +205,10 @@ fn height_index_catchup_does_not_hold_the_lock_across_header_reads() { #[test] fn seed_published_tip_refuses_a_tip_with_no_header() { let (dir, q) = temp_query("seed-no-header"); - q.store().confirmed.set_raw_for_test(Height(0), 0).unwrap(); + q.store() + .confirmed + .set_raw_for_test(&rbitcoin_store::ClassCWrite::for_test(), Height(0), 0) + .unwrap(); assert_eq!(q.tip_height(), Some(Height(0))); let err = q.seed_published_tip().unwrap_err(); assert!( @@ -606,7 +630,7 @@ fn height_of_hash_stale_snapshot_after_confirmed_shrink_is_none() { q.invalidate_height_by_hash_index(); q.store .confirmed - .disconnect_tip(Height(4)) + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(4)) .expect("shrink confirmed without map ensure"); q.store.height_fence_pop_tip(Height(4)); assert_eq!(q.tip_height(), Some(Height(3))); @@ -619,7 +643,10 @@ fn height_of_hash_stale_snapshot_after_confirmed_shrink_is_none() { assert_height(&q, &hashes[3], 3); while let Some(h) = q.tip_height() { - q.store.confirmed.disconnect_tip(h).unwrap(); + q.store + .confirmed + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), h) + .unwrap(); q.store.height_fence_pop_tip(h); } q.ensure_height_by_hash_index(Height(5)) @@ -881,7 +908,11 @@ fn recover_sh_writebehind_fails_open_on_interior_missing_header_txs() { let (h2, t2) = coinbase_block(2, hfk1, Some(hash1)); q.commit_class_a_only(&h2, &[t2]).unwrap(); q.confirm_block(Height(2), &h2.hash).unwrap(); - assert!(q.store().header_txs.clear_body(hfk1).unwrap()); + assert!(q + .store() + .header_txs + .clear_body(&rbitcoin_store::ClassCWrite::for_test(), hfk1) + .unwrap()); let err = q.recover_sh_writebehind().expect_err("interior hole"); let msg = format!("{err}"); assert!( @@ -891,6 +922,42 @@ fn recover_sh_writebehind_fails_open_on_interior_missing_header_txs() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn clear_archived_body_waits_for_the_confirm_write_channel() { + let (dir, q) = temp_query("clear-on-write-channel"); + let (h0, t0) = coinbase_block(0, Fk::NULL, None); + let hash = h0.hash; + q.connect_block(Height(0), &h0, &[t0]).unwrap(); + let hfk = q.tip_header_fk().unwrap().unwrap(); + assert!(q.store().header_txs.has_body(hfk).unwrap()); + let (tx, rx) = std::sync::mpsc::sync_channel(1); + q.attach_confirm_write(tx); + let cleared = std::thread::scope(|scope| { + let worker = scope.spawn(|| q.clear_archived_body(&hash).unwrap()); + let msg = rx + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("clear is a write-channel item"); + assert!( + q.store().header_txs.has_body(hfk).unwrap(), + "orchestration must not store Class C while the write thread has the channel" + ); + let rbitcoin_query::ConfirmWriteMsg::ClearArchived { hash: got, reply } = msg else { + panic!("clear must not arrive as a script batch"); + }; + assert_eq!(got, hash); + let result = { + let _commit = q.lock_tip_commit().unwrap(); + q.clear_archived_body_here(&got).unwrap() + }; + reply.send(Ok(result)).unwrap(); + worker.join().unwrap() + }); + assert!(cleared); + assert!(!q.store().header_txs.has_body(hfk).unwrap()); + q.detach_confirm_write(); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn recover_sh_writebehind_skips_bodyless_structural_tip() { let (dir, q) = temp_query("sh-wb-recover-tip-nobody"); @@ -905,7 +972,11 @@ fn recover_sh_writebehind_skips_bodyless_structural_tip() { let _stolen = q.take_sh_job_for_apply().expect("height-1 job"); q.finish_sh_job(Height(1)); assert_eq!(q.tip_height(), Some(Height(1))); - assert!(q.store().header_txs.clear_body(hfk1).unwrap()); + assert!(q + .store() + .header_txs + .clear_body(&rbitcoin_store::ClassCWrite::for_test(), hfk1) + .unwrap()); q.recover_sh_writebehind() .expect("body-less structural tip must not fail open"); assert_eq!(q.sh_indexed_through_height(), Some(0)); diff --git a/crates/rbitcoin-query/tests/spend_sync.rs b/crates/rbitcoin-query/tests/spend_sync.rs index dd2d279ae2..c898006ab8 100644 --- a/crates/rbitcoin-query/tests/spend_sync.rs +++ b/crates/rbitcoin-query/tests/spend_sync.rs @@ -7,7 +7,10 @@ use rbitcoin_query::spend_sync::SpendSync; #[test] fn shutdown_publishes_the_snapshot() { let (dir, q) = rbitcoin_query::testutil::tiny_query_labeled("spend-sync-stop"); - q.store().confirmed.set(Height(4), Fk(1)).unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(4), Fk(1)) + .unwrap(); q.store().note_spend_snapshot(4); let q = Arc::new(q); let sync = SpendSync::spawn(Arc::clone(&q)); diff --git a/crates/rbitcoin-query/tests/structure_rule_tests.rs b/crates/rbitcoin-query/tests/structure_rule_tests.rs index 91bd64ae78..8ae9e07916 100644 --- a/crates/rbitcoin-query/tests/structure_rule_tests.rs +++ b/crates/rbitcoin-query/tests/structure_rule_tests.rs @@ -473,8 +473,14 @@ fn plant_coinbase_with_outputs( true, ) .unwrap()[0]; - q.store().header_txs.put_range(Fk(1), fk, 1).unwrap(); - q.store().confirmed.set(Height(0), Fk(1)).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), fk, 1) + .unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); q.store().rebuild_height_fence().unwrap(); (path, q, first, fk) } @@ -491,7 +497,11 @@ fn plant_bip34_ancestor(q: &rbitcoin_query::Query, hash: [u8; 32]) { .unwrap(); q.store() .confirmed - .set(Height(main.btc.bip34_height), hfk) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(main.btc.bip34_height), + hfk, + ) .unwrap(); q.clear_bip34_ancestor(); } @@ -2387,10 +2397,19 @@ fn plant_confirmed_spend( true, ) .unwrap()[0]; - q.store().header_txs.put_range(Fk(2), spender, 1).unwrap(); - q.store().confirmed.set(Height(1), Fk(2)).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), spender, 1) + .unwrap(); + q.store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); q.store().rebuild_height_fence().unwrap(); - q.store().strong_tx.set_strong(spender, Fk(2)).unwrap(); + q.store() + .strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), spender, Fk(2)) + .unwrap(); q.store() .put_spend_create(create_fk, 0, spender, 0) .unwrap(); @@ -3395,10 +3414,20 @@ fn same_height_confirmed_spender_is_not_below_its_create() { let store = query.store(); store .confirmed - .set(rbitcoin_primitives::Height(0), Fk(1)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(0), + Fk(1), + ) + .unwrap(); + store + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + store + .strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) .unwrap(); - store.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - store.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); store.rebuild_height_fence().unwrap(); let mut scratch = rbitcoin_query::block_pipeline::StructuralScratch::default(); scratch.height_by_id.insert(9, 0); diff --git a/crates/rbitcoin-rpc/tests/methods/mod.rs b/crates/rbitcoin-rpc/tests/methods/mod.rs index c4ed49847e..c8a9304628 100644 --- a/crates/rbitcoin-rpc/tests/methods/mod.rs +++ b/crates/rbitcoin-rpc/tests/methods/mod.rs @@ -2163,7 +2163,11 @@ fn getrawtransaction_verbose_mid_disconnect_has_no_block_fields() { let cb_txid = blk["tx"][0]["txid"].as_str().unwrap().to_string(); let height = Height(blk["height"].as_u64().unwrap() as u32); // Stop halfway through Query::disconnect_tip: confirmed cleared, fence not. - ctx.query.store().confirmed.disconnect_tip(height).unwrap(); + ctx.query + .store() + .confirmed + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), height) + .unwrap(); let mid = dispatch( &ctx, "getrawtransaction", @@ -2182,7 +2186,11 @@ fn getrawtransaction_verbose_mid_disconnect_has_no_block_fields() { .header_at_height(Height(height.0 - 1)) .unwrap() .unwrap(); - ctx.query.store().confirmed.set(height, other_fk).unwrap(); + ctx.query + .store() + .confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), height, other_fk) + .unwrap(); let replaced = dispatch( &ctx, "getrawtransaction", diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 9fa32fb7d8..953e682518 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -26,10 +26,42 @@ use crate::error::StoreError; use crate::file::{TableFile, FILE_HEADER_LEN}; use rbitcoin_primitives::TableKind; +use std::marker::PhantomData; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Mutex, RwLock}; +/// Capability for the one Class C writer. +/// +/// `!Send` and `!Sync`: a shared [`crate::Store`] cannot carry this onto +/// another thread. Confirm-write, tip-accept, and startup each mint one on +/// the thread that stores. The flush thread reads live slots and does not +/// take this token. +pub struct ClassCWrite { + _thread: PhantomData<*const ()>, +} + +impl Default for ClassCWrite { + fn default() -> Self { + Self::new() + } +} + +impl ClassCWrite { + /// Mint a writer capability on the calling thread. + pub fn new() -> Self { + Self { + _thread: PhantomData, + } + } + + /// Same as [`Self::new`]. Tests are not the confirm-write thread. + #[doc(hidden)] + pub fn for_test() -> Self { + Self::new() + } +} + #[doc(hidden)] pub const ELEM: u64 = 8; @@ -204,7 +236,7 @@ impl ArrayTable { Ok(u64::from_le_bytes(buf)) } - pub fn set(&self, index: u64, value: u64) -> Result<(), StoreError> { + pub fn set(&self, _write: &ClassCWrite, index: u64, value: u64) -> Result<(), StoreError> { let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); self.refuse_while_flushing()?; if let Some(ref mut v) = *guard { @@ -233,7 +265,7 @@ impl ArrayTable { Ok(()) } - pub fn set_many(&self, pairs: &[(u64, u64)]) -> Result<(), StoreError> { + pub fn set_many(&self, _write: &ClassCWrite, pairs: &[(u64, u64)]) -> Result<(), StoreError> { if pairs.is_empty() { return Ok(()); } @@ -277,7 +309,7 @@ impl ArrayTable { Ok(()) } - pub fn truncate(&self, new_len: u64) -> Result<(), StoreError> { + pub fn truncate(&self, _write: &ClassCWrite, new_len: u64) -> Result<(), StoreError> { let len = self.len.load(Ordering::Acquire); if new_len > len { return Err(StoreError::Corrupt("array truncate grows")); diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index 334cdf932a..bab59b9938 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -54,21 +54,35 @@ impl ConfirmedTable { /// no header fk. Production `set` refuses 0. #[cfg(debug_assertions)] #[doc(hidden)] - pub fn set_raw_for_test(&self, height: Height, value: u64) -> Result<(), StoreError> { - self.arr.set(u64::from(height.0), value) + pub fn set_raw_for_test( + &self, + write: &crate::array_table::ClassCWrite, + height: Height, + value: u64, + ) -> Result<(), StoreError> { + self.arr.set(write, u64::from(height.0), value) } - pub fn set(&self, height: Height, header_fk: Fk) -> Result<(), StoreError> { + pub fn set( + &self, + write: &crate::array_table::ClassCWrite, + height: Height, + header_fk: Fk, + ) -> Result<(), StoreError> { if header_fk.is_null() { return Err(StoreError::InvalidFk); } - self.arr.set(u64::from(height.0), header_fk.0) + self.arr.set(write, u64::from(height.0), header_fk.0) } /// Set many height→header_fk pairs (one grow for max height, then bulk writes). /// /// Used by multi-block Class C confirm so tip extension is not N separate grows. - pub fn set_many(&self, pairs: &[(Height, Fk)]) -> Result<(), StoreError> { + pub fn set_many( + &self, + write: &crate::array_table::ClassCWrite, + pairs: &[(Height, Fk)], + ) -> Result<(), StoreError> { if pairs.is_empty() { return Ok(()); } @@ -79,14 +93,17 @@ impl ConfirmedTable { } raw.push((u64::from(h.0), fk.0)); } - self.arr.set_many(&raw) + self.arr.set_many(write, &raw) } /// Drop a trailing run of null header fks (HWM ahead of last real tip). /// /// Returns how many slots were removed. Mid-array holes are left for /// [`Store::revalidate_tip_window_n`] to shrink to last good. - pub fn trim_trailing_nulls(&self) -> Result { + pub fn trim_trailing_nulls( + &self, + write: &crate::array_table::ClassCWrite, + ) -> Result { let n = self.arr.len(); if n == 0 { return Ok(0); @@ -101,16 +118,20 @@ impl ConfirmedTable { } let trimmed = n - keep; if trimmed > 0 { - self.arr.truncate(keep)?; + self.arr.truncate(write, keep)?; } Ok(trimmed) } /// Disconnect tip: height must be current tip. - pub fn disconnect_tip(&self, height: Height) -> Result<(), StoreError> { + pub fn disconnect_tip( + &self, + write: &crate::array_table::ClassCWrite, + height: Height, + ) -> Result<(), StoreError> { match self.tip_height() { Some(t) if t == height => { - self.arr.truncate(u64::from(height.0))?; + self.arr.truncate(write, u64::from(height.0))?; Ok(()) } Some(_) => Err(StoreError::Corrupt("disconnect not at tip")), @@ -218,7 +239,11 @@ impl StrongTxTable { } #[doc(hidden)] - pub fn ensure_bits(&self, need_bits: u64) -> Result<(), StoreError> { + pub fn ensure_bits( + &self, + _write: &crate::array_table::ClassCWrite, + need_bits: u64, + ) -> Result<(), StoreError> { use std::sync::atomic::Ordering; let n = self.n_bits.load(Ordering::Acquire); if need_bits <= n { @@ -248,8 +273,13 @@ impl StrongTxTable { } #[doc(hidden)] - pub fn set_bit(&self, bit: u64, on: bool) -> Result<(), StoreError> { - self.ensure_bits(bit + 1)?; + pub fn set_bit( + &self, + write: &crate::array_table::ClassCWrite, + bit: u64, + on: bool, + ) -> Result<(), StoreError> { + self.ensure_bits(write, bit + 1)?; let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); self.refuse_while_flushing()?; let bi = (bit / 8) as usize; @@ -277,17 +307,23 @@ impl StrongTxTable { g.len() as u64 } - pub fn set_strong(&self, tx_fk: Fk, header_fk: Fk) -> Result<(), StoreError> { + pub fn set_strong( + &self, + write: &crate::array_table::ClassCWrite, + tx_fk: Fk, + header_fk: Fk, + ) -> Result<(), StoreError> { let id = tx_fk.get().ok_or(StoreError::InvalidFk)?; if header_fk.is_null() { return Err(StoreError::InvalidFk); } - self.set_bit(id - 1, true) + self.set_bit(write, id - 1, true) } /// Mark many consecutive tx fks strong (one ensure + bulk byte writes). pub fn set_strong_range( &self, + write: &crate::array_table::ClassCWrite, first_tx_fk: Fk, count: u32, header_fk: Fk, @@ -302,16 +338,22 @@ impl StrongTxTable { } let start = id - 1; let end = start + u64::from(count); - self.set_bits_range(start, end, true) + self.set_bits_range(write, start, end, true) } /// Set bits in half-open `[start, end)` (bit indices). #[doc(hidden)] - pub fn set_bits_range(&self, start: u64, end: u64, on: bool) -> Result<(), StoreError> { + pub fn set_bits_range( + &self, + write: &crate::array_table::ClassCWrite, + start: u64, + end: u64, + on: bool, + ) -> Result<(), StoreError> { if end <= start { return Ok(()); } - self.ensure_bits(end)?; + self.ensure_bits(write, end)?; let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); self.refuse_while_flushing()?; let mut bit = start; @@ -328,17 +370,26 @@ impl StrongTxTable { Ok(()) } - pub fn set_unstrong(&self, tx_fk: Fk) -> Result<(), StoreError> { + pub fn set_unstrong( + &self, + write: &crate::array_table::ClassCWrite, + tx_fk: Fk, + ) -> Result<(), StoreError> { let id = tx_fk.get().ok_or(StoreError::InvalidFk)?; let n = self.n_bits.load(std::sync::atomic::Ordering::Acquire); if id > n { return Ok(()); } - self.set_bit(id - 1, false) + self.set_bit(write, id - 1, false) } /// Clear many consecutive strong bits (Class C repair / disconnect ranges). - pub fn set_unstrong_range(&self, first_tx_fk: Fk, count: u32) -> Result<(), StoreError> { + pub fn set_unstrong_range( + &self, + write: &crate::array_table::ClassCWrite, + first_tx_fk: Fk, + count: u32, + ) -> Result<(), StoreError> { let id = first_tx_fk.get().ok_or(StoreError::InvalidFk)?; if count == 0 { return Ok(()); @@ -350,7 +401,7 @@ impl StrongTxTable { return Ok(()); } let end = end.min(n); - self.set_bits_range(start, end, false) + self.set_bits_range(write, start, end, false) } pub fn is_strong(&self, tx_fk: Fk) -> Result { @@ -526,7 +577,12 @@ impl HeaderTxsTable { } /// Store a contiguous range. `tx_fks` must be non-empty and contiguous. - pub fn put_list(&self, header_fk: Fk, tx_fks: &[Fk]) -> Result<(), StoreError> { + pub fn put_list( + &self, + write: &crate::array_table::ClassCWrite, + header_fk: Fk, + tx_fks: &[Fk], + ) -> Result<(), StoreError> { if tx_fks.is_empty() { return Err(StoreError::Corrupt("empty header tx list")); } @@ -542,21 +598,31 @@ impl HeaderTxsTable { { return Err(StoreError::Corrupt("header_txs not contiguous")); } - self.put_range(header_fk, tx_fks[0], tx_fks.len() as u32) + self.put_range(write, header_fk, tx_fks[0], tx_fks.len() as u32) } - pub fn put_range(&self, header_fk: Fk, first_tx_fk: Fk, n: u32) -> Result<(), StoreError> { + pub fn put_range( + &self, + write: &crate::array_table::ClassCWrite, + header_fk: Fk, + first_tx_fk: Fk, + n: u32, + ) -> Result<(), StoreError> { let id = header_fk.get().ok_or(StoreError::InvalidFk)?; if first_tx_fk.is_null() || n == 0 { return Err(StoreError::InvalidFk); } - self.first.set(id - 1, first_tx_fk.0)?; - self.count.set(id - 1, u64::from(n))?; + self.first.set(write, id - 1, first_tx_fk.0)?; + self.count.set(write, id - 1, u64::from(n))?; Ok(()) } /// Batch-append many header→tx ranges. - pub fn put_lists_batch(&self, items: &[(Fk, &[Fk])]) -> Result<(), StoreError> { + pub fn put_lists_batch( + &self, + write: &crate::array_table::ClassCWrite, + items: &[(Fk, &[Fk])], + ) -> Result<(), StoreError> { if items.is_empty() { return Ok(()); } @@ -576,13 +642,17 @@ impl HeaderTxsTable { first_pairs.push((id - 1, tx_fks[0].0)); count_pairs.push((id - 1, tx_fks.len() as u64)); } - self.first.set_many(&first_pairs)?; - self.count.set_many(&count_pairs)?; + self.first.set_many(write, &first_pairs)?; + self.count.set_many(write, &count_pairs)?; Ok(()) } /// Batch store ranges without expanding fk vectors. - pub fn put_ranges_batch(&self, items: &[(Fk, Fk, u32)]) -> Result<(), StoreError> { + pub fn put_ranges_batch( + &self, + write: &crate::array_table::ClassCWrite, + items: &[(Fk, Fk, u32)], + ) -> Result<(), StoreError> { if items.is_empty() { return Ok(()); } @@ -596,8 +666,8 @@ impl HeaderTxsTable { first_pairs.push((id - 1, first_tx_fk.0)); count_pairs.push((id - 1, u64::from(n))); } - self.first.set_many(&first_pairs)?; - self.count.set_many(&count_pairs)?; + self.first.set_many(write, &first_pairs)?; + self.count.set_many(write, &count_pairs)?; Ok(()) } @@ -666,7 +736,11 @@ impl HeaderTxsTable { /// /// Used when reconstruct produces a block that fails header checks (e.g. /// merkle root mismatch) — the header hash is fine; the association is bad. - pub fn clear_body(&self, header_fk: Fk) -> Result { + pub fn clear_body( + &self, + write: &crate::array_table::ClassCWrite, + header_fk: Fk, + ) -> Result { let id = header_fk.get().ok_or(StoreError::InvalidFk)?; if id == 0 { return Err(StoreError::InvalidFk); @@ -680,8 +754,8 @@ impl HeaderTxsTable { if idx >= self.first.len() { return Ok(false); } - self.first.set(idx, 0)?; - self.count.set(idx, 0)?; + self.first.set(write, idx, 0)?; + self.count.set(write, idx, 0)?; Ok(true) } diff --git a/crates/rbitcoin-store/src/integrity.rs b/crates/rbitcoin-store/src/integrity.rs index d4b3a31cba..abd120136c 100644 --- a/crates/rbitcoin-store/src/integrity.rs +++ b/crates/rbitcoin-store/src/integrity.rs @@ -236,10 +236,11 @@ impl Store { if let Some(h) = self.confirmed.tip_height() { report.tip_before = Some(h.0); } + let write = crate::array_table::ClassCWrite::new(); // HWM can include a trailing run of null slots (crash mid-grow). Trim // the whole suffix in one shot so the 6-height window sees a real tip. - let trimmed = self.confirmed.trim_trailing_nulls()?; + let trimmed = self.confirmed.trim_trailing_nulls(&write)?; if trimmed > 0 { self.flush_confirmed_only()?; self.rebuild_height_fence()?; @@ -253,13 +254,13 @@ impl Store { if let Some(tip) = self.confirmed.tip_height() { if tip.0 > seal.tip_height { // Unsealed extension past last barrier seal — drop to sealed tip. - self.shrink_tip_to(Some(seal.tip_height), &mut report)?; + self.shrink_tip_to(&write, Some(seal.tip_height), &mut report)?; } else if tip.0 == seal.tip_height { if let Some(fk) = self.confirmed.get(tip)? { if let Ok(rec) = self.headers.get(fk) { if rec.hash != seal.tip_hash { let lg = tip.0.checked_sub(1); - self.shrink_tip_to(lg, &mut report)?; + self.shrink_tip_to(&write, lg, &mut report)?; } } } @@ -287,7 +288,7 @@ impl Store { let mut progress = TipRevalidateLog::start(lo, tip.0); for h in lo..=tip.0 { - match self.check_confirmed_height(Height(h), tx_count, &mut report) { + match self.check_confirmed_height(&write, Height(h), tx_count, &mut report) { Ok(()) => last_good = Some(h), Err(reason) => { report.first_bad_height = Some(h); @@ -299,7 +300,7 @@ impl Store { } if report.first_bad_height.is_some() { - self.shrink_tip_to(last_good, &mut report)?; + self.shrink_tip_to(&write, last_good, &mut report)?; } else { report.tip_after = Some(tip.0); } @@ -319,6 +320,7 @@ impl Store { #[doc(hidden)] pub fn check_confirmed_height( &self, + write: &crate::array_table::ClassCWrite, height: Height, tx_count: u64, report: &mut TipRevalidateReport, @@ -378,16 +380,24 @@ impl Store { match self.header_txs.get_range(fk) { Ok(None) => Ok(()), - Ok(Some((first, count))) => { - self.check_header_tx_range(fk, first, count, tx_count, &rec.merkle_root, report) - } + Ok(Some((first, count))) => self.check_header_tx_range( + write, + fk, + first, + count, + tx_count, + &rec.merkle_root, + report, + ), Err(_) => Err("header_txs read"), } } #[doc(hidden)] + #[allow(clippy::too_many_arguments)] pub fn check_header_tx_range( &self, + write: &crate::array_table::ClassCWrite, fk: Fk, first: Fk, count: u32, @@ -396,37 +406,37 @@ impl Store { report: &mut TipRevalidateReport, ) -> Result<(), &'static str> { if count == 0 || first.is_null() { - let _ = self.header_txs.clear_body(fk); + let _ = self.header_txs.clear_body(write, fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err("header_txs empty association"); } let last = first.0.saturating_add(u64::from(count)).saturating_sub(1); if first.0 == 0 || last > tx_count { - let _ = self.header_txs.clear_body(fk); + let _ = self.header_txs.clear_body(write, fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err("header_txs range OOB"); } let leaves = match self.txs.body_txid_range(first.0, last) { Ok(v) if v.len() == count as usize => v, Ok(_) => { - let _ = self.header_txs.clear_body(fk); + let _ = self.header_txs.clear_body(write, fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err("txid.body short for header_txs range"); } Err(_) => { - let _ = self.header_txs.clear_body(fk); + let _ = self.header_txs.clear_body(write, fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err("txid.body read"); } }; let root = merkle_root_from_txids(&leaves); if root != *merkle_root { - let _ = self.header_txs.clear_body(fk); + let _ = self.header_txs.clear_body(write, fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err("merkle root mismatch"); } match self.strong_tx.all_strong_range(first, count) { - Ok(true) => self.check_block_input_edges(fk, first.0, count, report), + Ok(true) => self.check_block_input_edges(write, fk, first.0, count, report), Ok(false) => Err("strong bits missing in tip window"), Err(_) => Err("strong_tx read"), } @@ -441,6 +451,7 @@ impl Store { #[doc(hidden)] pub fn check_block_input_edges( &self, + write: &crate::array_table::ClassCWrite, header_fk: Fk, first: u64, count: u32, @@ -448,7 +459,7 @@ impl Store { ) -> Result<(), &'static str> { let reason = self.block_input_edge_reason(first, count); if let Err(reason) = reason { - let _ = self.header_txs.clear_body(header_fk); + let _ = self.header_txs.clear_body(write, header_fk); report.bodies_cleared = report.bodies_cleared.saturating_add(1); return Err(reason); } @@ -541,6 +552,7 @@ impl Store { #[doc(hidden)] pub fn shrink_tip_to( &self, + write: &crate::array_table::ClassCWrite, last_good: Option, report: &mut TipRevalidateReport, ) -> Result<(), StoreError> { @@ -566,7 +578,7 @@ impl Store { if !keep { break; } - self.confirmed.disconnect_tip(Height(cur))?; + self.confirmed.disconnect_tip(write, Height(cur))?; if cur == 0 { break; } diff --git a/crates/rbitcoin-store/src/lib.rs b/crates/rbitcoin-store/src/lib.rs index 14e2c927b9..ea8ec2dacd 100644 --- a/crates/rbitcoin-store/src/lib.rs +++ b/crates/rbitcoin-store/src/lib.rs @@ -121,6 +121,7 @@ pub mod uring_session; #[doc(hidden)] pub mod var_table; +pub use crate::array_table::ClassCWrite; pub use crate::chain::HeaderTxsTable; pub use crate::compact::output_flags; pub use address_head::{is_probe_exhausted_error, is_store_corrupt_display}; diff --git a/crates/rbitcoin-store/src/store.rs b/crates/rbitcoin-store/src/store.rs index fb3d14a4af..a8af08b2d4 100644 --- a/crates/rbitcoin-store/src/store.rs +++ b/crates/rbitcoin-store/src/store.rs @@ -1571,12 +1571,13 @@ impl Store { } let n_ranges = ranges.len(); let mut cleared = 0u64; + let write = crate::array_table::ClassCWrite::new(); for (lo, hi) in ranges { let ones = self.strong_tx.count_strong_in_fk_range(lo, hi)?; if ones == 0 { continue; } - self.clear_class_c_run(lo, hi)?; + self.clear_class_c_run(&write, lo, hi)?; cleared = cleared.saturating_add(ones); } let ms = t0.elapsed().as_millis(); @@ -1613,16 +1614,22 @@ impl Store { } #[doc(hidden)] - pub fn clear_class_c_run(&self, start: u64, end: u64) -> Result { + pub fn clear_class_c_run( + &self, + write: &crate::array_table::ClassCWrite, + start: u64, + end: u64, + ) -> Result { if end <= start { return Ok(0); } let count = end - start; if count <= u64::from(u32::MAX) { - self.strong_tx.set_unstrong_range(Fk(start), count as u32)?; + self.strong_tx + .set_unstrong_range(write, Fk(start), count as u32)?; } else { for id in start..end { - self.strong_tx.set_unstrong(Fk(id))?; + self.strong_tx.set_unstrong(write, Fk(id))?; } } Ok(count) diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index c521b8364b..170e900e00 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -24,7 +24,8 @@ fn get_racing_truncate_does_not_panic() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = Arc::new(ArrayTable::create(&path, TableKind::Confirmed).unwrap()); - t.set(1023, 7).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1023, 7) + .unwrap(); assert!(t.l2_resident_bytes() > 0, "test needs the in-RAM image"); let stop = Arc::new(AtomicBool::new(false)); let readers: Vec<_> = (0..4) @@ -40,8 +41,10 @@ fn get_racing_truncate_does_not_panic() { .collect(); let deadline = std::time::Instant::now() + std::time::Duration::from_millis(500); while std::time::Instant::now() < deadline { - t.truncate(0).unwrap(); - t.set(1023, 7).unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 0) + .unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1023, 7) + .unwrap(); } stop.store(true, Ordering::Relaxed); for r in readers { @@ -61,18 +64,20 @@ fn flush_writes_live_slots_and_refuses_a_store_until_it_finishes() { let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); let n = 600u64; let pairs: Vec<(u64, u64)> = (0..n).map(|i| (i, i + 1)).collect(); - t.set_many(&pairs).unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &pairs) + .unwrap(); t.flush().unwrap(); - t.set(0, 5).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 5) + .unwrap(); let mut during = Ok(()); t.flush_dirty_between(|t, step| { if step == FlushStep::Write { if let Some(v) = t.data.write().unwrap_or_else(|e| e.into_inner()).as_mut() { v[0] = 77; } - during = t.set(1, 9); - let _ = t.set_many(&[(2, 8)]); - let _ = t.truncate(n - 1); + during = t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 9); + let _ = t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[(2, 8)]); + let _ = t.truncate(&rbitcoin_store::ClassCWrite::for_test(), n - 1); } }) .unwrap(); @@ -88,7 +93,8 @@ fn flush_writes_live_slots_and_refuses_a_store_until_it_finishes() { assert!(n * ELEM > FLUSH_SCRATCH as u64); let chunk = t.flush_chunk_max.load(Ordering::Relaxed); assert!(chunk > 0 && chunk <= FLUSH_SCRATCH, "flush chunk {chunk}"); - t.set(1, 9).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 9) + .unwrap(); let mut buf = [0u8; 8]; t.file .read_at(FILE_HEADER_LEN as u64 + 8, &mut buf) @@ -113,12 +119,17 @@ fn same_length_rewrite_publishes_logical_len() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set_many(&[(0, 1), (1, 2), (2, 3), (3, 4)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 1), (1, 2), (2, 3), (3, 4)], + ) + .unwrap(); t.flush().unwrap(); let published = FILE_HEADER_LEN as u64 + 4 * ELEM; assert_eq!(t.file.logical_len(), published); t.file.set_logical_len(published + 64).unwrap(); - t.set(1, 99).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 99) + .unwrap(); assert_eq!(t.len(), 4); assert_eq!(t.disk_len.load(Ordering::Acquire), 4); t.flush().unwrap(); @@ -138,9 +149,14 @@ fn failed_write_keeps_the_range_dirty() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); - t.set_many(&[(0, 1), (1, 2), (2, 3), (3, 4)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 1), (1, 2), (2, 3), (3, 4)], + ) + .unwrap(); t.flush().unwrap(); - t.set(1, 42).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 42) + .unwrap(); let mut swapped = None; let failed = t.flush_dirty_between(|t, step| { if step == FlushStep::Write { @@ -177,13 +193,18 @@ fn set_after_flush_is_flushed_next() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); - t.set_many(&[(0, 10), (1, 11), (2, 12), (3, 13)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 10), (1, 11), (2, 12), (3, 13)], + ) + .unwrap(); t.flush().unwrap(); - t.set(3, 23).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 3, 23) + .unwrap(); let mut during = Ok(()); t.flush_dirty_between(|t, step| { if step == FlushStep::Clean { - during = t.set(1, 21); + during = t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 21); } }) .unwrap(); @@ -191,7 +212,8 @@ fn set_after_flush_is_flushed_next() { drop(t); let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); assert_eq!((t.get(1).unwrap(), t.get(3).unwrap()), (11, 23)); - t.set(1, 21).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 21) + .unwrap(); t.flush().unwrap(); drop(t); let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); @@ -206,14 +228,21 @@ fn array_set_get_grow_set_many_truncate_flush() { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); assert_eq!(t.len(), 0); assert_eq!(t.get(0).unwrap(), 0); - t.set(3, 30).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 3, 30) + .unwrap(); assert_eq!(t.len(), 4); assert_eq!(t.get(0).unwrap(), 0); assert_eq!(t.get(3).unwrap(), 30); - t.set(1, 11).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 11) + .unwrap(); assert_eq!(t.get(1).unwrap(), 11); - t.set_many(&[]).unwrap(); - t.set_many(&[(10, 100), (5, 50)]).unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[]) + .unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(10, 100), (5, 50)], + ) + .unwrap(); assert_eq!(t.len(), 11); assert_eq!(t.get(5).unwrap(), 50); assert_eq!(t.get(10).unwrap(), 100); @@ -223,9 +252,13 @@ fn array_set_get_grow_set_many_truncate_flush() { let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); assert_eq!(t.len(), 11); assert_eq!(t.get(3).unwrap(), 30); - t.truncate(4).unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 4) + .unwrap(); assert_eq!(t.len(), 4); - assert!(matches!(t.truncate(99), Err(StoreError::Corrupt(_)))); + assert!(matches!( + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 99), + Err(StoreError::Corrupt(_)) + )); t.flush().unwrap(); { let mut raw = std::fs::read(&path).unwrap(); @@ -246,10 +279,16 @@ fn write_behind_no_flush_reopen_keeps_old_image() { let _ = std::fs::remove_file(&path); { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set_many(&[(0, 10), (1, 20)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 10), (1, 20)], + ) + .unwrap(); t.flush().unwrap(); - t.set(0, 999).unwrap(); - t.set(2, 30).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 999) + .unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 2, 30) + .unwrap(); assert_eq!(t.get(0).unwrap(), 999); assert_eq!(t.len(), 3); } @@ -267,9 +306,14 @@ fn write_behind_flush_reopen_sees_new_image() { let _ = std::fs::remove_file(&path); { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set_many(&[(0, 1), (1, 2)]).unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[(0, 1), (1, 2)]) + .unwrap(); t.flush().unwrap(); - t.set_many(&[(0, 7), (1, 8), (2, 9)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 7), (1, 8), (2, 9)], + ) + .unwrap(); t.flush().unwrap(); } let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); @@ -286,14 +330,20 @@ fn write_behind_truncate_flush_reopen() { let _ = std::fs::remove_file(&path); { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set_many(&[(0, 1), (1, 2), (2, 3), (3, 4)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 1), (1, 2), (2, 3), (3, 4)], + ) + .unwrap(); t.flush().unwrap(); - t.truncate(2).unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 2) + .unwrap(); assert_eq!(t.len(), 2); drop(t); let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); assert_eq!(t.len(), 4); - t.truncate(2).unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 2) + .unwrap(); t.flush().unwrap(); } let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); @@ -310,10 +360,15 @@ fn append_only_flush_extends_without_losing_prefix() { let _ = std::fs::remove_file(&path); { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set_many(&[(0, 100), (1, 101)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 100), (1, 101)], + ) + .unwrap(); t.flush().unwrap(); // Pure tip extension (unstored index 2, disk length 2). - t.set(2, 102).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 2, 102) + .unwrap(); t.flush().unwrap(); } let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); @@ -329,13 +384,18 @@ fn set_gap_zeros_prefix_and_env_cap_and_flush_async() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set(3, 42).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 3, 42) + .unwrap(); assert_eq!(t.len(), 4); assert_eq!(t.get(0).unwrap(), 0); assert_eq!(t.get(1).unwrap(), 0); assert_eq!(t.get(2).unwrap(), 0); assert_eq!(t.get(3).unwrap(), 42); - t.set_many(&[(1, 11), (5, 55), (3, 33)]).unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(1, 11), (5, 55), (3, 33)], + ) + .unwrap(); assert_eq!(t.len(), 6); assert_eq!(t.get(1).unwrap(), 11); assert_eq!(t.get(3).unwrap(), 33); @@ -343,7 +403,8 @@ fn set_gap_zeros_prefix_and_env_cap_and_flush_async() { t.flush_async().unwrap(); t.flush_dirty().unwrap(); t.flush().unwrap(); - t.truncate(0).unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 0) + .unwrap(); t.flush().unwrap(); assert_eq!(t.len(), 0); drop(t); @@ -370,9 +431,11 @@ fn store_on_another_thread_during_flush_is_refused() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = Arc::new(ArrayTable::create(&path, TableKind::Confirmed).unwrap()); - t.set_many(&[(0, 1), (1, 2)]).unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[(0, 1), (1, 2)]) + .unwrap(); t.flush().unwrap(); - t.set(0, 5).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 5) + .unwrap(); let (started, start_rx) = std::sync::mpsc::channel(); let (release, release_rx) = std::sync::mpsc::channel(); let flush_t = { @@ -388,14 +451,15 @@ fn store_on_another_thread_during_flush_is_refused() { }) }; start_rx.recv().unwrap(); - let during = t.set(0, 9); + let during = t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 9); release.send(()).unwrap(); flush_t.join().unwrap(); assert!(during.is_err(), "store during flush refused={during:?}"); drop(t); let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); assert_eq!(t.get(0).unwrap(), 5); - t.set(0, 9).unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 9) + .unwrap(); t.flush().unwrap(); drop(t); let t = ArrayTable::open(&path, TableKind::Confirmed).unwrap(); diff --git a/crates/rbitcoin-store/tests/chain/chain_table_tests.rs b/crates/rbitcoin-store/tests/chain/chain_table_tests.rs index 18bb4ce3c8..3b6ccd063e 100644 --- a/crates/rbitcoin-store/tests/chain/chain_table_tests.rs +++ b/crates/rbitcoin-store/tests/chain/chain_table_tests.rs @@ -21,34 +21,51 @@ fn confirmed_tx_height_header_txs_surface() { let dir = tmp(); let c = ConfirmedTable::create(&dir).unwrap(); assert!(c.tip_height().is_none()); - c.set(Height(0), Fk(1)).unwrap(); - c.set(Height(1), Fk(2)).unwrap(); + c.set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + c.set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); assert_eq!(c.tip_height(), Some(Height(1))); assert_eq!(c.get(Height(0)).unwrap(), Some(Fk(1))); - c.set_many(&[]).unwrap(); - c.set_many(&[(Height(2), Fk(3)), (Height(3), Fk(4))]) + c.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[]) .unwrap(); + c.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Height(2), Fk(3)), (Height(3), Fk(4))], + ) + .unwrap(); assert_eq!(c.tip_height(), Some(Height(3))); assert!(matches!( - c.set(Height(4), Fk::NULL), + c.set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(4), + Fk::NULL + ), Err(StoreError::InvalidFk) )); assert!(matches!( - c.set_many(&[(Height(5), Fk::NULL)]), + c.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Height(5), Fk::NULL)] + ), Err(StoreError::InvalidFk) )); - c.disconnect_tip(Height(3)).unwrap(); + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(3)) + .unwrap(); assert_eq!(c.tip_height(), Some(Height(2))); assert!(matches!( - c.disconnect_tip(Height(0)), + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(0)), Err(StoreError::Corrupt(_)) )); - c.disconnect_tip(Height(2)).unwrap(); - c.disconnect_tip(Height(1)).unwrap(); - c.disconnect_tip(Height(0)).unwrap(); + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(2)) + .unwrap(); + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(1)) + .unwrap(); + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(0)) + .unwrap(); assert!(c.tip_height().is_none()); assert!(matches!( - c.disconnect_tip(Height(0)), + c.disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(0)), Err(StoreError::Corrupt(_)) )); c.flush().unwrap(); @@ -56,7 +73,8 @@ fn confirmed_tx_height_header_txs_surface() { let ht = HeaderTxsTable::create(&dir).unwrap(); assert!(ht.get_range(Fk(1)).unwrap().is_none()); - ht.put_range(Fk(1), Fk(100), 3).unwrap(); + ht.put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(100), 3) + .unwrap(); assert_eq!(ht.get_range(Fk(1)).unwrap(), Some((Fk(100), 3))); assert_eq!( ht.get_list(Fk(1)).unwrap().unwrap(), @@ -64,40 +82,63 @@ fn confirmed_tx_height_header_txs_surface() { ); assert!(ht.has_body(Fk(1)).unwrap()); assert!(!ht.has_body(Fk(2)).unwrap()); - ht.put_list(Fk(2), &[Fk(200), Fk(201)]).unwrap(); + ht.put_list( + &rbitcoin_store::ClassCWrite::for_test(), + Fk(2), + &[Fk(200), Fk(201)], + ) + .unwrap(); assert!(matches!( - ht.put_list(Fk(3), &[]), + ht.put_list(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), &[]), Err(StoreError::Corrupt(_)) )); // Non-contiguous triggers debug_assert in debug builds; only empty is a // stable Err path under RUSTFLAGS=-Dwarnings debug tests. assert!(matches!( - ht.put_range(Fk::NULL, Fk(1), 1), + ht.put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk::NULL, Fk(1), 1), Err(StoreError::InvalidFk) )); assert!(matches!( - ht.put_range(Fk(3), Fk::NULL, 1), + ht.put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), Fk::NULL, 1), Err(StoreError::InvalidFk) )); assert!(matches!( - ht.put_range(Fk(3), Fk(1), 0), + ht.put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), Fk(1), 0), Err(StoreError::InvalidFk) )); - ht.put_lists_batch(&[]).unwrap(); - ht.put_lists_batch(&[(Fk(3), &[Fk(300)] as &[_]), (Fk(4), &[Fk(400), Fk(401)])]) + ht.put_lists_batch(&rbitcoin_store::ClassCWrite::for_test(), &[]) .unwrap(); + ht.put_lists_batch( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Fk(3), &[Fk(300)] as &[_]), (Fk(4), &[Fk(400), Fk(401)])], + ) + .unwrap(); assert!(matches!( - ht.put_lists_batch(&[(Fk(5), &[] as &[_])]), + ht.put_lists_batch( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Fk(5), &[] as &[_])] + ), Err(StoreError::Corrupt(_)) )); - ht.put_ranges_batch(&[]).unwrap(); - ht.put_ranges_batch(&[(Fk(5), Fk(500), 2)]).unwrap(); + ht.put_ranges_batch(&rbitcoin_store::ClassCWrite::for_test(), &[]) + .unwrap(); + ht.put_ranges_batch( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Fk(5), Fk(500), 2)], + ) + .unwrap(); assert!(matches!( - ht.put_ranges_batch(&[(Fk::NULL, Fk(1), 1)]), + ht.put_ranges_batch( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Fk::NULL, Fk(1), 1)] + ), Err(StoreError::InvalidFk) )); assert!(matches!( - ht.put_ranges_batch(&[(Fk(6), Fk::NULL, 1)]), + ht.put_ranges_batch( + &rbitcoin_store::ClassCWrite::for_test(), + &[(Fk(6), Fk::NULL, 1)] + ), Err(StoreError::InvalidFk) )); assert_eq!(ht.count_bodies().unwrap(), 5); @@ -105,13 +146,25 @@ fn confirmed_tx_height_header_txs_surface() { // clear_body: drop association without freeing tx rows (merkle soft-reget path). assert!(matches!( - ht.clear_body(Fk::NULL), + ht.clear_body(&rbitcoin_store::ClassCWrite::for_test(), Fk::NULL), Err(StoreError::InvalidFk) )); - assert!(!ht.clear_body(Fk(99)).unwrap(), "no body → false"); - assert!(ht.clear_body(Fk(1)).unwrap(), "had body → true"); + assert!( + !ht.clear_body(&rbitcoin_store::ClassCWrite::for_test(), Fk(99)) + .unwrap(), + "no body → false" + ); + assert!( + ht.clear_body(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(), + "had body → true" + ); assert!(!ht.has_body(Fk(1)).unwrap()); - assert!(!ht.clear_body(Fk(1)).unwrap(), "already clear → false"); + assert!( + !ht.clear_body(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(), + "already clear → false" + ); assert_eq!(ht.count_bodies().unwrap(), 4); ht.flush().unwrap(); @@ -132,7 +185,8 @@ fn header_txs_contains_tx_range_edges() { let ht = HeaderTxsTable::create(&dir).unwrap(); assert!(!ht.contains_tx(Fk(1), Fk::NULL).unwrap()); assert!(!ht.contains_tx(Fk(1), Fk(1)).unwrap(), "no body"); - ht.put_range(Fk(1), Fk(10), 3).unwrap(); + ht.put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(10), 3) + .unwrap(); assert!(ht.contains_tx(Fk(1), Fk(10)).unwrap()); assert!(ht.contains_tx(Fk(1), Fk(12)).unwrap()); assert!(!ht.contains_tx(Fk(1), Fk(9)).unwrap()); diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index 98ae1f5914..6d21654ee9 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -24,14 +24,16 @@ fn strong_flush_writes_live_bytes_and_refuses_a_store() { let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); let bits = (FLUSH_SCRATCH as u32) * 8 + 64; - t.set_strong_range(Fk(1), bits, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), bits, Fk(1)) + .unwrap(); t.flush().unwrap(); - t.set_unstrong(Fk(1)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(); let mut during = Ok(()); t.flush_dirty_between(|t, step| { if step == FlushStep::Write { t.data.write().unwrap_or_else(|e| e.into_inner())[0] = 0xff; - during = t.set_unstrong(Fk(2)); + during = t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(2)); } }) .unwrap(); @@ -45,7 +47,8 @@ fn strong_flush_writes_live_bytes_and_refuses_a_store() { "live byte must be the flushed image" ); assert!(t.is_strong(Fk(2)).unwrap(), "refused store stays off disk"); - t.set_unstrong(Fk(2)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(2)) + .unwrap(); t.flush().unwrap(); drop(t); let t = StrongTxTable::open(&dir).unwrap(); @@ -61,9 +64,11 @@ fn strong_failed_write_keeps_the_range_dirty() { use rbitcoin_store::file::FILE_HEADER_LEN; let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong(Fk(1), Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); t.flush().unwrap(); - t.set_unstrong(Fk(1)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(); let mut swapped = None; let failed = t.flush_dirty_between(|t, step| { if step == FlushStep::Write { @@ -90,12 +95,14 @@ fn strong_same_length_rewrite_publishes_logical_len() { use rbitcoin_store::file::FILE_HEADER_LEN; let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong_range(Fk(1), 16, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 16, Fk(1)) + .unwrap(); t.flush().unwrap(); let published = t.bits.logical_len(); assert!(published > FILE_HEADER_LEN as u64); t.bits.set_logical_len(published + 64).unwrap(); - t.set_unstrong(Fk(3)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(3)) + .unwrap(); t.flush_dirty().unwrap(); assert_eq!(t.bits.logical_len(), published); drop(t); @@ -109,12 +116,14 @@ fn strong_same_length_rewrite_publishes_logical_len() { fn strong_bitset_range_and_clear() { let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong_range(Fk(1), 10, Fk(99)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 10, Fk(99)) + .unwrap(); for i in 1..=10 { assert!(t.is_strong(Fk(i)).unwrap()); } assert!(!t.is_strong(Fk(11)).unwrap()); - t.set_unstrong(Fk(5)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(5)) + .unwrap(); assert!(!t.is_strong(Fk(5)).unwrap()); assert!(t.is_strong(Fk(4)).unwrap()); assert!(t.get(Fk(1)).unwrap().is_some()); @@ -134,8 +143,10 @@ fn count_strong_in_fk_range_skips_zero_bytes() { assert_eq!(t.count_strong_in_fk_range(0, 8).unwrap(), 0); assert_eq!(t.count_strong_in_fk_range(5, 5).unwrap(), 0); assert_eq!(t.count_strong_in_fk_range(100, 200).unwrap(), 0); - t.set_strong(Fk(1), Fk(1)).unwrap(); - t.set_strong(Fk(16), Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(16), Fk(1)) + .unwrap(); assert_eq!(t.count_strong_in_fk_range(1, 17).unwrap(), 2); assert_eq!(t.count_strong_in_fk_range(2, 16).unwrap(), 0); t.flush().unwrap(); @@ -150,7 +161,8 @@ fn count_strong_in_fk_range_skips_zero_bytes() { fn strong_tx_count_reads_l2_not_the_file() { let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong_range(Fk(1), 20, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 20, Fk(1)) + .unwrap(); t.flush().unwrap(); assert!(t.l2_resident_bytes() > 0); assert_eq!(t.count_strong_in_fk_range(1, 21).unwrap(), 20); @@ -178,7 +190,8 @@ fn strong_tx_always_l2_above_class_c_cap() { let fk = Fk(8_388_608 + 8); { let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong(fk, Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), fk, Fk(1)) + .unwrap(); t.flush().unwrap(); assert!( t.l2_resident_bytes() > 1024 * 1024, @@ -206,9 +219,11 @@ fn strong_store_on_another_thread_during_flush_is_refused() { use std::sync::Arc; let dir = tmp(); let t = Arc::new(StrongTxTable::create(&dir).unwrap()); - t.set_strong(Fk(1), Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); t.flush().unwrap(); - t.set_unstrong(Fk(1)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(); let (started, start_rx) = std::sync::mpsc::channel(); let (release, release_rx) = std::sync::mpsc::channel(); let flush_t = { @@ -224,14 +239,15 @@ fn strong_store_on_another_thread_during_flush_is_refused() { }) }; start_rx.recv().unwrap(); - let during = t.set_strong(Fk(1), Fk(1)); + let during = t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)); release.send(()).unwrap(); flush_t.join().unwrap(); assert!(during.is_err(), "store during flush refused={during:?}"); drop(t); let t = StrongTxTable::open(&dir).unwrap(); assert!(!t.is_strong(Fk(1)).unwrap()); - t.set_strong(Fk(1), Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); t.flush().unwrap(); drop(t); let t = StrongTxTable::open(&dir).unwrap(); @@ -244,9 +260,11 @@ fn strong_write_behind_no_flush_keeps_old() { let dir = tmp(); { let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong_range(Fk(1), 8, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 8, Fk(1)) + .unwrap(); t.flush().unwrap(); - t.set_strong(Fk(9), Fk(1)).unwrap(); + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(9), Fk(1)) + .unwrap(); // Drop without flush. } let t = StrongTxTable::open(&dir).unwrap(); @@ -264,9 +282,16 @@ fn strong_high_fk_flush_writes_suffix_not_full_image() { let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); // 80_000 bits = 10_000 payload bytes. - t.set_strong_range(Fk(1), 80_000, Fk(1)).unwrap(); + t.set_strong_range( + &rbitcoin_store::ClassCWrite::for_test(), + Fk(1), + 80_000, + Fk(1), + ) + .unwrap(); t.flush_dirty().unwrap(); - t.set_unstrong(Fk(80_000)).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(80_000)) + .unwrap(); t.flush_dirty().unwrap(); assert!(t.is_strong(Fk(1)).unwrap()); assert!(!t.is_strong(Fk(80_000)).unwrap()); @@ -280,10 +305,22 @@ fn strong_suffix_dirty_flush_preserves_prefix() { { let s = StrongTxTable::create(&dir).unwrap(); // Allocate a wide bit image then flush so disk_bytes covers it. - s.set_strong_range(Fk(1), 10_000, Fk(1)).unwrap(); + s.set_strong_range( + &rbitcoin_store::ClassCWrite::for_test(), + Fk(1), + 10_000, + Fk(1), + ) + .unwrap(); s.flush().unwrap(); // Flip only high bits (still within disk image) — suffix dirty path. - s.set_strong_range(Fk(9000), 100, Fk(1)).unwrap(); + s.set_strong_range( + &rbitcoin_store::ClassCWrite::for_test(), + Fk(9000), + 100, + Fk(1), + ) + .unwrap(); s.flush().unwrap(); } let s = StrongTxTable::open(&dir).unwrap(); @@ -299,9 +336,11 @@ fn strong_write_behind_flush_reopen() { let dir = tmp(); { let t = StrongTxTable::create(&dir).unwrap(); - t.set_strong_range(Fk(1), 16, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 16, Fk(1)) + .unwrap(); t.flush().unwrap(); - t.set_unstrong_range(Fk(5), 4).unwrap(); + t.set_unstrong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(5), 4) + .unwrap(); t.flush().unwrap(); } let t = StrongTxTable::open(&dir).unwrap(); @@ -317,37 +356,43 @@ fn strong_partial_bytes_unstrong_range_and_errors() { let dir = tmp(); let t = StrongTxTable::create(&dir).unwrap(); // Unaligned start so partial-first-byte path runs. - t.set_strong_range(Fk(3), 20, Fk(1)).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), 20, Fk(1)) + .unwrap(); for i in 3..23 { assert!(t.is_strong(Fk(i)).unwrap(), "i={i}"); } assert!(!t.is_strong(Fk(2)).unwrap()); - t.set_strong_range(Fk(1), 0, Fk(1)).unwrap(); - t.set_unstrong_range(Fk(1), 0).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 0, Fk(1)) + .unwrap(); + t.set_unstrong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 0) + .unwrap(); // Clear a mid-range with partial ends. - t.set_unstrong_range(Fk(5), 10).unwrap(); + t.set_unstrong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(5), 10) + .unwrap(); for i in 5..15 { assert!(!t.is_strong(Fk(i)).unwrap()); } assert!(t.is_strong(Fk(4)).unwrap()); assert!(t.is_strong(Fk(15)).unwrap()); // past allocated → no-op - t.set_unstrong(Fk(9999)).unwrap(); - t.set_unstrong_range(Fk(9000), 10).unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(9999)) + .unwrap(); + t.set_unstrong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(9000), 10) + .unwrap(); assert!(matches!( - t.set_strong(Fk::NULL, Fk(1)), + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk::NULL, Fk(1)), Err(StoreError::InvalidFk) )); assert!(matches!( - t.set_strong(Fk(1), Fk::NULL), + t.set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk::NULL), Err(StoreError::InvalidFk) )); assert!(matches!( - t.set_strong_range(Fk::NULL, 1, Fk(1)), + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk::NULL, 1, Fk(1)), Err(StoreError::InvalidFk) )); assert!(matches!( - t.set_strong_range(Fk(1), 1, Fk::NULL), + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 1, Fk::NULL), Err(StoreError::InvalidFk) )); assert!(matches!(t.is_strong(Fk::NULL), Err(StoreError::InvalidFk))); diff --git a/crates/rbitcoin-store/tests/integrity.rs b/crates/rbitcoin-store/tests/integrity.rs index 39d639be6f..2735585266 100644 --- a/crates/rbitcoin-store/tests/integrity.rs +++ b/crates/rbitcoin-store/tests/integrity.rs @@ -72,7 +72,9 @@ fn clean_header_chain_revalidate_no_shrink() { parent_hash = rec.hash; let fk = s.put_header(&rec).unwrap(); prev = fk; - s.confirmed.set(Height(h), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), fk) + .unwrap(); } s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); @@ -105,15 +107,23 @@ fn poison_prev_edge_shrinks_tip() { let s = Store::create_tiny(&dir).unwrap(); let g = hdr(Fk::NULL, [0u8; 32], 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); let a = hdr(g_fk, g.hash, 1); let a_fk = s.put_header(&a).unwrap(); - s.confirmed.set(Height(1), a_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), a_fk) + .unwrap(); let b = hdr(a_fk, a.hash, 2); let b_fk = s.put_header(&b).unwrap(); - s.confirmed.set(Height(2), b_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(2), b_fk) + .unwrap(); // Steal tip height 2 to point at G (false conf edge). - s.confirmed.set(Height(2), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(2), g_fk) + .unwrap(); s.flush_class_c_tip().unwrap(); rbitcoin_store::spend_durable::SpendDurable::new(5, 5) .store(s.path()) @@ -142,7 +152,9 @@ fn tip_seal_clamps_unsealed_extension() { let s = Store::create_tiny(&dir).unwrap(); let g = hdr(Fk::NULL, [0u8; 32], 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); s.flush_class_c_tip().unwrap(); let seal = TipSeal::load(s.path()) .unwrap() @@ -154,7 +166,9 @@ fn tip_seal_clamps_unsealed_extension() { // simulate by writing a higher tip then restoring old seal file. let a = hdr(g_fk, g.hash, 1); let a_fk = s.put_header(&a).unwrap(); - s.confirmed.set(Height(1), a_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), a_fk) + .unwrap(); s.confirmed.flush().unwrap(); // Overwrite seal with the old tip-0 seal (as if barrier never finished). seal.store(s.path()).unwrap(); @@ -172,11 +186,15 @@ fn merkle_mismatch_clears_body_and_shrinks() { let s = Store::create_tiny(&dir).unwrap(); let g = hdr(Fk::NULL, [0u8; 32], 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); let tx_fk = put_coinbase(&s, 9); let txid = s.txs.body_txid(tx_fk).unwrap(); // Header merkle is salt-based, not txid → S3 fail. - s.header_txs.put_range(g_fk, tx_fk, 1).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), g_fk, tx_fk, 1) + .unwrap(); // Ensure genesis hash still consistent with its merkle field (no rewrite). let _ = txid; s.flush_class_c_tip().unwrap(); @@ -218,9 +236,15 @@ fn tip_window_missing_strong_shrinks() { let rec = hdr_merkle(prev, parent_hash, h as u8, txid); parent_hash = rec.hash; let hfk = s.put_header(&rec).unwrap(); - s.header_txs.put_range(hfk, tx_fk, 1).unwrap(); - s.strong_tx.set_strong(tx_fk, hfk).unwrap(); - s.confirmed.set(Height(h), hfk).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), hfk, tx_fk, 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), tx_fk, hfk) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), hfk) + .unwrap(); prev = hfk; tip_tx = tx_fk; } @@ -230,7 +254,9 @@ fn tip_window_missing_strong_shrinks() { s.txs.flush().unwrap(); s.header_txs.flush().unwrap(); assert_eq!(s.confirmed.tip_height(), Some(Height(2))); - s.strong_tx.set_unstrong(tip_tx).unwrap(); + s.strong_tx + .set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), tip_tx) + .unwrap(); let r = s.revalidate_tip_window_n(6).unwrap(); assert_eq!( @@ -260,7 +286,9 @@ fn trailing_null_confirmed_slots_trimmed_in_one_open() { parent_hash = rec.hash; let fk = s.put_header(&rec).unwrap(); prev = fk; - s.confirmed.set(Height(h), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), fk) + .unwrap(); } s.headers.flush().unwrap(); s.confirmed.flush().unwrap(); @@ -305,7 +333,9 @@ fn revalidate_n_zero_and_empty_store() { let rec = hdr(Fk::NULL, [0u8; 32], 0); let fk = s.put_header(&rec).unwrap(); - s.confirmed.set(Height(0), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), fk) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); let r = s.revalidate_tip_window_n(0).unwrap(); @@ -323,7 +353,9 @@ fn genesis_nonzero_prev_fk_and_header_txs_oob_shrink() { let dummy_fk = s.put_header(&dummy).unwrap(); let g = hdr(dummy_fk, dummy.hash, 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); let r = s.revalidate_tip_window_n(6).unwrap(); @@ -335,8 +367,17 @@ fn genesis_nonzero_prev_fk_and_header_txs_oob_shrink() { let s = Store::create_tiny(&dir).unwrap(); let g = hdr(Fk::NULL, [0u8; 32], 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); - s.header_txs.put_range(g_fk, Fk(1_000_000), 1).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); + s.header_txs + .put_range( + &rbitcoin_store::ClassCWrite::for_test(), + g_fk, + Fk(1_000_000), + 1, + ) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); s.header_txs.flush().unwrap(); @@ -352,7 +393,9 @@ fn tip_seal_hash_mismatch_drops_tip_block() { let s = Store::create_tiny(&dir).unwrap(); let g = hdr(Fk::NULL, [0u8; 32], 0); let g_fk = s.put_header(&g).unwrap(); - s.confirmed.set(Height(0), g_fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), g_fk) + .unwrap(); s.flush_class_c_tip().unwrap(); let mut seal = TipSeal::load(s.path()) .unwrap() @@ -381,9 +424,17 @@ fn checkblocks_zero_walks_below_default_window() { genesis_fk = fk; } prev = fk; - s.confirmed.set(Height(h), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), fk) + .unwrap(); } - s.confirmed.set(Height(1), genesis_fk).unwrap(); + s.confirmed + .set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(1), + genesis_fk, + ) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); // A marker at the tip keeps the default window short. Without one, @@ -421,9 +472,17 @@ fn durable_through_widens_past_the_six_block_window() { genesis_fk = fk; } prev = fk; - s.confirmed.set(Height(h), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), fk) + .unwrap(); } - s.confirmed.set(Height(1), genesis_fk).unwrap(); + s.confirmed + .set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(1), + genesis_fk, + ) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); rbitcoin_store::spend_durable::SpendDurable::new(0, 0) @@ -453,9 +512,17 @@ fn missing_marker_revalidates_from_genesis() { genesis_fk = fk; } prev = fk; - s.confirmed.set(Height(h), fk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(h), fk) + .unwrap(); } - s.confirmed.set(Height(1), genesis_fk).unwrap(); + s.confirmed + .set( + &rbitcoin_store::ClassCWrite::for_test(), + Height(1), + genesis_fk, + ) + .unwrap(); s.flush_class_c_tip().unwrap(); s.headers.flush().unwrap(); assert!(rbitcoin_store::spend_durable::SpendDurable::load(s.path()) diff --git a/crates/rbitcoin-store/tests/spend_durable.rs b/crates/rbitcoin-store/tests/spend_durable.rs index d14cdb69a9..8355d56a8a 100644 --- a/crates/rbitcoin-store/tests/spend_durable.rs +++ b/crates/rbitcoin-store/tests/spend_durable.rs @@ -38,7 +38,11 @@ fn eight_notes_do_not_publish_and_checkpoint_keeps_the_snapshot() { let _ = std::fs::remove_dir_all(&dir); let s = rbitcoin_store::Store::create_tiny(&dir).unwrap(); s.confirmed - .set(rbitcoin_primitives::Height(7), rbitcoin_primitives::Fk(1)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(7), + rbitcoin_primitives::Fk(1), + ) .unwrap(); for _ in 0..8 { s.note_spend_snapshot(4); @@ -80,7 +84,11 @@ fn clamp_drops_a_stale_height() { let _ = std::fs::remove_dir_all(&dir); let s = rbitcoin_store::Store::create_tiny(&dir).unwrap(); s.confirmed - .set(rbitcoin_primitives::Height(0), rbitcoin_primitives::Fk(1)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(0), + rbitcoin_primitives::Fk(1), + ) .unwrap(); SpendDurable::new(5, 0).store(s.path()).unwrap(); s.clamp_spend_durable().unwrap(); @@ -109,6 +117,7 @@ fn stale_snapshot_note_after_disconnect_stays_at_the_tip() { for h in 0..=7u32 { s.confirmed .set( + &rbitcoin_store::ClassCWrite::for_test(), rbitcoin_primitives::Height(h), rbitcoin_primitives::Fk(u64::from(h) + 1), ) @@ -118,7 +127,10 @@ fn stale_snapshot_note_after_disconnect_stays_at_the_tip() { s.note_spend_snapshot(write_read_tip); for h in (5..=7u32).rev() { s.confirmed - .disconnect_tip(rbitcoin_primitives::Height(h)) + .disconnect_tip( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(h), + ) .unwrap(); } s.clamp_spend_durable().unwrap(); @@ -142,7 +154,11 @@ fn checkpoint_stops_below_a_pending_annotate() { let _ = std::fs::remove_dir_all(&dir); let s = rbitcoin_store::Store::create_tiny(&dir).unwrap(); s.confirmed - .set(rbitcoin_primitives::Height(7), rbitcoin_primitives::Fk(1)) + .set( + &rbitcoin_store::ClassCWrite::for_test(), + rbitcoin_primitives::Height(7), + rbitcoin_primitives::Fk(1), + ) .unwrap(); s.note_spend_snapshot(7); s.note_spend_annotate_pending(5); diff --git a/crates/rbitcoin-store/tests/store/mod.rs b/crates/rbitcoin-store/tests/store/mod.rs index 18540cdd0a..c8b7700418 100644 --- a/crates/rbitcoin-store/tests/store/mod.rs +++ b/crates/rbitcoin-store/tests/store/mod.rs @@ -324,8 +324,12 @@ fn coinbase_fk_at_heights_matches_first_tx() { .unwrap(); let fks = [cb_fks[0], non_fks[0]]; assert_eq!(fks.len(), 2); - s.header_txs.put_range(hfk, fks[0], 2).unwrap(); - s.confirmed.set(Height(0), hfk).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), hfk, fks[0], 2) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), hfk) + .unwrap(); s.rebuild_height_fence().unwrap(); let map = s.coinbase_fk_at_heights(&[0, 1, 99]).unwrap(); @@ -540,14 +544,23 @@ fn store_create_open_archive_spend_and_meta_errors() { // Class C: confirm spenders + heights. Body list must include spend_fk // (membership is part of is_confirmed_strong). - s.confirmed.set(Height(0), hfk).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), hfk) + .unwrap(); // Contiguous body covering create..spend (sequential put order). let body_first = create_fk.0.min(spend_fk.0); let body_last = create_fk.0.max(spend_fk.0); s.header_txs - .put_range(hfk, Fk(body_first), (body_last - body_first + 1) as u32) + .put_range( + &rbitcoin_store::ClassCWrite::for_test(), + hfk, + Fk(body_first), + (body_last - body_first + 1) as u32, + ) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), spend_fk, hfk) .unwrap(); - s.strong_tx.set_strong(spend_fk, hfk).unwrap(); s.rebuild_height_fence().unwrap(); assert!(s.is_confirmed_strong(spend_fk).unwrap()); assert!(!s.is_confirmed_strong(spend2_fk).unwrap()); @@ -591,7 +604,9 @@ fn store_create_open_archive_spend_and_meta_errors() { s.flush_for_shutdown().unwrap(); // repair: strong not on the fence - s.strong_tx.set_strong(spend2_fk, hfk).unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), spend2_fk, hfk) + .unwrap(); let cleared = s.repair_class_c_above_tip().unwrap(); assert!(cleared >= 1); assert!(!s.is_confirmed_strong(spend2_fk).unwrap()); @@ -1015,17 +1030,29 @@ fn class_c_barrier_pre_tip_only_does_not_advance_tip() { { let s = Store::create_tiny(&dir).unwrap(); // Genesis tip: height 0 → header fk 1, one strong tx. - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - s.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); s.flush_class_c_tip().unwrap(); assert_eq!(s.confirmed.tip_height(), Some(Height(0))); // In-RAM tip extension (height 1) + strong for new txs — no full barrier. - s.strong_tx.set_strong_range(Fk(2), 3, Fk(2)).unwrap(); - s.header_txs.put_range(Fk(2), Fk(2), 3).unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), 3, Fk(2)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(2), 3) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); // Mid-barrier: strong/height durable, confirmed still unflushed. s.flush_class_c_pre_tip().unwrap(); assert_eq!(s.confirmed.tip_height(), Some(Height(1))); @@ -1057,15 +1084,27 @@ fn class_c_barrier_full_flush_reopen_tip_with_strong() { let dir = tmp(); { let s = Store::create_tiny(&dir).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - s.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); s.flush_class_c_tip().unwrap(); - s.strong_tx.set_strong_range(Fk(2), 2, Fk(2)).unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); - s.header_txs.put_range(Fk(2), Fk(2), 2).unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), 2, Fk(2)) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(2), 2) + .unwrap(); s.rebuild_height_fence().unwrap(); s.flush_class_c_tip().unwrap(); } @@ -1084,9 +1123,15 @@ fn class_c_barrier_full_flush_reopen_tip_with_strong() { fn get_fk_by_txid_tip_hits_pending_before_head_drain() { let dir = tmp(); let s = Store::create_tiny(&dir).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - s.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); let item = coinbase_item([0x51; 32], vec![OutputRecord::unspent(1, vec![0x51])]); let fks = s .put_tx_full_batch_indexed(&[item], /*index=*/ false) @@ -1110,14 +1155,22 @@ fn get_fk_by_txid_tip_hits_pending_before_head_drain() { fn fence_tip_height_lags_unextended_confirmed() { let dir = tmp(); let s = Store::create_tiny(&dir).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); s.rebuild_height_fence().unwrap(); assert_eq!(s.tip_height(), Some(Height(0))); assert_eq!(s.fence_tip_height(), Some(0)); - s.confirmed.set(Height(1), Fk(2)).unwrap(); - s.header_txs.put_range(Fk(2), Fk(2), 2).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(2), 2) + .unwrap(); assert_eq!( s.tip_height(), Some(Height(1)), @@ -1142,11 +1195,17 @@ fn fence_tip_height_lags_unextended_confirmed() { fn height_fence_extend_missing_header_txs_is_not_ok_hole() { let dir = tmp(); let s = Store::create_tiny(&dir).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); s.rebuild_height_fence().unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); let err = s .height_fence_extend(Height(1), Fk(2)) .expect_err("missing header_txs must not silently skip"); @@ -1333,18 +1392,32 @@ fn class_c_disconnect_tip_first_mid_barrier_is_repairable() { { let s = Store::create_tiny(&dir).unwrap(); // Tip 0 + tip 1 fully durable. - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - s.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); - s.header_txs.put_range(Fk(2), Fk(2), 2).unwrap(); - s.strong_tx.set_strong_range(Fk(2), 2, Fk(2)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(2), 2) + .unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), 2, Fk(2)) + .unwrap(); s.rebuild_height_fence().unwrap(); s.flush_class_c_tip().unwrap(); assert_eq!(s.confirmed.tip_height(), Some(Height(1))); // Disconnect tip-first: shrink tip + flush confirmed only (kill before unstrong). - s.confirmed.disconnect_tip(Height(1)).unwrap(); + s.confirmed + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(1)) + .unwrap(); s.flush_confirmed_only().unwrap(); // Do not unstrong — simulate kill mid-disconnect. } @@ -1372,19 +1445,35 @@ fn class_c_disconnect_full_sequence_reopen_clean() { let dir = tmp(); { let s = Store::create_tiny(&dir).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 1).unwrap(); - s.strong_tx.set_strong(Fk(1), Fk(1)).unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); - s.header_txs.put_range(Fk(2), Fk(2), 2).unwrap(); - s.strong_tx.set_strong_range(Fk(2), 2, Fk(2)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 1) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1)) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(2), 2) + .unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), 2, Fk(2)) + .unwrap(); s.rebuild_height_fence().unwrap(); s.flush_class_c_tip().unwrap(); // Production disconnect order (store half). - s.confirmed.disconnect_tip(Height(1)).unwrap(); + s.confirmed + .disconnect_tip(&rbitcoin_store::ClassCWrite::for_test(), Height(1)) + .unwrap(); s.flush_confirmed_only().unwrap(); - s.strong_tx.set_unstrong_range(Fk(2), 2).unwrap(); + s.strong_tx + .set_unstrong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), 2) + .unwrap(); s.height_fence_pop_tip(Height(1)); s.flush_class_c_after_disconnect_tip().unwrap(); } @@ -1402,12 +1491,20 @@ fn orphan_class_c_at_tip_height_not_confirmed_strong_and_repairable() { let dir = tmp(); let s = Store::create_tiny(&dir).unwrap(); // Real tip body: txs 1..=2 under header 1. - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 2).unwrap(); - s.strong_tx.set_strong_range(Fk(1), 2, Fk(1)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 2) + .unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 2, Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); // Orphan second copy: txs 3..=4 strong, not in header_txs. - s.strong_tx.set_strong_range(Fk(3), 2, Fk(99)).unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), 2, Fk(99)) + .unwrap(); s.flush_class_c_tip().unwrap(); assert!(s.is_confirmed_strong(Fk(1)).unwrap()); @@ -1447,12 +1544,22 @@ fn repair_class_c_above_tip_empty_tip_and_gapped_orphans() { "{text}" ); // Tip body 1..=2; orphans 5 and 10 (non-adjacent → two clear runs). - s.confirmed.set(Height(0), Fk(1)).unwrap(); - s.header_txs.put_range(Fk(1), Fk(1), 2).unwrap(); - s.strong_tx.set_strong_range(Fk(1), 2, Fk(1)).unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 2) + .unwrap(); + s.strong_tx + .set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 2, Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); - s.strong_tx.set_strong(Fk(5), Fk(99)).unwrap(); - s.strong_tx.set_strong(Fk(10), Fk(99)).unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(5), Fk(99)) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(10), Fk(99)) + .unwrap(); s.flush_class_c_tip().unwrap(); let n = s.repair_class_c_above_tip().unwrap(); assert_eq!(n, 2, "cleared gapped orphans"); @@ -1460,7 +1567,9 @@ fn repair_class_c_above_tip_empty_tip_and_gapped_orphans() { assert!(!s.strong_tx.is_strong(Fk(10)).unwrap()); assert!(s.is_confirmed_strong(Fk(1)).unwrap()); // Strong not on the fence (same repair as above-tip leftovers). - s.strong_tx.set_strong(Fk(20), Fk(1)).unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), Fk(20), Fk(1)) + .unwrap(); assert_eq!(s.repair_class_c_above_tip().unwrap(), 1); assert!(!s.strong_tx.is_strong(Fk(20)).unwrap()); } @@ -1594,8 +1703,12 @@ fn resolve_txid_prefers_connected_over_newer_unconnected() { assert_ne!(old, new); assert_eq!(s.get_fk_by_txid(&txid).unwrap(), Some(new)); assert_eq!(s.get_fk_by_txid_tip(&txid).unwrap(), None); - s.header_txs.put_range(Fk(1), old, 1).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), old, 1) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); assert_eq!( s.get_fk_by_txid_tip(&txid).unwrap(), @@ -1637,9 +1750,15 @@ fn resolve_txid_prefers_connected_over_newer_unconnected() { true, ) .unwrap()[0]; - s.header_txs.put_range(Fk(2), spender, 1).unwrap(); - s.confirmed.set(Height(1), Fk(2)).unwrap(); - s.strong_tx.set_strong(spender, Fk(2)).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), spender, 1) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(2)) + .unwrap(); + s.strong_tx + .set_strong(&rbitcoin_store::ClassCWrite::for_test(), spender, Fk(2)) + .unwrap(); s.rebuild_height_fence().unwrap(); s.put_spend_create(old, 0, spender, 0).unwrap(); assert!( @@ -1737,8 +1856,12 @@ fn tip_then_any_connected_in_cold_beats_unconnected_hot() { None ); - s.header_txs.put_range(Fk(1), old, 1).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), old, 1) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); s.rebuild_height_fence().unwrap(); assert_eq!(s.get_fk_by_txid_tip(&txid).unwrap(), Some(old)); @@ -1798,12 +1921,22 @@ fn create_does_not_write_tx_height_and_fence_has_reorg_holes() { !dir.join("tx_height.body").exists(), "live create must not write tx_height.body" ); - s.header_txs.put_range(Fk(1), Fk(1), 2).unwrap(); - s.confirmed.set(Height(0), Fk(1)).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), Fk(1), 2) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(0), Fk(1)) + .unwrap(); // Discarded block used fks 3..=5 under header 2 (not confirmed). - s.header_txs.put_range(Fk(2), Fk(3), 3).unwrap(); - s.header_txs.put_range(Fk(3), Fk(6), 2).unwrap(); - s.confirmed.set(Height(1), Fk(3)).unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(2), Fk(3), 3) + .unwrap(); + s.header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(3), Fk(6), 2) + .unwrap(); + s.confirmed + .set(&rbitcoin_store::ClassCWrite::for_test(), Height(1), Fk(3)) + .unwrap(); s.rebuild_height_fence().unwrap(); assert_eq!(s.tx_height_get(Fk(1)).unwrap(), Some(0)); assert_eq!(s.tx_height_get(Fk(3)).unwrap(), None); diff --git a/crates/rbitcoin-test/tests/scenarios.rs b/crates/rbitcoin-test/tests/scenarios.rs index 8f34bb7e64..81dc607c8d 100644 --- a/crates/rbitcoin-test/tests/scenarios.rs +++ b/crates/rbitcoin-test/tests/scenarios.rs @@ -1373,7 +1373,12 @@ fn confirm_survives_partial_class_c_without_tip_advance() { let first = tx_fks[0]; q.store() .strong_tx - .set_strong_range(first, tx_fks.len() as u32, header_fk) + .set_strong_range( + &rbitcoin_store::ClassCWrite::for_test(), + first, + tx_fks.len() as u32, + header_fk, + ) .unwrap(); assert_eq!(q.tip_height(), Some(tip_before)); assert!( @@ -1417,7 +1422,12 @@ fn confirm_survives_partial_class_c_without_tip_advance() { let fks2 = q.store().header_txs.get_list(hfk2).unwrap().unwrap(); q.store() .strong_tx - .set_strong_range(fks2[0], fks2.len() as u32, hfk2) + .set_strong_range( + &rbitcoin_store::ClassCWrite::for_test(), + fks2[0], + fks2.len() as u32, + hfk2, + ) .unwrap(); q.flush().unwrap(); drop(q); @@ -1608,7 +1618,10 @@ fn pin_plan_none_stamp_of_archived_spend( ); let n = fks.len() as u32; - q.store().header_txs.put_range(hfk, fks[0], n + 1).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), hfk, fks[0], n + 1) + .unwrap(); match confirm_wire_lookup_stamp(q, params, ms, &items, None) { Err(rbitcoin_query::ConfirmError::Store(StoreError::Corrupt(m))) => { assert_eq!(m, "invariant: archived stamp tx_fks/txids length"); @@ -1616,7 +1629,10 @@ fn pin_plan_none_stamp_of_archived_spend( Err(e) => panic!("list/wire length mismatch must be Corrupt, got {e}"), Ok(_) => panic!("list/wire length mismatch must fail the stamp"), } - q.store().header_txs.put_range(hfk, fks[0], n).unwrap(); + q.store() + .header_txs + .put_range(&rbitcoin_store::ClassCWrite::for_test(), hfk, fks[0], n) + .unwrap(); } /// Resume: spend archived with create_fk (archive sticky/head); confirm spends. diff --git a/docs/concurrency.md b/docs/concurrency.md index 963454b9b8..62c9e3767b 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -10,8 +10,8 @@ Short map of who may write which tables. **Format is unstable until 1.0.** | Confirm **lookup** | 1 OS thread | pack/hold from BQ stamped Σ inputs; emit: decode BQ raw + TipOnly `head_fk` + **`take_raw` onto loadq** (`Arc` + `TxPrecompute`). Does **not** plan_batch / structure | | Confirm **load** | 1 OS thread | `confirm_wire_lookup_stamp` (structure with **loadq `pres`**, plan_batch binds carried BQ keys, skeleton bind) + pin `txout` + assemble | | Confirm **scripts** | 1 OS thread (`ibd-confirm`) publishes waves + `rbtc-scripts` steal (lock-free claim). The interpreter runs here, not on a peer IO task (`verify_tx_scripts_detached`) | **none** — pure CPU | -| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` + one `rbtc-tip-flush` for the confirm engine (the Class C barrier) | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era on a clean run; no archive-ahead DONTNEED). A crash inside the era can leave it above the tip ([crash-recovery.md](./crash-recovery.md#class-a-archive)). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | -| IBD main loop | 1 tokio task | none (orchestration only). Event drain + confirm-offer every turn; getdata assign ≤50 ms (immediate if inflight empty); main-loop `getheaders`/`locator_hashes` ≤500 ms (empty path immediate); peer stall/relative-slow and work-path hygiene ≤1 s. Full-batch header continuation stays on the Headers event. Tip hash is the snapshot confirm published with `confirmed[]` (`published_tip`); each admitted header's parent stays on the work state. Those two reads are not store IO. The write holds `tip_commit` from the tip-height check through `flush_class_c_tip`; in IBD, through starting it on `rbtc-tip-flush`, and the next `tip_commit` taker joins that flush first. IBD rewind tries that lock, and treats a flush still running as held, and retries on the next pass. | +| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` + one `rbtc-tip-flush` for the confirm engine (the Class C barrier) | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C (`ClassCWrite`, minted on this thread, not `Send`) + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. Archived-body clear is an item on this thread's channel: it joins the in-flight tip flush, then clears and flushes, before the next Class C store. Header range puts on this path wait for that same join, including when a later phase rejects the batch. Class A body append still overlaps the flush. Class C still flushes once per confirm batch. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era on a clean run; no archive-ahead DONTNEED). A crash inside the era can leave it above the tip ([crash-recovery.md](./crash-recovery.md#class-a-archive)). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | +| IBD main loop | 1 tokio task | none (orchestration only; archived-body clear is sent to confirm-write, which stores it). Event drain + confirm-offer every turn; getdata assign ≤50 ms (immediate if inflight empty); main-loop `getheaders`/`locator_hashes` ≤500 ms (empty path immediate); peer stall/relative-slow and work-path hygiene ≤1 s. Full-batch header continuation stays on the Headers event. Tip hash is the snapshot confirm published with `confirmed[]` (`published_tip`); each admitted header's parent stays on the work state. Those two reads are not store IO. The write holds `tip_commit` from the tip-height check through `flush_class_c_tip`; in IBD, through starting it on `rbtc-tip-flush`, and the next `tip_commit` taker joins that flush first. IBD rewind tries that lock, and treats a flush still running as held, and retries on the next pass. | **IoSession TLS:** one completion session per OS thread (`with_thread_local`). Harvest / poison / drain / do-not-flatten: [`io-modality.md`](./io-modality.md). `RBITCOIN_IO=pread` disables the session. SH k-way merge submits 256 KiB ahead preads on that TLS session and waits only when promote needs a page that has not completed. @@ -50,7 +50,7 @@ Three thread kinds only. **Tokio workers must not wait on a `std` mutex/rwlock, | Role | Notes | |------|--------| | `peer_session` (split read/write) | Serve getdata. Full-block, compact, and `getblocktxn` reconstruct run on the blocking pool (block-decode permit). Inbound `cmpctblock` reconstruct uses that same permit. `headers` persist and `getheaders` / `getblocks` replies run on the blocking pool under the header-decode permit. The session applies notes, ban, and queued `getdata` after that join. Offers reconstructed blocks to **`tip-accept`** (does **not** take `connect_lock` or run confirm on the tokio worker). A reconstructed/received body whose header has valid PoW and **extends the current tip** is announced as `cmpctblock` to peers who sent `sendcmpct` announce=1 and already have the parent **before** connect (Core `NewPoWValidBlock`). That is not a connected tip. P2P `tx` is `accept_tx_async` (blocking pool). INV / getdata / compact **first-pass** fill use mempool **`try_read` only** (busy write → skip that item; never park). A pending compact owns those clones; `blocktxn` apply does not `try_read` again. Handshake `FeeFilter` reads the **atomic** published fee floor (no `inner`). 50 ms tick: age-INV is a due-log cursor (newly due only); `clock_due` / unbroadcast may full-walk ≤1/30 s. `any_tx_inv_due` is min live `accept_at`, not a map walk. `PeerHub::on_session_heartbeat` (headers-sync stall timeout). While the writer is past the 4 MiB send budget, or a paused `getdata` tail waits for serve room, the reader reads no frame; the heartbeat still runs, so the ping timeout still fires. Inbound accept is `inbound_connect_and_handshake` (60 s VERSION/VERACK); timeout drops the `max_inbound` permit. | -| `tip-accept` | **One** process-wide OS thread. Queue depth 8. Sole production thread for `accept_block` / `accept_branch` / `accept_received_block` / `generate_to_script` / `connect_lock` at tip. Confirm is **`confirm_wire_run_preverified`** (lookup stamp, load pin/assemble, `confirm_scripts_phase` → `rbtc-scripts-*`, write + `ibd-confirm-head` drain). TLS uring is this thread’s `with_thread_local` session. SIGINT stays on tokio; the current job finishes, then the session sees shutdown. Dropping the session’s join future **detaches** (does not condvar-wait on the worker). | +| `tip-accept` | **One** process-wide OS thread. Queue depth 8. Sole production thread for `accept_block` / `accept_branch` / `accept_received_block` / `generate_to_script` / `connect_lock` at tip. Mints `ClassCWrite` for that connect's Class C stores. Confirm is **`confirm_wire_run_preverified`** (lookup stamp, load pin/assemble, `confirm_scripts_phase` → `rbtc-scripts-*`, write + `ibd-confirm-head` drain). TLS uring is this thread’s `with_thread_local` session. SIGINT stays on tokio; the current job finishes, then the session sees shutdown. Dropping the session’s join future **detaches** (does not condvar-wait on the worker). | | `ibd-header` | **One** thread for the catch-up. Persists header batches, loads the walk origin, and reseeds. Joined when IBD stops. | | `rbtc-tip-flush` | **One** thread for the IBD confirm engine. Runs the Class C barrier. The next commit waits for the in-flight barrier. Joined when the engine stops. Tip mode flushes that barrier inline on `tip-accept`. | | `rbtc-sh-wb` | **One** Class B scripthash appender. A durable `--sh-index` head starts it before catch-up. Direct catch-up starts it only after materialize enters Tip. Absent for the whole Direct catch-up, including a cancelled or failed materialize. Confirm enqueues RAM records; `connect_at` / `note_confirmed_tip` **release** after `tip_tx`. This thread `put_create_batch_append` only for released heights, then advances `sh_indexed_through`. Apply errors re-queue and halt. The Class A tail holds `sh.appender` and finishes inside tip entry before this thread starts. The same lock is the exclusion if a tail and the worker ever overlap. There is no process-local head map. | diff --git a/docs/crash-recovery.md b/docs/crash-recovery.md index f45aee21a6..a1cc6ba2c5 100644 --- a/docs/crash-recovery.md +++ b/docs/crash-recovery.md @@ -98,7 +98,7 @@ Kill mid-payload before HWM publish: readers never see past previous published l Connect barrier (`flush_class_c_tip`): headers (if dirty) → strong → header_txs → **confirmed last** → soft `tip_seal` (tmp + `sync_all` + rename + parent-directory `fsync`). The height fence is RAM-only and is not flushed. The barrier does not `sync_data` Class A. -IBD runs that barrier on one long-lived `rbtc-tip-flush` thread for the confirm engine, so the next write batch's archive and checks overlap it. Every `tip_commit` taker joins it first: the next commit, a disconnect or rewind, `Query::flush`, `flush_for_shutdown`, and dropping the `Query`. The order on disk does not change. A kill can lose one more write batch than in Tip mode. Its body-queue rows are already gone (RAM only, fetched again), and its spend annotations and filter or tweak puts may sit above the durable tip: open replays annotations and trims both indexes to the tip. A failed IBD barrier is not retried, because a failed `fsync` may have dropped the dirty pages. Every later commit fails until restart, so IBD halts after its one engine-fault retry. Tip mode runs the barrier inline. +IBD runs that barrier on one long-lived `rbtc-tip-flush` thread for the confirm engine, so the next write batch's Class A body append and checks overlap it. Header range puts and the other Class C stores wait until that flush is joined. A structural reject after the body append still stores those header ranges after the join, so a retry sees `header_txs` and does not append the body again. Class C still flushes once per batch. Every `tip_commit` taker joins it first: the next commit, a disconnect or rewind, `Query::flush`, `flush_for_shutdown`, and dropping the `Query`. The order on disk does not change. A kill can lose one more write batch than in Tip mode. Its body-queue rows are already gone (RAM only, fetched again), and its spend annotations and filter or tweak puts may sit above the durable tip: open replays annotations and trims both indexes to the tip. A failed IBD barrier is not retried, because a failed `fsync` may have dropped the dirty pages. Every later commit fails until restart, so IBD halts after its one engine-fault retry. Tip mode runs the barrier inline. Disconnect: confirmed truncate + `flush_confirmed_only` (also refreshes `tip_seal`) before unstrong/height clear. If the new tip is below `spend_durable`, that marker is lowered to the tip. Confirm stores a snapshot height after annotations are written and does not `sync_data` these stems. `rbtc-spend-sync` does, about every 10 minutes: `sync_data` on the replay stems and `spenders` (not a high-water rewrite, not `tx.head`), then if the confirmed tip is still at least that snapshot, publish `A = D` at the snapshot. Blocks appended above the snapshot during the barrier are flushed along and are not claimed. A disconnect bumps an in-process generation before it lowers the snapshot and the marker, including when no marker file exists yet. The checkpoint samples that generation before `sync_data` and again under the marker lock; it publishes only when the generation is unchanged, the tip is still at least the snapshot, and no annotate is pending. A disconnect and a reconnect of a different block at the snapshot height during that window leave the previous marker in place. An append strictly above the snapshot does not bump the generation, so the marker still publishes. One still pending caps `A` below its first height. A failed marker publish leaves the previous marker and does not start a new interval. Shutdown wakes the thread for one checkpoint, joins it, then runs `flush_for_shutdown`. The barrier is `File::sync_data` (Linux `fdatasync`, macOS `F_FULLFSYNC`, Windows `FlushFileBuffers`). It is not inside `flush_class_c_tip`, and it is not charged on the confirm thread's `ann_sync` timer. A cookie inside a data file, or a clean page cache, is not this cursor: writeback is not a device flush, and the kernel may reorder data ahead of a later header write. From 656e146d5d61c618b109f1c6d855649cfa4a4ae4 Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Mon, 26 Oct 2026 06:44:16 -0700 Subject: [PATCH 03/20] store: Class C gets use a sequence counter The writer takes an odd step, a release fence, then the slot stores. The even advance publishes them. A sequentially consistent fence keeps a store out of the flush it races. strong_tx bits live in the same slot image. Growth keeps the previous buffer until the table drops. An fd-only store holds no odd guard. --- CONTRIBUTING.md | 5 +- Cargo.toml | 2 +- changelog.d/class-c-snapshot.md | 15 + crates/rbitcoin-query/src/scripthash.rs | 7 +- crates/rbitcoin-store/src/array_table.rs | 591 +++++++++++------- crates/rbitcoin-store/src/chain.rs | 546 +++++++--------- crates/rbitcoin-store/src/lib.rs | 2 + crates/rbitcoin-store/src/store.rs | 14 +- crates/rbitcoin-store/src/sync.rs | 22 + crates/rbitcoin-store/tests/array_table.rs | 86 ++- .../tests/chain/chain_table_tests.rs | 41 ++ .../tests/chain/strong_tests.rs | 70 ++- docs/concurrency.md | 28 +- docs/ibd-memory.md | 2 +- docs/io-modality.md | 5 +- 15 files changed, 867 insertions(+), 569 deletions(-) create mode 100644 crates/rbitcoin-store/src/sync.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aff98e1132..2d45716395 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -211,8 +211,9 @@ engineering. packed `g` (`mphf_g=0`) versus mmap sealed `.fuse8` (`fuse8=0` heap) and SH BDZ3 occupancy (`mphf_occ=` supers only; occ bits are `file=`) is that kind of choice. Reclaim of mapped fuse/occ is drop of file pages, - not swap. `strong_tx` and mempool stay process `Vec` (Class C tip-last write-behind - / live graph); do not mmap those because fuse mapping looked good. + not swap. `strong_tx` stays a process-resident bit image and mempool stays a + process `Vec` (Class C tip-last write-behind / live graph); do not mmap those + because fuse mapping looked good. Write as if the structure is huge, because in IBD it is. **Address** it (hash, slot, page, bit offset, subscript). Do not walk everything resident diff --git a/Cargo.toml b/Cargo.toml index 4da56da3e8..8f9da39bde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ warnings = "deny" unsafe_op_in_unsafe_fn = "deny" # cargo-llvm-cov sets `cfg(coverage)` so diagnostic microbenches can use # `#[cfg(not(coverage))]` and stay out of the first-party LCOV denominator. -unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)'] } +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(loom)', 'cfg(shuttle)'] } # Clippy: CI is `clippy --workspace --all-targets -- -D warnings`. # No workspace allow list. A leftover lint is `#[allow(clippy::…)]` on diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index c02c719f56..65f5437dda 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -9,3 +9,18 @@ Changed stores the header ranges after the join, so a retry sees the body. Class A device sync stays on `rbtc-spend-sync` about every 10 minutes. Class C still flushes once per confirm batch. +- **In-RAM Class C gets take no reader-writer lock.** One sequence counter + covers a store. The odd bit is sequentially consistent, a release fence + runs before the slot stores, and the even advance publishes them. Header + pairs, which have no flush flag, still take a relaxed odd step and that + same release fence. A get acquire-fences before it checks the counter + again. `header_txs` reads first and count under one counter. A header + column load acquires that store, then the same fence checks the counter. + `strong_tx` keeps its bits in that slot buffer. One flush serves the + arrays and the bit image. The bit image's length is the body in bytes. + A flush excludes new stores, then reads the dirty range. The flushing + flag is sequentially consistent in both directions, with a sequentially + consistent fence between the odd bit and the flag, so a store cannot + pass the flag and still be marked clean. The flush does not bump the + counter. Growth keeps the previous buffer. A get retries when the + length it sampled does not fit that buffer. diff --git a/crates/rbitcoin-query/src/scripthash.rs b/crates/rbitcoin-query/src/scripthash.rs index 54cb46e45c..52a2e39da7 100644 --- a/crates/rbitcoin-query/src/scripthash.rs +++ b/crates/rbitcoin-query/src/scripthash.rs @@ -646,6 +646,10 @@ impl Query { }; #[cfg(debug_assertions)] let _probe = admit.as_ref().map(|a| self.sh_join_probe.enter(a.heavy)); + // Release builds have no probe. Moving the guard into `_admit` keeps + // it until this function returns. Debug drops `_probe` first. + #[cfg(not(debug_assertions))] + let _admit = admit; let t_pages = std::time::Instant::now(); let entries = self.store.scripthash.create_fks(scripthash)?; let pages_us = t_pages.elapsed().as_micros(); @@ -714,7 +718,8 @@ impl Query { spends_us ); } - drop(admit); + // Debug drops `_probe` before `admit`. Releasing the heavy lock first + // lets the next heavy join enter the probe while this one is still counted. Ok(out) } diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 953e682518..89d8249771 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -1,58 +1,30 @@ -//! Growable dense array of little-endian `u64` values. +//! Growable dense array of little-endian `u64` values, 0-based. //! -//! 0-based indexing. Used for Class C: `confirmed[height]`, `header_txs` arrays, etc. -//! -//! # L2 write-behind (phase 6) -//! -//! Compact **array** tables (`confirmed`, `header_txs_*`; body ≤ -//! [`class_c_inram_max_bytes`]) load fully into process RAM on open. -//! `strong_tx` is always L2 (see [`crate::chain::StrongTxTable`]). -//! Mutates update the `Vec` only and record the lowest unstored index. Disk -//! is updated on `Self::flush` / barrier by reading those live slots: -//! -//! - **Append** (unstored index ≥ last flushed len): pwrite the new suffix. -//! - **Truncate**: shrink the published length only. -//! - **In-prefix mutate**: pwrite from the first unstored index through the -//! length recorded when the flush started, then publish that length even -//! when it equals the previous disk length. -//! -//! Each pwrite is at most [`FLUSH_SCRATCH`] bytes. The unstored mark clears -//! only after that persist succeeds. A store that starts after the flush has -//! recorded its range is refused until the flush returns. -//! -//! Tables over the cap stay pure fd L0 (per-slot write-through). The flush -//! does not read that path. +//! `confirmed` and `header_txs_*` load into RAM up to [`class_c_inram_max_bytes`]; +//! `strong_tx` is always in RAM. Stores update live slots. `flush` writes those +//! slots in [`FLUSH_SCRATCH`] chunks and clears the unstored mark only after the +//! persist succeeds. A clean table returns before new stores are excluded. Over +//! the cap, tables stay fd write-through. In-RAM gets take no reader-writer lock. +//! Counter order is the Class C paragraph of `docs/concurrency.md`. use crate::error::StoreError; use crate::file::{TableFile, FILE_HEADER_LEN}; +use crate::sync::atomic::{self, AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering}; use rbitcoin_primitives::TableKind; use std::marker::PhantomData; use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Mutex, RwLock}; +use std::sync::Mutex; -/// Capability for the one Class C writer. -/// -/// `!Send` and `!Sync`: a shared [`crate::Store`] cannot carry this onto -/// another thread. Confirm-write, tip-accept, and startup each mint one on -/// the thread that stores. The flush thread reads live slots and does not -/// take this token. -pub struct ClassCWrite { - _thread: PhantomData<*const ()>, -} - -impl Default for ClassCWrite { - fn default() -> Self { - Self::new() - } -} +/// One Class C writer (`!Send` + `!Sync`). A shared [`crate::Store`] cannot carry +/// it onto another thread. Confirm-write, tip-accept, and startup each mint one +/// on the storing thread. The flush thread reads live slots and does not take it. +#[derive(Default)] +pub struct ClassCWrite(PhantomData<*const ()>); impl ClassCWrite { /// Mint a writer capability on the calling thread. pub fn new() -> Self { - Self { - _thread: PhantomData, - } + Self(PhantomData) } /// Same as [`Self::new`]. Tests are not the confirm-write thread. @@ -71,12 +43,15 @@ pub const FLUSH_SCRATCH: usize = 4096; const CLEAN: u64 = u64::MAX; -/// Where a flush calls its test hook: after the range is recorded, before the -/// write; or after the persist, before the unstored mark clears. +/// Where a flush calls its test hook. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[doc(hidden)] pub enum FlushStep { + /// Dirty, before new stores are excluded. A store here belongs in this flush. + Admit, + /// After the range is recorded, before the write. Write, + /// After the persist, before the unstored mark clears. Clean, } @@ -96,37 +71,215 @@ pub fn class_c_inram_max_bytes() -> u64 { } } -/// Pwrite `len` bytes at `base` in [`FLUSH_SCRATCH`] chunks. `fill` loads one -/// chunk from the live image. Returns the largest chunk written. -pub(crate) fn pwrite_scratched( - mut write: impl FnMut(u64, &[u8]) -> Result<(), StoreError>, - base: u64, - len: u64, - mut fill: impl FnMut(&mut [u8], u64) -> Result<(), StoreError>, -) -> Result { - let mut scratch = [0u8; FLUSH_SCRATCH]; - let mut off = 0u64; - let mut max_chunk = 0usize; - while off < len { - let n = ((len - off) as usize).min(FLUSH_SCRATCH); - fill(&mut scratch[..n], off)?; - max_chunk = max_chunk.max(n); - write(base + off, &scratch[..n])?; - off += n as u64; - } - Ok(max_chunk) +/// Copy `dst.len()` bytes from packed words at byte `start`. +pub(crate) fn copy_bit_bytes(words: &[AtomicU64], start: usize, dst: &mut [u8]) { + let mut i = 0; + while i < dst.len() { + let abs = start + i; + let packed = words[abs / 8].load(Ordering::Relaxed).to_le_bytes(); + let off = abs % 8; + let n = (8 - off).min(dst.len() - i); + dst[i..i + n].copy_from_slice(&packed[off..off + n]); + i += n; + } +} + +struct Buf { + slots: Box<[AtomicU64]>, + /// Previous buffer. Growth publishes `slots` first; this box keeps the old + /// pointer until the table drops. + prev: Option>, +} + +/// In-RAM `u64` slots. Growth keeps the previous buffer on the new box. +pub(crate) struct RamSlots { + ptr: AtomicPtr, +} + +impl Drop for RamSlots { + fn drop(&mut self) { + let p = self.ptr.load(Ordering::Relaxed); + if p.is_null() { + return; + } + // SAFETY: `p` came from `Box::into_raw`. Dropping `prev` frees every + // buffer this table published. + drop(unsafe { Box::from_raw(p) }.prev); + } +} + +impl RamSlots { + pub(crate) fn empty() -> Self { + Self { + ptr: AtomicPtr::new(std::ptr::null_mut()), + } + } + + pub(crate) fn from_values(values: Vec) -> Self { + let buf = Box::new(Buf { + slots: values.into_iter().map(AtomicU64::new).collect(), + prev: None, + }); + Self { + ptr: AtomicPtr::new(Box::into_raw(buf)), + } + } + + pub(crate) fn grow(&self, need: usize) { + let cur = self.ptr.load(Ordering::Relaxed); + // SAFETY: `cur` is null or a box this writer published. The copy finishes + // before `from_raw` moves that box onto the new buffer. + let old = unsafe { cur.as_ref().map(|buf| &buf.slots[..]) }; + let cap = old.map_or(0, |slots| slots.len()); + if old.is_some() && need <= cap { + return; + } + let new_cap = if cap == 0 { + need.max(1).next_power_of_two() + } else { + cap.saturating_mul(2).max(need) + }; + let slots: Box<[AtomicU64]> = (0..new_cap) + .map(|i| { + let v = old + .and_then(|s| s.get(i)) + .map_or(0, |a| a.load(Ordering::Relaxed)); + AtomicU64::new(v) + }) + .collect(); + let prev = (!cur.is_null()).then(|| { + // SAFETY: `cur` came from `Box::into_raw` on this writer. + unsafe { Box::from_raw(cur) } + }); + let buf = Box::into_raw(Box::new(Buf { slots, prev })); + self.ptr.store(buf, Ordering::Release); + } + + /// Writer (Relaxed) or flush (Acquire), after the buffer covers the index. + pub(crate) fn slots(&self, order: Ordering) -> &[AtomicU64] { + let ptr = self.ptr.load(order); + // SAFETY: caller indexes a buffer this table still owns. + unsafe { &(*ptr).slots } + } + + /// `None` when the published buffer is missing or shorter than `need`. + /// A torn length can name a slot this buffer does not have. + pub(crate) fn try_slots(&self, need: usize) -> Option<&[AtomicU64]> { + let ptr = self.ptr.load(Ordering::Acquire); + // SAFETY: growth keeps every published buffer until this table drops. + let buf = unsafe { ptr.as_ref() }?; + (buf.slots.len() >= need).then_some(&buf.slots) + } +} + +/// Evens the counter on drop. The raw pointer keeps the guard from holding +/// a field borrow across the slot stores. An fd-only store holds `None`. +pub(crate) struct SeqOdd(*const AtomicU64); + +impl Drop for SeqOdd { + fn drop(&mut self) { + // SAFETY: the pointer is the counter just borrowed. The guard does not outlive that table. + unsafe { &*self.0 }.fetch_add(1, Ordering::Release); + } +} + +/// Order is the Class C paragraph of `docs/concurrency.md`. Loom 0.7 does +/// not order `SeqCst` accesses; the two `SeqCst` fences do. +pub(crate) fn seq_begin(seq: &AtomicU64, flushing: &AtomicBool) -> Result { + if flushing.load(Ordering::SeqCst) { + return Err(StoreError::Rejected("class C store during flush")); + } + seq.fetch_add(1, Ordering::SeqCst); + atomic::fence(Ordering::SeqCst); + if flushing.load(Ordering::SeqCst) { + seq.fetch_add(1, Ordering::SeqCst); + return Err(StoreError::Rejected("class C store during flush")); + } + atomic::fence(Ordering::Release); + Ok(SeqOdd(seq as *const AtomicU64)) +} + +/// Header pairs have no flush flag. Same odd-then-fence order as [`seq_begin`]. +pub(crate) fn seq_odd(seq: &AtomicU64) -> SeqOdd { + seq.fetch_add(1, Ordering::Relaxed); + atomic::fence(Ordering::Release); + SeqOdd(seq as *const AtomicU64) +} + +pub(crate) fn seq_arm(seq: &AtomicU64, flushing: &AtomicBool) { + loop { + let g1 = seq.load(Ordering::SeqCst); + if g1 & 1 != 0 { + crate::sync::spin(); + continue; + } + flushing.store(true, Ordering::SeqCst); + atomic::fence(Ordering::SeqCst); + if seq.load(Ordering::SeqCst) == g1 { + return; + } + while seq.load(Ordering::SeqCst) & 1 != 0 { + crate::sync::spin(); + } + } +} + +/// On success, disk length and the clean mark publish before `flushing` clears. +/// Fields are the flag, the disk length, the unstored mark, and the persisted length. +#[must_use] +struct FlushGuard<'a>(&'a AtomicBool, &'a AtomicU64, &'a AtomicU64, Option); + +impl Drop for FlushGuard<'_> { + fn drop(&mut self) { + if let Some(n) = self.3 { + self.1.store(n, Ordering::Release); + self.2.store(u64::MAX, Ordering::Release); + } + self.0.store(false, Ordering::SeqCst); + } +} + +/// Acquire fence, then the counter. See `docs/concurrency.md`. +pub(crate) fn seq_holds(seq: &AtomicU64, g1: u64) -> bool { + atomic::fence(Ordering::Acquire); + seq.load(Ordering::Relaxed) == g1 +} + +/// `sample` returns `None` to retry this even generation. `Some` is accepted +/// only when the counter is still `g1` after the acquire fence. +pub(crate) fn seq_read(seq: &AtomicU64, mut sample: impl FnMut() -> Option) -> T { + loop { + let g1 = seq.load(Ordering::Acquire); + if g1 & 1 != 0 { + crate::sync::spin(); + continue; + } + if let Some(v) = sample() { + if seq_holds(seq, g1) { + return v; + } + } else if seq_holds(seq, g1) { + crate::sync::spin(); + } + } } pub struct ArrayTable { pub file: TableFile, pub len: AtomicU64, - /// `Some` = L2 authoritative image; `None` = L0 fd-only path. - pub data: RwLock>>, - /// Lowest index not yet persisted. [`CLEAN`] means none. + /// False = L0 fd-only. Never changes after `new`. + in_ram: bool, + /// `len` and `disk_len` count body bytes, and `unstored_from` is a bit. + /// Slot tables count elements. `strong_tx` is the byte table. + byte_body: bool, + pub(crate) slots: RamSlots, + /// Even means the slots and `len` published together are complete. + pub(crate) seq: AtomicU64, + /// Lowest index not yet persisted. `u64::MAX` means none. pub unstored_from: AtomicU64, /// True from the moment a flush records its range until it returns. pub flushing: AtomicBool, - /// Element count last successfully flushed to disk (L2). + /// Length last flushed. Elements, or body bytes when `byte_body` is set. pub disk_len: AtomicU64, /// One flusher at a time. pub flush_lock: Mutex<()>, @@ -135,28 +288,10 @@ pub struct ArrayTable { pub flush_chunk_max: AtomicUsize, } -/// Clears `flushing` on drop. On success, also publishes `disk_len` and -/// clears the unstored mark, in that order, while stores are still refused. -#[must_use] -struct EndFlush<'a> { - table: &'a ArrayTable, - persisted: Option, -} - -impl Drop for EndFlush<'_> { - fn drop(&mut self) { - if let Some(n) = self.persisted { - self.table.disk_len.store(n, Ordering::Release); - self.table.unstored_from.store(CLEAN, Ordering::Release); - } - self.table.flushing.store(false, Ordering::Release); - } -} - impl ArrayTable { pub fn create(path: impl AsRef, kind: TableKind) -> Result { let file = TableFile::create(path.as_ref(), kind)?; - Ok(Self::new(file, Some(Vec::new()), 0)) + Ok(Self::new(file, Some(Vec::new()), 0, false)) } pub fn open(path: impl AsRef, kind: TableKind) -> Result { @@ -179,14 +314,23 @@ impl ArrayTable { } else { None }; - Ok(Self::new(file, data, n)) + Ok(Self::new(file, data, n, false)) + } + + /// In-RAM bit image. `bytes` is the body length and is not rounded to a word. + pub(crate) fn bit_image(file: TableFile, bytes: u64) -> Self { + Self::new(file, None, bytes, true) } - fn new(file: TableFile, data: Option>, n: u64) -> Self { + fn new(file: TableFile, data: Option>, n: u64, byte_body: bool) -> Self { + let in_ram = byte_body || data.is_some(); Self { file, len: AtomicU64::new(n), - data: RwLock::new(data), + in_ram, + byte_body, + slots: data.map_or_else(RamSlots::empty, RamSlots::from_values), + seq: AtomicU64::new(0), unstored_from: AtomicU64::new(CLEAN), flushing: AtomicBool::new(false), disk_len: AtomicU64::new(n), @@ -208,12 +352,11 @@ impl ArrayTable { FILE_HEADER_LEN as u64 + index * ELEM } - fn refuse_while_flushing(&self) -> Result<(), StoreError> { - if self.flushing.load(Ordering::Acquire) { - Err(StoreError::Rejected("class C store during flush")) - } else { - Ok(()) + pub(crate) fn begin_store(&self) -> Result, StoreError> { + if self.in_ram { + return seq_begin(&self.seq, &self.flushing).map(Some); } + Ok(None) } fn mark_unstored(&self, index: u64) { @@ -221,47 +364,63 @@ impl ArrayTable { } pub fn get(&self, index: u64) -> Result { - let len = self.len.load(Ordering::Acquire); - if index >= len { - return Ok(0); - } - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - if let Some(ref v) = *guard { - // `len` was read before the lock; a truncate may have landed since. - return Ok(v.get(index as usize).copied().unwrap_or(0)); + Ok(self.get_sample(index)?.0) + } + + /// One slot, with no sequence counter. A header pair checks its own counter + /// after both columns; an inner check would accept one column early. + pub(crate) fn peek_slot(&self, index: u64) -> Result, StoreError> { + if !self.in_ram { + return Ok(Some(self.get(index)?)); } - drop(guard); - let mut buf = [0u8; 8]; - self.file.read_at(Self::offset(index), &mut buf)?; - Ok(u64::from_le_bytes(buf)) + Ok(self.sample_ram(index, Ordering::Acquire).map(|(v, _)| v)) } - pub fn set(&self, _write: &ClassCWrite, index: u64, value: u64) -> Result<(), StoreError> { - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - if let Some(ref mut v) = *guard { - let need = index as usize + 1; - if v.len() < need { - v.resize(need, 0); + /// `(value, len)` from one even generation. `get` is the value. + #[doc(hidden)] + pub fn get_sample(&self, index: u64) -> Result<(u64, u64), StoreError> { + if !self.in_ram { + let len = self.len.load(Ordering::Acquire); + if index >= len { + return Ok((0, len)); } - v[index as usize] = value; - self.len.store(v.len() as u64, Ordering::Release); - self.mark_unstored(index); - return Ok(()); + let mut buf = [0u8; 8]; + self.file.read_at(Self::offset(index), &mut buf)?; + return Ok((u64::from_le_bytes(buf), len)); } - drop(guard); - let len = self.len.load(Ordering::Acquire); + Ok(seq_read(&self.seq, || { + self.sample_ram(index, Ordering::Relaxed) + })) + } + + /// `None` when the published buffer is shorter than `index`. Past `len` is + /// `(0, len)`, not a missing slot. + fn sample_ram(&self, index: u64, order: Ordering) -> Option<(u64, u64)> { + let len = self.len.load(Ordering::Relaxed); if index >= len { - for i in len..index { - self.file.write_at(Self::offset(i), &0u64.to_le_bytes())?; + return Some((0, len)); + } + let slots = self.slots.try_slots(index as usize + 1)?; + Some((slots[index as usize].load(order), len)) + } + + pub fn set(&self, _write: &ClassCWrite, index: u64, value: u64) -> Result<(), StoreError> { + let _odd = self.begin_store()?; + if !self.in_ram { + let len = self.len.load(Ordering::Acquire); + if index >= len { + for i in len..index { + self.file.write_at(Self::offset(i), &0u64.to_le_bytes())?; + } } self.file .write_at(Self::offset(index), &value.to_le_bytes())?; - self.len.store(index + 1, Ordering::Release); - } else { - self.file - .write_at(Self::offset(index), &value.to_le_bytes())?; + if index >= len { + self.len.store(index + 1, Ordering::Release); + } + return Ok(()); } + self.store_ram(index, index, &[(index, value)]); Ok(()) } @@ -271,42 +430,45 @@ impl ArrayTable { } let max_idx = pairs.iter().map(|(i, _)| *i).max().unwrap(); let min_idx = pairs.iter().map(|(i, _)| *i).min().unwrap(); - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - if let Some(ref mut v) = *guard { - let need = max_idx as usize + 1; - if v.len() < need { - v.resize(need, 0); + let _odd = self.begin_store()?; + if !self.in_ram { + let len = self.len.load(Ordering::Acquire); + if max_idx >= len { + let new_len = max_idx + 1; + let gap = new_len - len; + const CHUNK: usize = 1024 * 1024; + let mut offset = Self::offset(len); + let mut remaining = gap * ELEM; + let zeros = vec![0u8; CHUNK]; + while remaining > 0 { + let n = remaining.min(zeros.len() as u64) as usize; + self.file.write_at(offset, &zeros[..n])?; + offset += n as u64; + remaining -= n as u64; + } + self.len.store(new_len, Ordering::Release); } for &(index, value) in pairs { - v[index as usize] = value; + self.file + .write_at(Self::offset(index), &value.to_le_bytes())?; } - self.len.store(v.len() as u64, Ordering::Release); - self.mark_unstored(min_idx); return Ok(()); } - drop(guard); - let len = self.len.load(Ordering::Acquire); - if max_idx >= len { - let new_len = max_idx + 1; - let gap = new_len - len; - const CHUNK: usize = 1024 * 1024; - let mut offset = Self::offset(len); - let mut remaining = gap * ELEM; - let zeros = vec![0u8; CHUNK]; - while remaining > 0 { - let n = remaining.min(zeros.len() as u64) as usize; - self.file.write_at(offset, &zeros[..n])?; - offset += n as u64; - remaining -= n as u64; - } - self.len.store(new_len, Ordering::Release); - } + self.store_ram(min_idx, max_idx, pairs); + Ok(()) + } + + /// Release-store each slot, then the length. A relaxed slot store can move + /// one header column while that pair's counter still looks even. + fn store_ram(&self, min_idx: u64, max_idx: u64, pairs: &[(u64, u64)]) { + self.slots.grow(max_idx as usize + 1); + let slots = self.slots.slots(Ordering::Relaxed); for &(index, value) in pairs { - self.file - .write_at(Self::offset(index), &value.to_le_bytes())?; + slots[index as usize].store(value, Ordering::Release); } - Ok(()) + let len = self.len.load(Ordering::Relaxed).max(max_idx + 1); + self.len.store(len, Ordering::Relaxed); + self.mark_unstored(min_idx); } pub fn truncate(&self, _write: &ClassCWrite, new_len: u64) -> Result<(), StoreError> { @@ -314,28 +476,39 @@ impl ArrayTable { if new_len > len { return Err(StoreError::Corrupt("array truncate grows")); } - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - if let Some(ref mut v) = *guard { - v.truncate(new_len as usize); + let _odd = self.begin_store()?; + if !self.in_ram { self.len.store(new_len, Ordering::Release); - // Length-only publish. Mark the start so flush takes the shrink path. - self.mark_unstored(0); + let logical = FILE_HEADER_LEN as u64 + new_len * ELEM; + self.file.set_logical_len(logical)?; return Ok(()); } - drop(guard); - self.len.store(new_len, Ordering::Release); - let logical = FILE_HEADER_LEN as u64 + new_len * ELEM; - self.file.set_logical_len(logical)?; + self.len.store(new_len, Ordering::Relaxed); + self.mark_unstored(0); Ok(()) } /// In-RAM L2 image bytes (0 when FdOnly). pub fn l2_resident_bytes(&self) -> u64 { - let g = self.data.read().unwrap_or_else(|e| e.into_inner()); - g.as_ref() - .map(|v| (v.len() as u64).saturating_mul(ELEM)) - .unwrap_or(0) + if !self.in_ram { + return 0; + } + let n = self.len.load(Ordering::Relaxed); + n.saturating_mul(if self.byte_body { 1 } else { ELEM }) + } + + /// Slot a flush would copy. Not a sequence-counter read: the flush model + /// records this at [`FlushStep::Write`] without waiting out an odd section. + #[doc(hidden)] + pub fn live_slot(&self, index: u64) -> u64 { + self.slots.slots(Ordering::Acquire)[index as usize].load(Ordering::Relaxed) + } + + /// Write one live slot without the sequence counter. Flush tests use this + /// to show the persist reads the slot the writer would read. + #[doc(hidden)] + pub fn poke_live_slot(&self, index: u64, value: u64) { + self.slots.slots(Ordering::Acquire)[index as usize].store(value, Ordering::Relaxed); } pub fn flush_dirty(&self) -> Result<(), StoreError> { @@ -347,52 +520,50 @@ impl ArrayTable { &self, mut between: impl FnMut(&Self, FlushStep), ) -> Result<(), StoreError> { - let _one = self.flush_lock.lock().unwrap_or_else(|e| e.into_inner()); - let (from, n, disk) = { - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let Some(ref v) = *guard else { - return Ok(()); - }; - let from = self.unstored_from.load(Ordering::Acquire); - if from == CLEAN { - return Ok(()); - } - self.flushing.store(true, Ordering::Release); - (from, v.len() as u64, self.disk_len.load(Ordering::Acquire)) - }; - let mut end = EndFlush { - table: self, - persisted: None, - }; + // `strong_tx` has one flusher. Slot tables keep the lock. + let _one = + (!self.byte_body).then(|| self.flush_lock.lock().unwrap_or_else(|e| e.into_inner())); + if !self.in_ram { + return Ok(()); + } + let from = self.unstored_from.load(Ordering::Acquire); + if from == CLEAN { + return Ok(()); + } + between(self, FlushStep::Admit); + seq_arm(&self.seq, &self.flushing); + let from = self.unstored_from.load(Ordering::Acquire); + let n = self.len.load(Ordering::Acquire); + let disk = self.disk_len.load(Ordering::Acquire); + let mut end = FlushGuard(&self.flushing, &self.disk_len, &self.unstored_from, None); between(self, FlushStep::Write); - // Disconnect shrinks `confirmed` once per block. Rewriting the kept - // prefix would copy the whole image on that path. - if n >= disk { - let start = from.min(disk); - if start < n { - let max = pwrite_scratched( - |at, chunk| self.file.write_at(at, chunk), - Self::offset(start), - (n - start) * ELEM, - |dst, off| { - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let v = guard - .as_ref() - .ok_or(StoreError::Corrupt("class C flush lost its image"))?; - let base = start as usize + (off as usize) / 8; - for (i, slot) in dst.chunks_exact_mut(8).enumerate() { - slot.copy_from_slice(&v[base + i].to_le_bytes()); - } - Ok(()) - }, - )?; - self.flush_chunk_max.store(max, Ordering::Relaxed); + // A byte body counts `len` in bytes and the mark in bits. Slot tables + // count elements. The kept prefix is not rewritten, and the file is + // not padded out to a word. + let stride = if self.byte_body { 1 } else { ELEM }; + let start = if self.byte_body { from / 8 } else { from }.min(disk); + if n >= disk && start < n { + let mut scratch = [0u8; FLUSH_SCRATCH]; + let mut off = 0u64; + let mut max_chunk = 0usize; + let base = FILE_HEADER_LEN as u64 + start * stride; + let span = (n - start) * stride; + // One acquire: stores stay excluded until this flush returns. + let words = self.slots.slots(Ordering::Acquire); + while off < span { + let nbyte = ((span - off) as usize).min(FLUSH_SCRATCH); + let src = (start * stride + off) as usize; + copy_bit_bytes(words, src, &mut scratch[..nbyte]); + max_chunk = max_chunk.max(nbyte); + self.file.write_at(base + off, &scratch[..nbyte])?; + off += nbyte as u64; } + self.flush_chunk_max.store(max_chunk, Ordering::Relaxed); } - let logical = FILE_HEADER_LEN as u64 + n * ELEM; + let logical = FILE_HEADER_LEN as u64 + n * stride; self.file.set_logical_len(logical)?; between(self, FlushStep::Clean); - end.persisted = Some(n); + end.3 = Some(n); Ok(()) } diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index bab59b9938..f1228459cd 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -1,7 +1,9 @@ //! Class C tip/confirmation tables and Class A header→tx ranges. -use crate::array_table::ArrayTable; +use crate::array_table::{copy_bit_bytes, seq_odd, seq_read, ArrayTable, ClassCWrite, FlushStep}; use crate::error::StoreError; +use crate::file::{TableFile, FILE_HEADER_LEN}; +use crate::sync::atomic::{AtomicU64, Ordering}; use rbitcoin_primitives::{Fk, Height, TableKind}; use std::path::Path; @@ -29,11 +31,7 @@ impl ConfirmedTable { pub fn tip_height(&self) -> Option { let n = self.arr.len(); - if n == 0 { - None - } else { - Some(Height((n - 1) as u32)) - } + (n != 0).then(|| Height((n - 1) as u32)) } pub fn get(&self, height: Height) -> Result, StoreError> { @@ -43,11 +41,7 @@ impl ConfirmedTable { /// Batch height → header_fk (0-based height indices). Missing/unset → `None`. pub fn get_many(&self, heights: &[Height]) -> Result>, StoreError> { - let mut out = Vec::with_capacity(heights.len()); - for &h in heights { - out.push(self.get(h)?); - } - Ok(out) + heights.iter().map(|&h| self.get(h)).collect() } /// Write a raw slot, including 0. For tests that need a tip length with @@ -56,7 +50,7 @@ impl ConfirmedTable { #[doc(hidden)] pub fn set_raw_for_test( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, height: Height, value: u64, ) -> Result<(), StoreError> { @@ -65,7 +59,7 @@ impl ConfirmedTable { pub fn set( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, height: Height, header_fk: Fk, ) -> Result<(), StoreError> { @@ -78,11 +72,7 @@ impl ConfirmedTable { /// Set many height→header_fk pairs (one grow for max height, then bulk writes). /// /// Used by multi-block Class C confirm so tip extension is not N separate grows. - pub fn set_many( - &self, - write: &crate::array_table::ClassCWrite, - pairs: &[(Height, Fk)], - ) -> Result<(), StoreError> { + pub fn set_many(&self, write: &ClassCWrite, pairs: &[(Height, Fk)]) -> Result<(), StoreError> { if pairs.is_empty() { return Ok(()); } @@ -100,10 +90,7 @@ impl ConfirmedTable { /// /// Returns how many slots were removed. Mid-array holes are left for /// [`Store::revalidate_tip_window_n`] to shrink to last good. - pub fn trim_trailing_nulls( - &self, - write: &crate::array_table::ClassCWrite, - ) -> Result { + pub fn trim_trailing_nulls(&self, write: &ClassCWrite) -> Result { let n = self.arr.len(); if n == 0 { return Ok(0); @@ -124,11 +111,7 @@ impl ConfirmedTable { } /// Disconnect tip: height must be current tip. - pub fn disconnect_tip( - &self, - write: &crate::array_table::ClassCWrite, - height: Height, - ) -> Result<(), StoreError> { + pub fn disconnect_tip(&self, write: &ClassCWrite, height: Height) -> Result<(), StoreError> { match self.tip_height() { Some(t) if t == height => { self.arr.truncate(write, u64::from(height.0))?; @@ -149,167 +132,156 @@ impl ConfirmedTable { } /// Per-tx strong bit (schema v3): bit `(tx_fk - 1)` set ⇒ strong on best chain. -/// -/// ~64× smaller than u64-per-tx; call sites only need `is_strong`. Header fk is -/// not stored (derived from confirmed range when needed). -/// -/// Always L2 write-behind (unlike [`crate::array_table::class_c_inram_max_bytes`], -/// which still caps `confirmed` / `header_txs_*`). 1 bit/tx stays in RAM. +/// ~64× smaller than a `u64` per tx; the header fk comes from the confirmed range. +/// Always in RAM, ignoring [`crate::array_table::class_c_inram_max_bytes`]. +/// The words are [`ArrayTable`]'s slot buffer. Its length is body bytes, not +/// rounded up to a word. Flush is that table's flush. pub struct StrongTxTable { - pub bits: crate::file::TableFile, + pub table: ArrayTable, /// Number of bits allocated (covers tx fks 1..=n_bits). - pub n_bits: std::sync::atomic::AtomicU64, - /// L2 byte image. `strong_tx` stays in this `Vec`; open and create never go FdOnly. - pub data: std::sync::RwLock>, - /// Lowest bit not yet persisted (`u64::MAX` = clean). - pub unstored_bit: std::sync::atomic::AtomicU64, - /// True from the moment a flush records its range until it returns. - pub flushing: std::sync::atomic::AtomicBool, - /// Body byte length last flushed to disk (L2). - pub disk_bytes: std::sync::atomic::AtomicU64, - /// Largest chunk the last flush passed to `write_at`. - #[doc(hidden)] - pub flush_chunk_max: std::sync::atomic::AtomicUsize, -} - -#[must_use] -struct EndStrongFlush<'a> { - table: &'a StrongTxTable, - persisted: Option, -} - -impl Drop for EndStrongFlush<'_> { - fn drop(&mut self) { - use std::sync::atomic::Ordering; - if let Some(n) = self.persisted { - self.table.disk_bytes.store(n, Ordering::Release); - self.table.unstored_bit.store(u64::MAX, Ordering::Release); - } - self.table.flushing.store(false, Ordering::Release); - } + pub n_bits: AtomicU64, } impl StrongTxTable { pub fn create(dir: &Path) -> Result { + let file = TableFile::create(dir.join("strong_tx.body"), TableKind::StrongTx)?; Ok(Self { - bits: crate::file::TableFile::create(dir.join("strong_tx.body"), TableKind::StrongTx)?, - n_bits: std::sync::atomic::AtomicU64::new(0), - data: std::sync::RwLock::new(Vec::new()), - unstored_bit: std::sync::atomic::AtomicU64::new(u64::MAX), - flushing: std::sync::atomic::AtomicBool::new(false), - disk_bytes: std::sync::atomic::AtomicU64::new(0), - flush_chunk_max: std::sync::atomic::AtomicUsize::new(0), + table: ArrayTable::bit_image(file, 0), + n_bits: AtomicU64::new(0), }) } pub fn open(dir: &Path) -> Result { - let bits = crate::file::TableFile::open(dir.join("strong_tx.body"), TableKind::StrongTx)?; - let body = bits - .logical_len() - .saturating_sub(crate::file::FILE_HEADER_LEN as u64); - let n_bits = body.saturating_mul(8); - let mut v = vec![0u8; body as usize]; + let bits = TableFile::open(dir.join("strong_tx.body"), TableKind::StrongTx)?; + let body = bits.logical_len().saturating_sub(FILE_HEADER_LEN as u64); + let mut bytes = vec![0u8; body as usize]; + if body > 0 { + bits.read_at(FILE_HEADER_LEN as u64, &mut bytes)?; + } + let table = ArrayTable::bit_image(bits, body); if body > 0 { - bits.read_at(crate::file::FILE_HEADER_LEN as u64, &mut v)?; + table.slots.grow((body as usize).div_ceil(8)); + Self::copy_bytes_in(table.slots.slots(Ordering::Relaxed), &bytes); } Ok(Self { - bits, - n_bits: std::sync::atomic::AtomicU64::new(n_bits), - data: std::sync::RwLock::new(v), - unstored_bit: std::sync::atomic::AtomicU64::new(u64::MAX), - flushing: std::sync::atomic::AtomicBool::new(false), - disk_bytes: std::sync::atomic::AtomicU64::new(body), - flush_chunk_max: std::sync::atomic::AtomicUsize::new(0), + table, + n_bits: AtomicU64::new(body.saturating_mul(8)), }) } - fn refuse_while_flushing(&self) -> Result<(), StoreError> { - use std::sync::atomic::Ordering; - if self.flushing.load(Ordering::Acquire) { - Err(StoreError::Rejected("class C store during flush")) - } else { - Ok(()) + fn ensure_held(&self, need_bits: u64) { + if need_bits <= self.n_bits.load(Ordering::Relaxed) { + return; + } + let need_bytes = need_bits.div_ceil(8) as usize; + self.table.slots.grow(need_bytes.div_ceil(8)); + self.table.len.store(need_bytes as u64, Ordering::Relaxed); + self.n_bits.store(need_bits, Ordering::Relaxed); + } + + fn for_each_mask(start: u64, end: u64, mut f: impl FnMut(usize, u64)) { + let mut bit = start; + while bit < end { + let wi = (bit / 64) as usize; + let word_lo = (wi as u64) * 64; + let lo = bit - word_lo; + let hi = end.min(word_lo + 64) - word_lo; + // `u64::MAX << lo` is `MAX ^ ((1 << lo) - 1)`. A short word stops at `hi`. + let hi_bits = if hi == 64 { u64::MAX } else { (1u64 << hi) - 1 }; + let mask = hi_bits ^ ((1u64 << lo) - 1); + f(wi, mask); + bit = word_lo + hi; + } + } + + fn apply_bits(&self, start: u64, end: u64, on: bool) { + let words = self.table.slots.slots(Ordering::Relaxed); + Self::for_each_mask(start, end, |wi, mask| { + if on { + words[wi].fetch_or(mask, Ordering::Relaxed); + } else { + words[wi].fetch_and(!mask, Ordering::Relaxed); + } + }); + } + + fn count_word_ones(words: &[AtomicU64], start: u64, end: u64) -> u64 { + let mut ones = 0u64; + Self::for_each_mask(start, end, |wi, mask| { + ones += (words[wi].load(Ordering::Relaxed) & mask).count_ones() as u64; + }); + ones + } + + fn store_byte(words: &[AtomicU64], index: usize, value: u8) { + let w = &words[index / 8]; + let shift = (index % 8) * 8; + let _ = w.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { + Some((cur & !(0xffu64 << shift)) | (u64::from(value) << shift)) + }); + } + + fn copy_bytes_in(words: &[AtomicU64], src: &[u8]) { + for (i, chunk) in src.chunks(8).enumerate() { + let mut buf = [0u8; 8]; + buf[..chunk.len()].copy_from_slice(chunk); + words[i].store(u64::from_le_bytes(buf), Ordering::Relaxed); } } #[doc(hidden)] pub fn mark_dirty_bit(&self, bit: u64) { - use std::sync::atomic::Ordering; - let _ = self.unstored_bit.fetch_min(bit, Ordering::AcqRel); + let _ = self.table.unstored_from.fetch_min(bit, Ordering::AcqRel); } #[doc(hidden)] - pub fn ensure_bits( - &self, - _write: &crate::array_table::ClassCWrite, - need_bits: u64, - ) -> Result<(), StoreError> { - use std::sync::atomic::Ordering; - let n = self.n_bits.load(Ordering::Acquire); - if need_bits <= n { + pub fn ensure_bits(&self, _write: &ClassCWrite, need_bits: u64) -> Result<(), StoreError> { + if need_bits <= self.n_bits.load(Ordering::Acquire) { return Ok(()); } - let need_bytes = need_bits.div_ceil(8); - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - if need_bytes as usize > guard.len() { - guard.resize(need_bytes as usize, 0); - } - self.n_bits.store(need_bits, Ordering::Release); - // Growth alone is not a bit mutate; callers set_bit mark dirty. + let _odd = self.table.begin_store()?; + self.ensure_held(need_bits); Ok(()) } #[doc(hidden)] pub fn get_bit(&self, bit: u64) -> Result { - use std::sync::atomic::Ordering; - let n = self.n_bits.load(Ordering::Acquire); - if bit >= n { - return Ok(false); - } - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let bi = (bit / 8) as usize; - Ok((guard[bi] >> (bit % 8)) & 1 != 0) + Ok(seq_read(&self.table.seq, || { + let n = self.n_bits.load(Ordering::Relaxed); + if bit >= n { + return Some(false); + } + let words = self.table.slots.try_slots((bit / 64) as usize + 1)?; + let word = words[(bit / 64) as usize].load(Ordering::Relaxed); + Some((word >> (bit % 64)) & 1 == 1) + })) } #[doc(hidden)] - pub fn set_bit( - &self, - write: &crate::array_table::ClassCWrite, - bit: u64, - on: bool, - ) -> Result<(), StoreError> { - self.ensure_bits(write, bit + 1)?; - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - let bi = (bit / 8) as usize; - if on { - guard[bi] |= 1 << (bit % 8); - } else { - guard[bi] &= !(1 << (bit % 8)); - } - self.mark_dirty_bit(bit); - Ok(()) + pub fn set_bit(&self, write: &ClassCWrite, bit: u64, on: bool) -> Result<(), StoreError> { + self.set_bits_range(write, bit, bit + 1, on) } /// Compatibility: Some(dummy nonzero) if strong, else None. pub fn get(&self, tx_fk: Fk) -> Result, StoreError> { - if self.is_strong(tx_fk)? { - Ok(Some(Fk(1))) - } else { - Ok(None) - } + Ok(self.is_strong(tx_fk)?.then_some(Fk(1))) } /// In-RAM L2 bit-image bytes. pub fn l2_resident_bytes(&self) -> u64 { - let g = self.data.read().unwrap_or_else(|e| e.into_inner()); - g.len() as u64 + self.table.l2_resident_bytes() + } + + /// Set one live body byte without the sequence counter. Flush tests use + /// this to show the persist reads the image the writer would read. + #[doc(hidden)] + pub fn poke_live_byte(&self, index: usize, value: u8) { + Self::store_byte(self.table.slots.slots(Ordering::Acquire), index, value); } pub fn set_strong( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, tx_fk: Fk, header_fk: Fk, ) -> Result<(), StoreError> { @@ -323,19 +295,18 @@ impl StrongTxTable { /// Mark many consecutive tx fks strong (one ensure + bulk byte writes). pub fn set_strong_range( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, first_tx_fk: Fk, count: u32, header_fk: Fk, ) -> Result<(), StoreError> { - let id = first_tx_fk.get().ok_or(StoreError::InvalidFk)?; - if header_fk.is_null() || count == 0 { - return if count == 0 { - Ok(()) - } else { - Err(StoreError::InvalidFk) - }; + if count == 0 { + return Ok(()); } + if header_fk.is_null() { + return Err(StoreError::InvalidFk); + } + let id = first_tx_fk.get().ok_or(StoreError::InvalidFk)?; let start = id - 1; let end = start + u64::from(count); self.set_bits_range(write, start, end, true) @@ -345,7 +316,7 @@ impl StrongTxTable { #[doc(hidden)] pub fn set_bits_range( &self, - write: &crate::array_table::ClassCWrite, + _write: &ClassCWrite, start: u64, end: u64, on: bool, @@ -353,30 +324,16 @@ impl StrongTxTable { if end <= start { return Ok(()); } - self.ensure_bits(write, end)?; - let mut guard = self.data.write().unwrap_or_else(|e| e.into_inner()); - self.refuse_while_flushing()?; - let mut bit = start; - while bit < end { - let bi = (bit / 8) as usize; - if on { - guard[bi] |= 1 << (bit % 8); - } else { - guard[bi] &= !(1 << (bit % 8)); - } - bit += 1; - } + let _odd = self.table.begin_store()?; + self.ensure_held(end); + self.apply_bits(start, end, on); self.mark_dirty_bit(start); Ok(()) } - pub fn set_unstrong( - &self, - write: &crate::array_table::ClassCWrite, - tx_fk: Fk, - ) -> Result<(), StoreError> { + pub fn set_unstrong(&self, write: &ClassCWrite, tx_fk: Fk) -> Result<(), StoreError> { let id = tx_fk.get().ok_or(StoreError::InvalidFk)?; - let n = self.n_bits.load(std::sync::atomic::Ordering::Acquire); + let n = self.n_bits.load(Ordering::Acquire); if id > n { return Ok(()); } @@ -386,7 +343,7 @@ impl StrongTxTable { /// Clear many consecutive strong bits (Class C repair / disconnect ranges). pub fn set_unstrong_range( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, first_tx_fk: Fk, count: u32, ) -> Result<(), StoreError> { @@ -396,7 +353,7 @@ impl StrongTxTable { } let start = id - 1; let end = start + u64::from(count); - let n = self.n_bits.load(std::sync::atomic::Ordering::Acquire); + let n = self.n_bits.load(Ordering::Acquire); if start >= n { return Ok(()); } @@ -411,7 +368,7 @@ impl StrongTxTable { /// Allocated bit count (covers create fks `1..=n`). pub fn allocated_bits(&self) -> u64 { - self.n_bits.load(std::sync::atomic::Ordering::Acquire) + self.n_bits.load(Ordering::Acquire) } /// Count set bits for create fks in half-open `[start_fk, end_fk)`. @@ -433,23 +390,15 @@ impl StrongTxTable { if end_bit <= start_bit { return Ok(0); } - let mut ones = 0u64; - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let mut bit = start_bit; - while bit < end_bit { - let bi = (bit / 8) as usize; - let b = guard[bi]; - if b == 0 { - bit = (bit + 8) & !7; - continue; - } - let off = (bit % 8) as u32; - if (b >> off) & 1 != 0 { - ones += 1; + Ok(seq_read(&self.table.seq, || { + let n = self.n_bits.load(Ordering::Relaxed); + let end = end_bit.min(n); + if end <= start_bit { + return Some(0); } - bit += 1; - } - Ok(ones) + let words = self.table.slots.try_slots(((end - 1) / 64) as usize + 1)?; + Some(Self::count_word_ones(words, start_bit, end)) + })) } /// Read `dst.len()` bit-image bytes starting at payload byte `byte_off`. @@ -457,16 +406,20 @@ impl StrongTxTable { if dst.is_empty() { return Ok(()); } - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let start = byte_off as usize; - let end = start.saturating_add(dst.len()).min(guard.len()); - if start >= guard.len() { - dst.fill(0); - return Ok(()); - } - let n = end - start; - dst[..n].copy_from_slice(&guard[start..end]); - dst[n..].fill(0); + seq_read(&self.table.seq, || { + let n = self.table.len.load(Ordering::Relaxed) as usize; + let start = byte_off as usize; + if start >= n { + dst.fill(0); + return Some(()); + } + let take = dst.len().min(n - start); + let last = start + take - 1; + let words = self.table.slots.try_slots(last / 8 + 1)?; + copy_bit_bytes(words, start, &mut dst[..take]); + dst[take..].fill(0); + Some(()) + }); Ok(()) } @@ -483,76 +436,41 @@ impl StrongTxTable { /// Persist dirty L2 bit image by reading live bytes. The unstored mark /// clears only after the length publish succeeds. pub fn flush_dirty(&self) -> Result<(), StoreError> { - self.flush_dirty_between(|_, _| {}) + self.table.flush_dirty() } + /// Same flush as [`ArrayTable::flush_dirty_between`]. The hook sees this + /// table so a test can store during admit or the write. #[doc(hidden)] - pub fn flush_dirty_between( + pub fn flush_observing( &self, - mut between: impl FnMut(&Self, crate::array_table::FlushStep), + mut between: impl FnMut(&Self, FlushStep), ) -> Result<(), StoreError> { - use std::sync::atomic::Ordering; - let (body_len, disk, dirty_byte) = { - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let bit = self.unstored_bit.load(Ordering::Acquire); - if bit == u64::MAX { - return Ok(()); - } - self.flushing.store(true, Ordering::Release); - let body_len = guard.len() as u64; - let disk = self.disk_bytes.load(Ordering::Acquire); - (body_len, disk, bit / 8) - }; - let mut end = EndStrongFlush { - table: self, - persisted: None, - }; - between(self, crate::array_table::FlushStep::Write); - if body_len >= disk { - let start = dirty_byte.min(disk); - if start < body_len { - let max = crate::array_table::pwrite_scratched( - |at, chunk| self.bits.write_at(at, chunk), - crate::file::FILE_HEADER_LEN as u64 + start, - body_len - start, - |dst, off| { - let guard = self.data.read().unwrap_or_else(|e| e.into_inner()); - let at = start as usize + off as usize; - dst.copy_from_slice(&guard[at..at + dst.len()]); - Ok(()) - }, - )?; - self.flush_chunk_max.store(max, Ordering::Relaxed); - } - } - let logical = crate::file::FILE_HEADER_LEN as u64 + body_len; - self.bits.set_logical_len(logical)?; - between(self, crate::array_table::FlushStep::Clean); - end.persisted = Some(body_len); - Ok(()) + let this = self as *const Self; + // SAFETY: `this` is `self`, and the hook runs before this method + // returns. Both uses are shared. + self.table + .flush_dirty_between(|_, step| between(unsafe { &*this }, step)) } pub fn flush(&self) -> Result<(), StoreError> { - self.flush_dirty()?; - self.bits.flush() + self.table.flush() } pub fn flush_async(&self) -> Result<(), StoreError> { - self.flush_dirty()?; - self.bits.flush_async() + self.table.flush_async() } } -/// Per-header **contiguous** tx_fk range (Class A archive body association). -/// -/// Schema v2 stores `(first_tx_fk, count)` only — not a u64 fk vector. -/// Writer must assign contiguous FKs per block (archive plan does). -/// -/// - `first.body[header_fk-1]` = first_tx_fk (0 = no body) -/// - `count.body[header_fk-1]` = n txs (0 = no body) +/// Per-header contiguous tx_fk range (Class A). Schema v2 stores `(first, count)` +/// only, and the writer assigns contiguous fks per block. +/// `first.body[header_fk-1]` is the first tx fk (0 = no body). +/// `count.body[header_fk-1]` is the tx count (0 = no body). pub struct HeaderTxsTable { pub first: ArrayTable, pub count: ArrayTable, + /// One counter for a header's `(first, count)` pair. + seq: AtomicU64, } impl HeaderTxsTable { @@ -560,6 +478,7 @@ impl HeaderTxsTable { Ok(Self { first: ArrayTable::create(dir.join("header_txs_first.body"), TableKind::ArrayLink)?, count: ArrayTable::create(dir.join("header_txs_count.body"), TableKind::ArrayLink)?, + seq: AtomicU64::new(0), }) } @@ -567,6 +486,7 @@ impl HeaderTxsTable { Ok(Self { first: ArrayTable::open(dir.join("header_txs_first.body"), TableKind::ArrayLink)?, count: ArrayTable::open(dir.join("header_txs_count.body"), TableKind::ArrayLink)?, + seq: AtomicU64::new(0), }) } @@ -579,31 +499,30 @@ impl HeaderTxsTable { /// Store a contiguous range. `tx_fks` must be non-empty and contiguous. pub fn put_list( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, header_fk: Fk, tx_fks: &[Fk], ) -> Result<(), StoreError> { + let (first, n) = Self::list_span(tx_fks)?; + self.put_range(write, header_fk, first, n) + } + + fn list_span(tx_fks: &[Fk]) -> Result<(Fk, u32), StoreError> { if tx_fks.is_empty() { return Err(StoreError::Corrupt("empty header tx list")); } - debug_assert!( - tx_fks - .windows(2) - .all(|w| w[1].0 == w[0].0.saturating_add(1)), - "header_txs must be contiguous" - ); if !tx_fks .windows(2) .all(|w| w[1].0 == w[0].0.saturating_add(1)) { return Err(StoreError::Corrupt("header_txs not contiguous")); } - self.put_range(write, header_fk, tx_fks[0], tx_fks.len() as u32) + Ok((tx_fks[0], tx_fks.len() as u32)) } pub fn put_range( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, header_fk: Fk, first_tx_fk: Fk, n: u32, @@ -612,6 +531,7 @@ impl HeaderTxsTable { if first_tx_fk.is_null() || n == 0 { return Err(StoreError::InvalidFk); } + let _pair = seq_odd(&self.seq); self.first.set(write, id - 1, first_tx_fk.0)?; self.count.set(write, id - 1, u64::from(n))?; Ok(()) @@ -620,7 +540,7 @@ impl HeaderTxsTable { /// Batch-append many header→tx ranges. pub fn put_lists_batch( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, items: &[(Fk, &[Fk])], ) -> Result<(), StoreError> { if items.is_empty() { @@ -629,19 +549,12 @@ impl HeaderTxsTable { let mut first_pairs = Vec::with_capacity(items.len()); let mut count_pairs = Vec::with_capacity(items.len()); for (header_fk, tx_fks) in items { - if tx_fks.is_empty() { - return Err(StoreError::Corrupt("empty header tx list")); - } - if !tx_fks - .windows(2) - .all(|w| w[1].0 == w[0].0.saturating_add(1)) - { - return Err(StoreError::Corrupt("header_txs not contiguous")); - } + let (first, n) = Self::list_span(tx_fks)?; let id = header_fk.get().ok_or(StoreError::InvalidFk)?; - first_pairs.push((id - 1, tx_fks[0].0)); - count_pairs.push((id - 1, tx_fks.len() as u64)); + first_pairs.push((id - 1, first.0)); + count_pairs.push((id - 1, u64::from(n))); } + let _pair = seq_odd(&self.seq); self.first.set_many(write, &first_pairs)?; self.count.set_many(write, &count_pairs)?; Ok(()) @@ -650,7 +563,7 @@ impl HeaderTxsTable { /// Batch store ranges without expanding fk vectors. pub fn put_ranges_batch( &self, - write: &crate::array_table::ClassCWrite, + write: &ClassCWrite, items: &[(Fk, Fk, u32)], ) -> Result<(), StoreError> { if items.is_empty() { @@ -666,6 +579,7 @@ impl HeaderTxsTable { first_pairs.push((id - 1, first_tx_fk.0)); count_pairs.push((id - 1, u64::from(n))); } + let _pair = seq_odd(&self.seq); self.first.set_many(write, &first_pairs)?; self.count.set_many(write, &count_pairs)?; Ok(()) @@ -673,18 +587,29 @@ impl HeaderTxsTable { pub fn get_range(&self, header_fk: Fk) -> Result, StoreError> { let id = header_fk.get().ok_or(StoreError::InvalidFk)?; - if id > self.first.len() { - return Ok(None); - } - let first_raw = self.first.get(id - 1)?; - let Some(first) = Fk::new(first_raw) else { - return Ok(None); - }; - let n = self.count.get(id - 1)?; - if n == 0 || n > u64::from(u32::MAX) { - return Ok(None); - } - Ok(Some((first, n as u32))) + let mut failed = None; + let pair = seq_read(&self.seq, || { + failed = None; + if id > self.first.len() { + return Some(None); + } + // Column loads acquire the stores. `seq_read` rechecks behind its + // acquire fence. `Ok(None)` retries; `Err` returns once that generation holds. + let mut raw = [0u64; 2]; + for (i, col) in [&self.first, &self.count].into_iter().enumerate() { + match col.peek_slot(id - 1) { + Ok(Some(v)) => raw[i] = v, + Ok(None) => return None, + Err(e) => { + failed = Some(e); + return Some(None); + } + } + } + let count = u32::try_from(raw[1]).ok().filter(|&n| n > 0); + Some(Fk::new(raw[0]).zip(count)) + }); + failed.map_or(Ok(pair), Err) } /// Expand range to a vec of fks (convenience for existing call sites). @@ -692,11 +617,7 @@ impl HeaderTxsTable { let Some((first, n)) = self.get_range(header_fk)? else { return Ok(None); }; - let mut out = Vec::with_capacity(n as usize); - for i in 0..n { - out.push(Fk(first.0 + u64::from(i))); - } - Ok(Some(out)) + Ok(Some((0..n).map(|i| Fk(first.0 + u64::from(i))).collect())) } pub fn has_body(&self, header_fk: Fk) -> Result { @@ -707,14 +628,10 @@ impl HeaderTxsTable { pub fn index_in_span(first: Fk, n: u32, tx_fk: Fk) -> Option { let tid = tx_fk.get()?; let lo = first.get()?; - if n == 0 { - return None; - } let end = lo.saturating_add(u64::from(n)); - if tid < lo || tid >= end { - return None; - } - u32::try_from(tid - lo).ok() + (n != 0 && (lo..end).contains(&tid)) + .then(|| u32::try_from(tid - lo).ok()) + .flatten() } /// True if `tx_fk` is in this header's contiguous Class A body range. @@ -732,21 +649,14 @@ impl HeaderTxsTable { Ok(Self::index_in_span(first, n, tx_fk)) } - /// Drop Class A body association for `header_fk` (does not free tx rows). - /// - /// Used when reconstruct produces a block that fails header checks (e.g. - /// merkle root mismatch) — the header hash is fine; the association is bad. - pub fn clear_body( - &self, - write: &crate::array_table::ClassCWrite, - header_fk: Fk, - ) -> Result { + /// Drop the Class A body association for `header_fk` (tx rows stay). + /// Reconstruct uses this when a block fails a header check such as merkle. + pub fn clear_body(&self, write: &ClassCWrite, header_fk: Fk) -> Result { let id = header_fk.get().ok_or(StoreError::InvalidFk)?; if id == 0 { return Err(StoreError::InvalidFk); } - let had = self.has_body(header_fk)?; - if !had { + if !self.has_body(header_fk)? { return Ok(false); } // Ensure array slots exist then zero (0 = no body). @@ -754,26 +664,18 @@ impl HeaderTxsTable { if idx >= self.first.len() { return Ok(false); } + let _pair = seq_odd(&self.seq); self.first.set(write, idx, 0)?; self.count.set(write, idx, 0)?; Ok(true) } - /// Number of headers that currently have a Class A body (`count > 0`). - /// - /// Full-array scan — use only for rare status/startup (`archived_block_count`). - /// **Not** on the plan hot path: Class A never leads tip, so plan must not - /// call this for “archive far ahead” heuristics. + /// Headers with a Class A body (`count > 0`). Full scan for status and startup + /// (`archived_block_count`) only. Not a plan hot path: Class A never leads tip. pub fn count_bodies(&self) -> Result { - let n = self.count.len(); - let mut total = 0u64; - for i in 0..n { - let c = self.count.get(i)?; - if c > 0 { - total += 1; - } - } - Ok(total) + (0..self.count.len()).try_fold(0u64, |total, i| { + Ok(total + u64::from(self.count.get(i)? > 0)) + }) } pub fn flush(&self) -> Result<(), StoreError> { diff --git a/crates/rbitcoin-store/src/lib.rs b/crates/rbitcoin-store/src/lib.rs index ea8ec2dacd..0f71222d48 100644 --- a/crates/rbitcoin-store/src/lib.rs +++ b/crates/rbitcoin-store/src/lib.rs @@ -107,6 +107,8 @@ pub mod spender_table; pub mod store; #[doc(hidden)] pub mod store_secret; +#[doc(hidden)] +pub mod sync; pub mod testutil; #[doc(hidden)] pub mod tx_head_mphf; diff --git a/crates/rbitcoin-store/src/store.rs b/crates/rbitcoin-store/src/store.rs index a8af08b2d4..59175b6f0f 100644 --- a/crates/rbitcoin-store/src/store.rs +++ b/crates/rbitcoin-store/src/store.rs @@ -307,7 +307,7 @@ pub struct Store { pub spend_reorg_gen: std::sync::atomic::AtomicU64, /// Even while confirmed spentness is stable. Odd while a confirm annotate /// or a disconnect is publishing a change. - pub utxo_view: std::sync::atomic::AtomicU64, + pub utxo_view: crate::sync::atomic::AtomicU64, /// Confirm readers wait here while a `--datadir-hdd` write holds the spindle. /// Flag off never locks, so [`Self::spindle_wait`] returns immediately. pub spindle: std::sync::Mutex<()>, @@ -320,12 +320,12 @@ pub struct SpindleHold<'a> { /// Holds [`Store::utxo_view`] odd until drop. pub struct UtxoViewGuard<'a> { - pub view: &'a std::sync::atomic::AtomicU64, + pub view: &'a crate::sync::atomic::AtomicU64, } impl Drop for UtxoViewGuard<'_> { fn drop(&mut self) { - use std::sync::atomic::Ordering; + use crate::sync::atomic::Ordering; self.view.fetch_add(1, Ordering::Release); } } @@ -412,7 +412,7 @@ impl Store { spend_snapshot: std::sync::atomic::AtomicU64::new(0), spend_annotate_from: std::sync::atomic::AtomicU64::new(0), spend_reorg_gen: std::sync::atomic::AtomicU64::new(0), - utxo_view: std::sync::atomic::AtomicU64::new(0), + utxo_view: crate::sync::atomic::AtomicU64::new(0), spend_marker: std::sync::Mutex::new(()), spindle: std::sync::Mutex::new(()), path, @@ -475,7 +475,7 @@ impl Store { spend_snapshot: std::sync::atomic::AtomicU64::new(0), spend_annotate_from: std::sync::atomic::AtomicU64::new(0), spend_reorg_gen: std::sync::atomic::AtomicU64::new(0), - utxo_view: std::sync::atomic::AtomicU64::new(0), + utxo_view: crate::sync::atomic::AtomicU64::new(0), spend_marker: std::sync::Mutex::new(()), spindle: std::sync::Mutex::new(()), path, @@ -1750,7 +1750,7 @@ impl Store { /// Odd for the guard's life. Callers who sample an even stamp and read /// coins can trust that read only when the stamp is unchanged afterward. pub fn hold_utxo_view(&self) -> UtxoViewGuard<'_> { - use std::sync::atomic::Ordering; + use crate::sync::atomic::Ordering; let prev = self.utxo_view.fetch_add(1, Ordering::AcqRel); debug_assert_eq!( prev & 1, @@ -1764,7 +1764,7 @@ impl Store { /// `Some` even generation, or `None` while [`Self::hold_utxo_view`] is held. pub fn utxo_view_stamp(&self) -> Option { - use std::sync::atomic::Ordering; + use crate::sync::atomic::Ordering; let v = self.utxo_view.load(Ordering::Acquire); (v & 1 == 0).then_some(v) } diff --git a/crates/rbitcoin-store/src/sync.rs b/crates/rbitcoin-store/src/sync.rs new file mode 100644 index 0000000000..ffe4574835 --- /dev/null +++ b/crates/rbitcoin-store/src/sync.rs @@ -0,0 +1,22 @@ +//! `std::sync::atomic` behind one import. `--cfg loom` swaps in loom's +//! atomics so a model can see Class C sequence counters and the UTXO view +//! word. Production is `std`. Shuttle does not rebuild this crate's atomics: +//! its model is the script-pool idle turn. + +#[cfg(loom)] +pub use loom::sync::atomic; + +#[cfg(not(loom))] +pub use std::sync::atomic; + +/// Pause a reader while a writer is in the odd section. A model has to yield +/// the thread here. Production is a processor hint. +#[cfg(loom)] +pub fn spin() { + atomic::spin_loop_hint(); +} + +#[cfg(not(loom))] +pub fn spin() { + std::hint::spin_loop(); +} diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index 170e900e00..525bf9fdc4 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -15,8 +15,9 @@ fn tmp_path() -> std::path::PathBuf { )) } -/// `get` checks `len` before taking the read lock, so a `truncate` can -/// land in between. The reader must see the shorter table, not panic. +/// `get` samples `len` and the slot under one sequence counter. A `truncate` +/// between those loads retries. The reader sees the previous value or the +/// shortened table, not a mix and not a panic. #[test] fn get_racing_truncate_does_not_panic() { use std::sync::atomic::AtomicBool; @@ -53,6 +54,83 @@ fn get_racing_truncate_does_not_panic() { let _ = std::fs::remove_file(&path); } +/// A multi-slot store publishes one generation. Each `get` is the previous +/// value or the new one for that index. +#[test] +fn get_during_set_many_is_old_or_new() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let path = tmp_path(); + let _ = std::fs::remove_file(&path); + let t = Arc::new(ArrayTable::create(&path, TableKind::Confirmed).unwrap()); + let n = 32u64; + let old: Vec<_> = (0..n).map(|i| (i, 10 + i)).collect(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &old) + .unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let readers: Vec<_> = (0..4) + .map(|_| { + let (t, stop) = (Arc::clone(&t), Arc::clone(&stop)); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + for i in 0..n { + let v = t.get(i).unwrap(); + assert!(v == 10 + i || v == 100 + i, "mixed slot {i} value {v}"); + } + } + }) + }) + .collect(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300); + while std::time::Instant::now() < deadline { + let new: Vec<_> = (0..n).map(|i| (i, 100 + i)).collect(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &new) + .unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &old) + .unwrap(); + } + stop.store(true, Ordering::Relaxed); + for r in readers { + assert!(r.join().is_ok(), "get panicked during set_many"); + } + let _ = std::fs::remove_file(&path); +} + +/// A store that lands after the flush has noticed it is dirty, and before +/// new stores are excluded, is still in this flush. Sampling the dirty index +/// before that exclusion would clear the mark and leave the slot off disk. +#[test] +fn store_before_exclude_is_in_this_flush() { + let path = tmp_path(); + let _ = std::fs::remove_file(&path); + let t = ArrayTable::create(&path, TableKind::ArrayLink).unwrap(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[(0, 1), (1, 2)]) + .unwrap(); + t.flush().unwrap(); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 9) + .unwrap(); + t.flush_dirty_between(|t, step| { + if step == FlushStep::Admit { + t.set(&rbitcoin_store::ClassCWrite::for_test(), 0, 7) + .unwrap(); + } + }) + .unwrap(); + let mut buf = [0u8; 8]; + t.file.read_at(FILE_HEADER_LEN as u64, &mut buf).unwrap(); + assert_eq!( + u64::from_le_bytes(buf), + 7, + "earlier slot must be in this flush" + ); + assert_eq!( + t.unstored_from.load(Ordering::Acquire), + u64::MAX, + "the mark must stay set until that slot is persisted" + ); + let _ = std::fs::remove_file(&path); +} + /// Flush persists the live slots of the dirty range recorded at start. /// A store from the write hook is refused; the bytes on disk are the slots /// after that hook, so the flush did not keep a detached image. A store @@ -72,9 +150,7 @@ fn flush_writes_live_slots_and_refuses_a_store_until_it_finishes() { let mut during = Ok(()); t.flush_dirty_between(|t, step| { if step == FlushStep::Write { - if let Some(v) = t.data.write().unwrap_or_else(|e| e.into_inner()).as_mut() { - v[0] = 77; - } + t.poke_live_slot(0, 77); during = t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 9); let _ = t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &[(2, 8)]); let _ = t.truncate(&rbitcoin_store::ClassCWrite::for_test(), n - 1); diff --git a/crates/rbitcoin-store/tests/chain/chain_table_tests.rs b/crates/rbitcoin-store/tests/chain/chain_table_tests.rs index 3b6ccd063e..088edc80cf 100644 --- a/crates/rbitcoin-store/tests/chain/chain_table_tests.rs +++ b/crates/rbitcoin-store/tests/chain/chain_table_tests.rs @@ -15,6 +15,47 @@ fn tmp() -> std::path::PathBuf { p } +/// `get_range` samples first and count together. A paired update or a clear +/// returns the previous pair, the new pair, or no body. Never a new first +/// with the old count, and never a zeroed first with the old count. +#[test] +fn get_range_during_put_and_clear_is_one_pair() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + let dir = tmp(); + let ht = Arc::new(HeaderTxsTable::create(&dir).unwrap()); + let write = rbitcoin_store::ClassCWrite::for_test(); + ht.put_range(&write, Fk(1), Fk(10), 4).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let readers: Vec<_> = (0..4) + .map(|_| { + let (ht, stop) = (Arc::clone(&ht), Arc::clone(&stop)); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + match ht.get_range(Fk(1)).unwrap() { + Some((Fk(10), 4)) => {} + Some((Fk(50), 9)) => {} + None => {} + other => panic!("torn header range {other:?}"), + } + } + }) + }) + .collect(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300); + while std::time::Instant::now() < deadline { + let write = rbitcoin_store::ClassCWrite::for_test(); + ht.put_range(&write, Fk(1), Fk(50), 9).unwrap(); + ht.clear_body(&write, Fk(1)).unwrap(); + ht.put_range(&write, Fk(1), Fk(10), 4).unwrap(); + } + stop.store(true, Ordering::Relaxed); + for r in readers { + assert!(r.join().is_ok(), "get_range panicked"); + } + let _ = std::fs::remove_dir_all(&dir); +} + #[allow(clippy::cognitive_complexity)] // one fixture, many chain surface arms #[test] fn confirmed_tx_height_header_txs_surface() { diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index 6d21654ee9..5b836b14c0 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -15,6 +15,35 @@ fn tmp() -> std::path::PathBuf { p } +/// A bit store after the flush has noticed dirt, and before new stores are +/// excluded, is still in this flush. +#[test] +fn strong_store_before_exclude_is_in_this_flush() { + use rbitcoin_store::array_table::FlushStep; + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 16, Fk(1)) + .unwrap(); + t.flush().unwrap(); + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(9)) + .unwrap(); + t.flush_observing(|t, step| { + if step == FlushStep::Admit { + t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) + .unwrap(); + } + }) + .unwrap(); + drop(t); + let t = StrongTxTable::open(&dir).unwrap(); + assert!( + !t.is_strong(Fk(1)).unwrap(), + "earlier byte must be in this flush" + ); + assert!(!t.is_strong(Fk(9)).unwrap()); + let _ = std::fs::remove_dir_all(&dir); +} + /// Flush persists the live bit image. A bit store from the write hook is /// refused; a store after flush returns lands in the next flush. #[test] @@ -30,15 +59,15 @@ fn strong_flush_writes_live_bytes_and_refuses_a_store() { t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) .unwrap(); let mut during = Ok(()); - t.flush_dirty_between(|t, step| { + t.flush_observing(|t, step| { if step == FlushStep::Write { - t.data.write().unwrap_or_else(|e| e.into_inner())[0] = 0xff; + t.poke_live_byte(0, 0xff); during = t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(2)); } }) .unwrap(); assert!(during.is_err(), "store during flush refused={during:?}"); - let chunk = t.flush_chunk_max.load(Ordering::Relaxed); + let chunk = t.table.flush_chunk_max.load(Ordering::Relaxed); assert!(chunk > 0 && chunk <= FLUSH_SCRATCH, "flush chunk {chunk}"); drop(t); let t = StrongTxTable::open(&dir).unwrap(); @@ -70,9 +99,9 @@ fn strong_failed_write_keeps_the_range_dirty() { t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(1)) .unwrap(); let mut swapped = None; - let failed = t.flush_dirty_between(|t, step| { + let failed = t.flush_observing(|t, step| { if step == FlushStep::Write { - swapped = Some(crate::dev_full::DevFull::on(&t.bits.read_file)); + swapped = Some(crate::dev_full::DevFull::on(&t.table.file.read_file)); } }); drop(swapped); @@ -81,10 +110,16 @@ fn strong_failed_write_keeps_the_range_dirty() { "write_at must fail while the fd is /dev/full" ); let mut byte = [0u8; 1]; - t.bits.read_at(FILE_HEADER_LEN as u64, &mut byte).unwrap(); + t.table + .file + .read_at(FILE_HEADER_LEN as u64, &mut byte) + .unwrap(); assert_eq!(byte[0] & 1, 1, "failed persist must not publish the clear"); t.flush().unwrap(); - t.bits.read_at(FILE_HEADER_LEN as u64, &mut byte).unwrap(); + t.table + .file + .read_at(FILE_HEADER_LEN as u64, &mut byte) + .unwrap(); assert_eq!(byte[0] & 1, 0, "retry writes the same slots"); let _ = std::fs::remove_dir_all(&dir); } @@ -98,13 +133,13 @@ fn strong_same_length_rewrite_publishes_logical_len() { t.set_strong_range(&rbitcoin_store::ClassCWrite::for_test(), Fk(1), 16, Fk(1)) .unwrap(); t.flush().unwrap(); - let published = t.bits.logical_len(); + let published = t.table.file.logical_len(); assert!(published > FILE_HEADER_LEN as u64); - t.bits.set_logical_len(published + 64).unwrap(); + t.table.file.set_logical_len(published + 64).unwrap(); t.set_unstrong(&rbitcoin_store::ClassCWrite::for_test(), Fk(3)) .unwrap(); t.flush_dirty().unwrap(); - assert_eq!(t.bits.logical_len(), published); + assert_eq!(t.table.file.logical_len(), published); drop(t); let t = StrongTxTable::open(&dir).unwrap(); assert!(!t.is_strong(Fk(3)).unwrap()); @@ -229,13 +264,14 @@ fn strong_store_on_another_thread_during_flush_is_refused() { let flush_t = { let t = Arc::clone(&t); std::thread::spawn(move || { - t.flush_dirty_between(|_, step| { - if step == FlushStep::Write { - started.send(()).unwrap(); - release_rx.recv().unwrap(); - } - }) - .unwrap(); + t.table + .flush_dirty_between(|_, step| { + if step == FlushStep::Write { + started.send(()).unwrap(); + release_rx.recv().unwrap(); + } + }) + .unwrap(); }) }; start_rx.recv().unwrap(); diff --git a/docs/concurrency.md b/docs/concurrency.md index 62c9e3767b..f0dbd47585 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -137,7 +137,10 @@ and no environment knob for these choices. capped by how many workers are parked. A worker is on that idle list only while it is not holding a job. The epoch check still runs after the debug park delay, so a wave published after the steal check is not missed. -`steal_publish_wakes_idle_workers` is the pin. +`steal_publish_wakes_idle_workers` is the pin. A claim increments +`in_wave`, then release-increments `next`. `is_complete` acquire-loads +`next` before it reads `in_wave`, so it does not drop the wave under a +claim that still applies. **Fixed.** `serve_getdata_tail` used to call `ChainHub::knows_header` on the session worker for every block `getdata`. An unknown hash took @@ -256,6 +259,29 @@ Single Class A writer is intentional. Multi‑GiB **FdOnly grow** is fallocate-o (no remap). Hash heads do not rewrite occupied tables while serving; a leftover undersized `header.head` may be **rewritten via `header.head.grow` then rename** once on open. Class C tip tables use L2 write-behind (barrier `flush_class_c_tip`); large tables stay L0. +In-RAM gets of `confirmed`, `header_txs`, and `strong_tx` take no reader-writer +lock. `strong_tx` packs its bits into the same slot buffer and flush as those +arrays. Its length is the body in bytes and is not rounded up to a word. +The writer advances a sequence counter to odd, release-fences, stores +the slots, and advances the counter to even. The odd advance does not publish +those stores: a release on it would publish only the writes that came before +it, and a `SeqCst` increment does not either. A get loads the counter, loads +the slots, acquire-fences, and keeps the sample only when the counter is still +that even value. An acquire load of the counter does not by itself keep the +slot loads from moving after it. Gets are not `SeqCst`. +`header_txs` `get_range` uses one counter for the first and the count. A +column load acquires that store, then the same acquire fence checks the +counter again. A flush excludes new stores, then reads the dirty range and +the live slots. The odd bit and the flushing flag are `SeqCst` in both +directions. A `SeqCst` fence sits between the odd store and the flag load, +and between the flag store and the flusher's counter load. A relaxed odd bit +and a release store of the flag do not exclude each other, and a `SeqCst` +access is not that fence. The flag clears with `SeqCst` after the disk length +and the clean mark, which stay release stores. The flush does not bump the +counter, so a get during the flush does not retry because of the flush. Readers stay up +during IBD. Growth keeps the previous buffer until the table is dropped. A +get retries when the length it sampled does not fit the buffer it loaded. A +bulk bit scan rechecks the counter once at the end. See **[io-modality.md](./io-modality.md)** for operator IO levers. ### Confirm load read pipeline diff --git a/docs/ibd-memory.md b/docs/ibd-memory.md index 4fe9c92e5b..eded100497 100644 --- a/docs/ibd-memory.md +++ b/docs/ibd-memory.md @@ -173,7 +173,7 @@ intentional: | **Open `tx.head` map** | **0 heap**; 128 MiB `file=` per unsealed segment (one; two while a seal is in flight) | Read-only `MAP_SHARED` view of the slot file; probes and inserts hop it, so a probe costs no syscall. Kernel reclaim may drop its pages; the next hop faults the file. A meta v1 head gets its slot fingerprints written from `txid.body` on first open (one mix per row, `rebuild_workers` threads, once). `head_map_bytes` in the sizes line only; it is not heap and not in the in-flight retain budget. | | **Sealed BDZ `g`** | **0 heap** | Header only; 4 KiB `g` pages via uring stream (`KIND_MPHF_G`). Hot pages are kernel `RssFile`. FdOnly because a mapped miss serializes one fault per lookup thread; the ring keeps 128 pages in flight. Do not mmap packed `g`. | | **SH BDZ3 occupancy** | **supers heap** (`mphf_occ=`); ~150 MiB `file=` at ~1 B SH keys | Read-only map of the `NN.mphf` prefix through occ (not tags). Rank popcounts mapped bytes. | -| **Class C L2 `strong_tx`** | **~177 MiB** | 1 bit/create, under the 256 MiB in-RAM cap. Stays process `Vec`: `MAP_SHARED` would write before the tip barrier; `MAP_PRIVATE` COWs `set_bit` back into anon. | +| **Class C L2 `strong_tx`** | **~177 MiB** | 1 bit/create, under the 256 MiB in-RAM cap. Stays a process-resident bit image: `MAP_SHARED` would write before the tip barrier; `MAP_PRIVATE` COWs `set_bit` back into anon. | | **Mempool schema 4** | Slot table ~6 MiB at 128k + body weight | InRam `slots`/`body` Vecs + `pwrite`. Live set is also the graph. Not mapped. | | **`height_by_hash`** | **~60 MiB** | In-process confirmed hash→height map. Incremental on tip extend/shrink; full `0..=tip` walk on open / invalidate only. | | **mimalloc arenas** | **anon − accounted** after IBD | Product bins are global mimalloc. IBD allocates/frees GiB-class transients; `free` is not `munmap`. After tip catch-up, `anon=` can stay fat while `accounted=` collapses. That is allocator residue, not a store leak. Fuse mmap does not mean RSS equals fuse. Do not `malloc_trim` a mimalloc process. Optional operator: `MIMALLOC_PURGE_DELAY=0`. The `mimalloc` crate does not expose `mi_collect` on `MiMalloc` (no in-process purge hook). | diff --git a/docs/io-modality.md b/docs/io-modality.md index 6d36ecc454..51fedd69e4 100644 --- a/docs/io-modality.md +++ b/docs/io-modality.md @@ -11,7 +11,8 @@ cache keeps the view coherent). Unix `mmap` `PROT_READ`/`MAP_SHARED`, Windows `MapViewOfFile` `PAGE_READONLY`. Packed MPHF `g` stays FdOnly 4 KiB: a mapped miss is one synchronous fault on the lookup thread, while `KIND_MPHF_G` keeps 128 pages in flight on the completion -session. `strong_tx` and mempool stay process `Vec`. +session. `strong_tx` stays a process-resident bit image. Mempool stays +process `Vec`. **Map vs FdOnly:** map when the bytes are immutable after write, touched as a few random bytes per key, and must stay resident for the hot path (sealed @@ -131,7 +132,7 @@ If a change seems to require collapsing a machine, **stop and ask**. |------|----------------------|------| | **L0** | Kernel page cache via pread/pwrite; process holds staging only | Payload then HWM publish; `sync_data` on flush barriers | | **L1** | 4 KiB head pages / 3–4 KiB SH chunks (working-set caches) | Write-back dirty page/chunk with one pwrite | -| **L2** | Compact Class C (`confirmed`, `header_txs_*`, `strong_tx`) full `Vec` in process | **Write-behind:** RAM mutate during commit; complete-or-fail body image on `flush_class_c_tip` ([when](./crash-recovery.md#flush)) | +| **L2** | Compact Class C (`confirmed`, `header_txs_*`, `strong_tx`) full image in process | **Write-behind:** RAM mutate during commit; live slots on `flush_class_c_tip` ([when](./crash-recovery.md#flush)) | **Never L2:** `txout` / `seqsigwit` / `spent`, full `tx.head` / `*.idx`. `RBITCOIN_CLASS_C_INRAM_MAX_MB` (default 256) caps **`confirmed`** and From 0ae80739705234670527e332b032d35b3d558eaa Mon Sep 17 00:00:00 2001 From: Brandon Black Date: Mon, 26 Oct 2026 06:44:42 -0700 Subject: [PATCH 04/20] store: model the Class C counter and the script-pool wake The loom and shuttle harness follows phlip9's #1214: cfg-gated model tests, a separate target directory, and the same pull-request, push, dispatch, and nightly jobs. This pull request is an alternative to that ArrayTable model. Those scenarios stay on #1214. The models here are the script-pool claim order, the idle turn, the UTXO view stamp, the Class C sequence counter, and Class C flush exclusion. A script-pool claim release-publishes next after in_wave. --- .github/workflows/model.yml | 63 +++ Cargo.lock | 397 +++++++++++++++++- TESTING.md | 21 + changelog.d/class-c-snapshot.md | 10 + crates/rbitcoin-consensus/Cargo.toml | 10 + crates/rbitcoin-consensus/src/lib.rs | 2 + crates/rbitcoin-consensus/src/script_pool.rs | 142 ++++--- crates/rbitcoin-consensus/src/sync.rs | 41 ++ .../rbitcoin-consensus/tests/model/claim.rs | 55 +++ .../rbitcoin-consensus/tests/model/engine.rs | 84 ++++ crates/rbitcoin-consensus/tests/model/idle.rs | 31 ++ crates/rbitcoin-consensus/tests/model/main.rs | 17 + crates/rbitcoin-store/Cargo.toml | 7 + crates/rbitcoin-store/tests/model/engine.rs | 49 +++ crates/rbitcoin-store/tests/model/flush.rs | 54 +++ crates/rbitcoin-store/tests/model/main.rs | 15 + crates/rbitcoin-store/tests/model/seq.rs | 46 ++ crates/rbitcoin-store/tests/model/utxo.rs | 36 ++ deny.toml | 2 + scripts/model-check.sh | 26 ++ 20 files changed, 1040 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/model.yml create mode 100644 crates/rbitcoin-consensus/src/sync.rs create mode 100644 crates/rbitcoin-consensus/tests/model/claim.rs create mode 100644 crates/rbitcoin-consensus/tests/model/engine.rs create mode 100644 crates/rbitcoin-consensus/tests/model/idle.rs create mode 100644 crates/rbitcoin-consensus/tests/model/main.rs create mode 100644 crates/rbitcoin-store/tests/model/engine.rs create mode 100644 crates/rbitcoin-store/tests/model/flush.rs create mode 100644 crates/rbitcoin-store/tests/model/main.rs create mode 100644 crates/rbitcoin-store/tests/model/seq.rs create mode 100644 crates/rbitcoin-store/tests/model/utxo.rs create mode 100755 scripts/model-check.sh diff --git a/.github/workflows/model.yml b/.github/workflows/model.yml new file mode 100644 index 0000000000..7133de94b8 --- /dev/null +++ b/.github/workflows/model.yml @@ -0,0 +1,63 @@ +# Model-checked concurrency (`--test model`): loom explores script-pool claim +# order, the UTXO view stamp, and the Class C sequence counter. shuttle runs +# the script-pool idle turn. Not a required PR check. The engine is a +# `--cfg`, not a feature, so each job has its own target dir and cache. The +# nightly schedule raises the budget (TESTING.md) and seeds shuttle with the +# run id. The index-release order is not a model: it needs a Query full of +# std locks. + +name: model + +on: + pull_request: + push: + branches: [master, main, "v*.*.x"] + schedule: + - cron: "23 6 * * *" + workflow_dispatch: + +concurrency: + group: model-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: -D warnings + # Actions only. rust-cache restores dep rlibs (not workspace crates) and + # strips incremental artifacts before persist; it also sets this so rustc + # does not write throwaway dep-graphs. + CARGO_INCREMENTAL: "0" + +jobs: + model: + name: ${{ matrix.engine }} + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + # A loom failure and a shuttle failure are separate findings. + fail-fast: false + matrix: + engine: [loom, shuttle] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@035fceb3c12fb0c6a9461b52775e16473133e591 # 1.95.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: model-${{ matrix.engine }} + workspaces: ". -> target/model-${{ matrix.engine }}" + - name: model-check ${{ matrix.engine }} + if: github.event_name != 'schedule' + run: ./scripts/model-check.sh ${{ matrix.engine }} + - name: model-check ${{ matrix.engine }} (nightly deep budget) + if: github.event_name == 'schedule' + env: + LOOM_MAX_PREEMPTIONS: "4" + MODEL_SHUTTLE_ITERS: "50000" + MODEL_SHUTTLE_DEPTH: "5" + SHUTTLE_RANDOM_SEED: ${{ github.run_id }} + run: ./scripts/model-check.sh ${{ matrix.engine }} diff --git a/Cargo.lock b/Cargo.lock index c06bc0879d..664a2301d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "arc-swap" version = "1.9.3" @@ -52,12 +61,24 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.8.9" @@ -253,6 +274,18 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "buffer_sv2" version = "3.0.1" @@ -353,6 +386,25 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03efed02df0504d71e44bfd51d3329f401b8303a2fd14254acf05c95a0af0153" + +[[package]] +name = "corosensei" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6886a0c0f263965933c438626e7179139a62b978a33aa18281cbf0cd5a975f34" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "scopeguard", + "windows-sys 0.59.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -422,7 +474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -449,6 +501,12 @@ dependencies = [ "binary_sv2", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.34" @@ -488,6 +546,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be81b31c9ce4caf7f3178eeb90662bb1864d3ad363ec6bdada3201adff19c37" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -530,6 +603,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hex-conservative" version = "0.1.2" @@ -694,6 +773,12 @@ dependencies = [ "cpufeatures 0.3.1", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + [[package]] name = "libc" version = "0.2.191" @@ -715,6 +800,28 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -750,7 +857,7 @@ checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -771,6 +878,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -783,6 +899,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "owo-colors" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -851,6 +973,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.8" @@ -881,6 +1009,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", +] + [[package]] name = "rbitcoin-bench" version = "0.8.99" @@ -908,10 +1045,12 @@ dependencies = [ "arc-swap", "bitcoin", "bitcoin_hashes 0.14.101", + "loom", "mutants", "rbitcoin-primitives", "ruzstd", "serde_json", + "shuttle", ] [[package]] @@ -1081,6 +1220,7 @@ dependencies = [ "getrandom 0.4.3", "io-uring", "libc", + "loom", "rbitcoin-log", "rbitcoin-primitives", ] @@ -1128,6 +1268,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "rustversion" version = "1.0.23" @@ -1149,6 +1306,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "secp256k1" version = "0.28.2" @@ -1266,12 +1435,90 @@ dependencies = [ "sponge-cursor", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "shuttle" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f796ae4996c3fbf039c26d69536f8e112f49fd5e9a25fd353fe4787134b7cb" +dependencies = [ + "bitvec", + "cfg-if", + "const-siphasher", + "corosensei", + "hex", + "owo-colors", + "rand", + "rand_core", + "rand_pcg", + "scoped-tls", + "shuttle-engine", + "shuttle-schedulers", + "shuttle-std", + "tracing", +] + +[[package]] +name = "shuttle-engine" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf952ac286e32a98c2f1540b44747ee93a3435a4835eb266eeb53658b9782ca" +dependencies = [ + "assoc", + "bitvec", + "cfg-if", + "const-siphasher", + "corosensei", + "hex", + "owo-colors", + "rand", + "rand_core", + "rand_pcg", + "scoped-tls", + "smallvec", + "tracing", +] + +[[package]] +name = "shuttle-schedulers" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7673a579e1660404067f5aa1344d6530f15b59f53b0fe98767d7ab117d14067" +dependencies = [ + "rand", + "rand_pcg", + "shuttle-engine", + "smallvec", + "tracing", +] + +[[package]] +name = "shuttle-std" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05fa35d22b2671e0184a13d0221055ae9e07a570862798e866118acd4be50f15" +dependencies = [ + "assoc", + "owo-colors", + "shuttle-engine", + "smallvec", + "tracing", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -1301,7 +1548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1344,6 +1591,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "template_distribution_sv2" version = "7.0.0" @@ -1353,6 +1606,15 @@ dependencies = [ "binary_sv2", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "tokio" version = "1.54.1" @@ -1366,7 +1628,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1458,6 +1720,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1488,6 +1780,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "version_check" version = "0.9.5" @@ -1506,6 +1804,24 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1515,6 +1831,79 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "zerocopy" version = "0.8.57" diff --git a/TESTING.md b/TESTING.md index 7519d9393e..05664a9a44 100644 --- a/TESTING.md +++ b/TESTING.md @@ -134,6 +134,7 @@ Host **gnu** objects are not interchangeable with **musl** release or with | **Dev** | `target/dev` (`CARGO_TARGET_DIR`; `nix-shell` / `nix develop` set `$PWD/target/dev` when unset — rustup users should export it) | fmt, clippy, `cargo test`, ad-hoc `cargo build` | | **Coverage** | `target/cov` (forced in `scripts/coverage.sh`) | `./scripts/coverage.sh` only | | **Musl release** | Nix store via crane (`cargoArtifacts` + app) | `nix build .#rbitcoin-musl` — **not** `./target` | +| **Model** | `target/model-loom`, `target/model-shuttle` (`CARGO_TARGET_DIR_MODEL` overrides the script default) | `./scripts/model-check.sh loom` and `./scripts/model-check.sh shuttle`. Not part of `cargo test --workspace`. | Override dev dir only when intentional: `CARGO_TARGET_DIR=…` (Nix shell reads it; rustup users export it). Override coverage dir: @@ -219,6 +220,26 @@ cargo test -p rbitcoin-ibd --test suite frontier_competing -- --ignored --test-t The default six-block P2P cases still cover out-of-order body delivery and frontier reassignment after EOF without waiting for header timeout intervals. +## Model checking + +`./scripts/model-check.sh` builds `--test model` with `--cfg loom` or `--cfg shuttle` and its own target dir. These jobs are not required PR checks. [`.github/workflows/model.yml`](./.github/workflows/model.yml) runs them on pull request, push, and dispatch. The nightly cron raises the budget: loom preemption bound 4 (`LOOM_MAX_PREEMPTIONS`), shuttle 50_000 iterations at PCT depth 5 (`MODEL_SHUTTLE_ITERS`, `MODEL_SHUTTLE_DEPTH`), seeded with the GitHub run id. The harness follows phlip9's #1214: cfg-gated model tests, a separate target directory, and the same pull-request and nightly jobs. This pull request is an alternative to that ArrayTable model. Those scenarios stay on #1214. + +| Engine | Subject | What a bad schedule shows | +|--------|---------|---------------------------| +| loom | script-pool `claim_chunk` | `next` advances before `in_wave`, so `is_complete` frees the wave under a claim that still applies | +| shuttle | script-pool idle turn (`park_if_still_idle`) | the worker parks after the epoch bump and the unpark have both missed it. The check does not start the OS pool | +| loom | `Store::hold_utxo_view` / `utxo_view_stamp` | a stamp is visible while the hold is live | +| loom | Class C sequence counter (`get_sample`, `get_range`) | a new length arrives with the slot from before that store, or a header first/count pair mixes two generations | +| loom | Class C flush exclusion (`set`, `flush_dirty_between`) | a store returns Ok, the flush marks the table clean, and the slot it sampled is the one left on disk | + +A `Relaxed` counter, or publishing a length before its slot stores, fails the sequence-counter model. One counter covers a header first/count pair. Header pairs take a relaxed odd step and a release fence before the column stores. Array and `strong_tx` stores take a sequentially consistent odd step, then that same release fence. The reader's second counter load sits behind an acquire fence. That model stays at preemption bound 2: the schedule is already millions of executions, so the nightly bound does not raise it. The bound does not explore every aarch64 reorder of a release on the odd advance, so the fence order is part of the protocol and not something this model is claimed to exhaust. It also does not flush, so it does not cover exclusion. + +The flush model calls `ArrayTable::set` and `flush_dirty_between`. A relaxed odd increment and a release store of `flushing` fail it: the store returns Ok, the mark is clean, and disk still holds the slot the flush sampled (`disk=5`, `sampled=5`). The odd bit and `flushing` are `SeqCst` in both directions. A `SeqCst` fence between the odd store and the flag load, and between the flag store and the counter load, is the order this model checks: loom 0.7 does not order `SeqCst` accesses with each other. `check_at_most(3)` keeps the model at preemption bound 3 when the nightly job sets `LOOM_MAX_PREEMPTIONS=4`. The pull-request budget stays 2, which is enough for that lost store. `strong_tx` uses that same flush. Its body length is in bytes. + +`enqueue_sh_pending` then `release_index_writebehind` is not a model. The enqueue takes `Query`'s scripthash pending and RAM-head locks. Reproducing that order needs a `Query` full of `std` locks, so this harness leaves it out. + +PR budget: loom preemption bound 2, shuttle 2_000 PCT iterations and 2_000 random iterations at depth 3, seed `0x5eed_a77a_7ab1_e5ee`. `MODEL_LOOM_EXHAUSTIVE=1` removes the loom bound. + ## Coverage | Metric | Required | diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index 65f5437dda..0c03119131 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -24,3 +24,13 @@ Changed pass the flag and still be marked clean. The flush does not bump the counter. Growth keeps the previous buffer. A get retries when the length it sampled does not fit that buffer. +- **A script-pool claim release-publishes `next` after `in_wave`.** + `is_complete` acquire-loads `next`, so it does not drop the wave under + a claim that still applies. +- **Loom and shuttle model the claim order, the idle turn, the UTXO view + stamp, the Class C sequence counter, and Class C flush exclusion.** + They are not required PR checks. The index-release order is not a model: + it needs a `Query` full of `std` locks. The job shape follows phlip9's + #1214: cfg-gated model tests, a separate target directory, and the same + pull-request and nightly jobs. This change is an alternative to that + ArrayTable model. Those scenarios stay on #1214. diff --git a/crates/rbitcoin-consensus/Cargo.toml b/crates/rbitcoin-consensus/Cargo.toml index f65b9e1e58..ce663bd504 100644 --- a/crates/rbitcoin-consensus/Cargo.toml +++ b/crates/rbitcoin-consensus/Cargo.toml @@ -43,5 +43,15 @@ name = "suite" path = "tests/suite.rs" required-features = ["corpus"] +[[test]] +name = "model" +path = "tests/model/main.rs" + +[target.'cfg(loom)'.dependencies] +loom = "0.7" + +[target.'cfg(shuttle)'.dependencies] +shuttle = "0.9" + [lints] workspace = true diff --git a/crates/rbitcoin-consensus/src/lib.rs b/crates/rbitcoin-consensus/src/lib.rs index 3ca796f46d..e4c5e4a983 100644 --- a/crates/rbitcoin-consensus/src/lib.rs +++ b/crates/rbitcoin-consensus/src/lib.rs @@ -24,6 +24,8 @@ pub mod script_pool; pub mod signet; pub mod silent_payments; #[doc(hidden)] +pub mod sync; +#[doc(hidden)] pub mod trust; #[doc(hidden)] pub mod tx_precompute; diff --git a/crates/rbitcoin-consensus/src/script_pool.rs b/crates/rbitcoin-consensus/src/script_pool.rs index dcf15fa0d6..f5e91672e9 100644 --- a/crates/rbitcoin-consensus/src/script_pool.rs +++ b/crates/rbitcoin-consensus/src/script_pool.rs @@ -13,6 +13,7 @@ //! //! No rayon / crossbeam. +use crate::sync::atomic::Ordering as ModelOrdering; use arc_swap::ArcSwap; use std::collections::VecDeque; use std::future::Future; @@ -56,8 +57,8 @@ pub struct Wave { /// Jobs taken per successful steal. Script verify uses [`STEAL_CHUNK`]. /// An index window claims one height. pub chunk: usize, - pub next: AtomicUsize, - pub in_wave: AtomicUsize, + pub next: crate::sync::atomic::AtomicUsize, + pub in_wave: crate::sync::atomic::AtomicUsize, pub failed: AtomicBool, pub first_err: Mutex>, /// When set, warn-and-apply script faults are kept and the rest of the @@ -85,10 +86,10 @@ impl Wave { if wave.failed.load(Ordering::Acquire) { return None; } - // `in_wave` before `next`: a last-chunk claimer is visible to - // `is_complete` before `next >= n`, so the publisher cannot free ctx - // under a worker about to `apply`. - wave.in_wave.fetch_add(1, Ordering::AcqRel); + // `in_wave` before `next`. `next` is the release, and `is_complete` + // acquire-loads it, so that load sees this increment. A relaxed + // `next` can become visible first and look like the claim finished. + wave.in_wave.fetch_add(1, ModelOrdering::AcqRel); let hold = InWave { wave: Arc::clone(wave), }; @@ -96,7 +97,7 @@ impl Wave { return None; } let chunk = wave.chunk.max(1); - let i = wave.next.fetch_add(chunk, Ordering::Relaxed); + let i = wave.next.fetch_add(chunk, ModelOrdering::Release); if i >= wave.n { return None; } @@ -110,8 +111,8 @@ impl Wave { #[doc(hidden)] pub fn is_complete(&self) -> bool { let claimed_out = - self.next.load(Ordering::Relaxed) >= self.n || self.failed.load(Ordering::Acquire); - claimed_out && self.in_wave.load(Ordering::Acquire) == 0 + self.next.load(ModelOrdering::Acquire) >= self.n || self.failed.load(Ordering::Acquire); + claimed_out && self.in_wave.load(ModelOrdering::Acquire) == 0 } #[doc(hidden)] @@ -143,14 +144,14 @@ impl Wave { #[doc(hidden)] pub fn has_unclaimed(&self) -> bool { - !self.failed.load(Ordering::Acquire) && self.next.load(Ordering::Relaxed) < self.n + !self.failed.load(Ordering::Acquire) && self.next.load(ModelOrdering::Relaxed) < self.n } fn unclaimed_chunks(&self) -> usize { if !self.has_unclaimed() { return 0; } - let next = self.next.load(Ordering::Relaxed); + let next = self.next.load(ModelOrdering::Relaxed); let chunk = self.chunk.max(1); self.n.saturating_sub(next).div_ceil(chunk) } @@ -234,7 +235,7 @@ pub struct InWave { impl Drop for InWave { fn drop(&mut self) { - self.wave.in_wave.fetch_sub(1, Ordering::AcqRel); + self.wave.in_wave.fetch_sub(1, ModelOrdering::AcqRel); self.wave.notify_if_complete(); } } @@ -433,8 +434,8 @@ pub fn start_wave( let wave = Arc::new(Wave { n: items.len(), chunk, - next: AtomicUsize::new(0), - in_wave: AtomicUsize::new(0), + next: crate::sync::atomic::AtomicUsize::new(0), + in_wave: crate::sync::atomic::AtomicUsize::new(0), failed: AtomicBool::new(false), first_err: Mutex::new(None), keep_faults: AtomicBool::new(keep_faults), @@ -464,8 +465,8 @@ pub type Job = Box; #[doc(hidden)] pub struct ScriptWorkers { - pub jobs: Mutex>, - pub epoch: AtomicUsize, + pub jobs: crate::sync::Mutex>, + pub epoch: crate::sync::atomic::AtomicUsize, } #[doc(hidden)] @@ -485,10 +486,7 @@ pub static STEAL_UNPARKS: AtomicUsize = AtomicUsize::new(0); #[doc(hidden)] pub fn take_detached_job(pool: &ScriptWorkers) -> Option { - pool.jobs - .lock() - .unwrap_or_else(|p| p.into_inner()) - .pop_front() + crate::sync::lock(&pool.jobs).pop_front() } #[doc(hidden)] @@ -511,7 +509,7 @@ pub fn steal_or_job(pool: &ScriptWorkers) -> bool { #[doc(hidden)] pub fn wake_steal_workers() { let pool = workers(); - pool.epoch.fetch_add(1, Ordering::Release); + pool.epoch.fetch_add(1, ModelOrdering::Release); unpark_idle(queued_wake_count(pool)); } @@ -519,11 +517,7 @@ fn work_pending(pool: &ScriptWorkers) -> bool { if fg_has_unclaimed() { return true; } - !pool - .jobs - .lock() - .unwrap_or_else(|p| p.into_inner()) - .is_empty() + !crate::sync::lock(&pool.jobs).is_empty() } fn queued_wake_count(pool: &ScriptWorkers) -> usize { @@ -531,34 +525,40 @@ fn queued_wake_count(pool: &ScriptWorkers) -> usize { for wave in waves_snap().load().iter() { n = n.saturating_add(wave.unclaimed_chunks()); } - let jobs = pool.jobs.lock().unwrap_or_else(|p| p.into_inner()).len(); + let jobs = crate::sync::lock(&pool.jobs).len(); n.saturating_add(jobs) } -static IDLE_THREADS: Mutex> = Mutex::new(Vec::new()); +static IDLE_THREADS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| crate::sync::Mutex::new(Vec::new())); -fn register_idle_worker() { - let current = thread::current(); - let mut idle = IDLE_THREADS.lock().unwrap_or_else(|p| p.into_inner()); +#[doc(hidden)] +pub fn register_idle(idle: &crate::sync::Mutex>) { + let current = crate::sync::thread::current(); + let mut idle = crate::sync::lock(idle); if idle.iter().all(|parked| parked.id() != current.id()) { idle.push(current); } } -fn unregister_idle_worker() { - let id = thread::current().id(); - IDLE_THREADS - .lock() - .unwrap_or_else(|p| p.into_inner()) - .retain(|parked| parked.id() != id); +#[doc(hidden)] +pub fn unregister_idle(idle: &crate::sync::Mutex>) { + let id = crate::sync::thread::current().id(); + crate::sync::lock(idle).retain(|parked| parked.id() != id); } fn unpark_idle(n: usize) { + unpark_idle_from(&IDLE_THREADS, n); +} + +/// Wake up to `n` threads on this idle list. The OS pool uses [`IDLE_THREADS`]. +#[doc(hidden)] +pub fn unpark_idle_from(idle: &crate::sync::Mutex>, n: usize) { if n == 0 { return; } let taken = { - let mut idle = IDLE_THREADS.lock().unwrap_or_else(|p| p.into_inner()); + let mut idle = crate::sync::lock(idle); let n = n.min(idle.len()); idle.drain(..n).collect::>() }; @@ -569,12 +569,46 @@ fn unpark_idle(n: usize) { } } +/// Register, recheck, then park. Returns false when work or a newer epoch +/// arrived before park, and true after a park that was woken. +/// +/// The second check is the lost-wakeup close: a publisher that bumps the +/// epoch after the first check must either be seen here or find this thread +/// already on `idle` so its unpark is a park permit. +#[doc(hidden)] +pub fn park_if_still_idle( + pool: &ScriptWorkers, + epoch: usize, + idle: &crate::sync::Mutex>, +) -> bool { + register_idle(idle); + let missed = work_pending(pool) || pool.epoch.load(ModelOrdering::Acquire) != epoch; + if missed { + unregister_idle(idle); + return false; + } + #[cfg(debug_assertions)] + maybe_delay_park(); + let missed = work_pending(pool) || pool.epoch.load(ModelOrdering::Acquire) != epoch; + if missed { + unregister_idle(idle); + return false; + } + #[cfg(debug_assertions)] + IDLE_WAITERS.fetch_add(1, Ordering::SeqCst); + crate::sync::thread::park(); + #[cfg(debug_assertions)] + IDLE_WAITERS.fetch_sub(1, Ordering::SeqCst); + unregister_idle(idle); + true +} + #[doc(hidden)] pub fn workers() -> &'static ScriptWorkers { static SPAWN: OnceLock<()> = OnceLock::new(); let pool = WORKERS.get_or_init(|| ScriptWorkers { - jobs: Mutex::new(VecDeque::new()), - epoch: AtomicUsize::new(0), + jobs: crate::sync::Mutex::new(VecDeque::new()), + epoch: crate::sync::atomic::AtomicUsize::new(0), }); SPAWN.get_or_init(|| { let n = thread::available_parallelism() @@ -592,37 +626,17 @@ pub fn workers() -> &'static ScriptWorkers { if steal_or_job(pool) { continue; } - let epoch = pool.epoch.load(Ordering::Acquire); + let epoch = pool.epoch.load(ModelOrdering::Acquire); if steal_or_job(pool) { continue; } - if pool.epoch.load(Ordering::Acquire) != epoch { + if pool.epoch.load(ModelOrdering::Acquire) != epoch { continue; } - register_idle_worker(); // Do not run a job while registered. A busy worker on // the idle list would take a wake meant for one that // is actually parked. - let missed = - work_pending(pool) || pool.epoch.load(Ordering::Acquire) != epoch; - if missed { - unregister_idle_worker(); - continue; - } - #[cfg(debug_assertions)] - maybe_delay_park(); - let missed = - work_pending(pool) || pool.epoch.load(Ordering::Acquire) != epoch; - if missed { - unregister_idle_worker(); - continue; - } - #[cfg(debug_assertions)] - IDLE_WAITERS.fetch_add(1, Ordering::SeqCst); - thread::park(); - #[cfg(debug_assertions)] - IDLE_WAITERS.fetch_sub(1, Ordering::SeqCst); - unregister_idle_worker(); + let _ = park_if_still_idle(pool, epoch, &IDLE_THREADS); } }) else { @@ -717,7 +731,7 @@ where { let pool = workers(); { - let mut q = pool.jobs.lock().unwrap_or_else(|p| p.into_inner()); + let mut q = crate::sync::lock(&pool.jobs); q.push_back(Box::new(work)); } wake_steal_workers(); diff --git a/crates/rbitcoin-consensus/src/sync.rs b/crates/rbitcoin-consensus/src/sync.rs new file mode 100644 index 0000000000..82c83a7ccb --- /dev/null +++ b/crates/rbitcoin-consensus/src/sync.rs @@ -0,0 +1,41 @@ +//! `std` synchronization behind one import. `--cfg loom` and `--cfg shuttle` +//! swap in that model's primitives. Production is `std`. The two cfgs are +//! exclusive. The OS script pool still spawns `std::thread`s; the idle turn +//! and the claim atomics go through this module so a model can run them. + +#[cfg(all(loom, shuttle))] +compile_error!("--cfg loom and --cfg shuttle are exclusive"); + +#[cfg(loom)] +pub use loom::{ + sync::{atomic, Mutex}, + thread, +}; + +#[cfg(shuttle)] +pub use shuttle::{ + sync::{atomic, Mutex}, + thread, +}; + +#[cfg(not(any(loom, shuttle)))] +pub use std::{ + sync::{atomic, Mutex}, + thread, +}; + +/// `lock` returns the guard. A poisoned mutex yields the guard. +#[cfg(not(any(loom, shuttle)))] +pub fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +#[cfg(loom)] +pub fn lock(mutex: &Mutex) -> loom::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +#[cfg(shuttle)] +pub fn lock(mutex: &Mutex) -> shuttle::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} diff --git a/crates/rbitcoin-consensus/tests/model/claim.rs b/crates/rbitcoin-consensus/tests/model/claim.rs new file mode 100644 index 0000000000..9329d41bad --- /dev/null +++ b/crates/rbitcoin-consensus/tests/model/claim.rs @@ -0,0 +1,55 @@ +//! `in_wave` is published before `next`. A publisher that sees +//! [`Wave::is_complete`] must not free `ctx` under a claim that still applies. + +use rbitcoin_consensus::script_pool::{Apply, Wave}; +use rbitcoin_consensus::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use rbitcoin_consensus::sync::thread; +use rbitcoin_consensus::ConsensusError; +use std::sync::Arc; + +unsafe fn nop(_: *const (), _: usize) -> Result<(), ConsensusError> { + Ok(()) +} + +#[test] +fn claim_holds_in_wave_before_next_advances() { + super::engine::check(|| { + let wave = Arc::new(Wave { + n: 1, + chunk: 1, + next: AtomicUsize::new(0), + in_wave: AtomicUsize::new(0), + failed: std::sync::atomic::AtomicBool::new(false), + first_err: std::sync::Mutex::new(None), + keep_faults: std::sync::atomic::AtomicBool::new(false), + faults: std::sync::Mutex::new(Vec::new()), + apply: Apply { + f: nop, + ctx: std::ptr::null(), + }, + done: std::sync::Mutex::new(false), + done_cv: std::sync::Condvar::new(), + }); + let freed = Arc::new(AtomicBool::new(false)); + let publishing = Arc::clone(&wave); + let flag = Arc::clone(&freed); + let publisher = thread::spawn(move || { + if publishing.is_complete() { + flag.store(true, Ordering::Release); + } + }); + let claiming = Arc::clone(&wave); + let flag = Arc::clone(&freed); + let claimer = thread::spawn(move || { + if let Some((_, hold)) = Wave::claim_chunk(&claiming) { + assert!( + !flag.load(Ordering::Acquire), + "is_complete freed ctx under an in-flight claim" + ); + drop(hold); + } + }); + claimer.join().unwrap(); + publisher.join().unwrap(); + }); +} diff --git a/crates/rbitcoin-consensus/tests/model/engine.rs b/crates/rbitcoin-consensus/tests/model/engine.rs new file mode 100644 index 0000000000..cb9be724df --- /dev/null +++ b/crates/rbitcoin-consensus/tests/model/engine.rs @@ -0,0 +1,84 @@ +//! Loom and shuttle budgets. Nightly raises them through the environment +//! (`.github/workflows/model.yml`). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +#[cfg(loom)] +const DEFAULT_PREEMPTIONS: usize = 2; + +#[cfg(shuttle)] +const SEED: u64 = 0x5eed_a77a_7ab1_e5ee; +#[cfg(shuttle)] +const PCT_DEPTH: usize = 3; +#[cfg(shuttle)] +const PCT_ITERS: usize = 2_000; +#[cfg(shuttle)] +const RANDOM_ITERS: usize = 2_000; + +#[cfg(loom)] +pub fn check(f: impl Fn() + Send + Sync + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = match std::env::var("MODEL_LOOM_EXHAUSTIVE") + .ok() + .and_then(|v| v.parse::().ok()) + { + Some(1) => None, + _ => builder.preemption_bound.or(Some(DEFAULT_PREEMPTIONS)), + }; + counted(f, |f| builder.check(move || f())); +} + +#[cfg(shuttle)] +pub fn check(f: impl Fn() + Send + Sync + 'static) { + use shuttle::scheduler::{PctScheduler, RandomScheduler}; + let iters = std::env::var("MODEL_SHUTTLE_ITERS") + .ok() + .and_then(|v| v.parse().ok()); + let depth = std::env::var("MODEL_SHUTTLE_DEPTH") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(PCT_DEPTH); + let seed = std::env::var("SHUTTLE_RANDOM_SEED") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(SEED); + counted(f, |f| { + let pct = Arc::clone(&f); + shuttle::Runner::new( + PctScheduler::new_from_seed(seed, depth, iters.unwrap_or(PCT_ITERS)), + config(), + ) + .run(move || pct()); + shuttle::Runner::new( + RandomScheduler::new_from_seed(seed, iters.unwrap_or(RANDOM_ITERS)), + config(), + ) + .run(move || f()); + }); +} + +#[cfg(shuttle)] +fn config() -> shuttle::Config { + let mut config = shuttle::Config::new(); + config.silence_warnings = true; + config +} + +fn counted( + f: F, + explore: impl FnOnce(Arc), +) { + let runs = Arc::new(AtomicUsize::new(0)); + let start = std::time::Instant::now(); + let seen = Arc::clone(&runs); + explore(Arc::new(move || { + seen.fetch_add(1, Ordering::Relaxed); + f(); + })); + eprintln!( + "model: {} executions in {:.1?}", + runs.load(Ordering::Relaxed), + start.elapsed() + ); +} diff --git a/crates/rbitcoin-consensus/tests/model/idle.rs b/crates/rbitcoin-consensus/tests/model/idle.rs new file mode 100644 index 0000000000..183f1f263d --- /dev/null +++ b/crates/rbitcoin-consensus/tests/model/idle.rs @@ -0,0 +1,31 @@ +//! Idle turn: register, recheck, park. A publisher that bumps the epoch +//! either is observed or unparks the registered worker. This does not start +//! the OS pool. + +use rbitcoin_consensus::script_pool::{park_if_still_idle, unpark_idle_from, ScriptWorkers}; +use rbitcoin_consensus::sync::atomic::{AtomicUsize, Ordering}; +use rbitcoin_consensus::sync::{thread, Mutex}; +use std::collections::VecDeque; +use std::sync::Arc; + +#[test] +fn idle_turn_observes_the_epoch_or_wakes() { + super::engine::check(|| { + let pool = Arc::new(ScriptWorkers { + jobs: Mutex::new(VecDeque::new()), + epoch: AtomicUsize::new(0), + }); + let idle = Arc::new(Mutex::new(Vec::new())); + let pool_p = Arc::clone(&pool); + let idle_p = Arc::clone(&idle); + let publisher = thread::spawn(move || { + pool_p.epoch.fetch_add(1, Ordering::Release); + unpark_idle_from(&idle_p, 1); + }); + let worker = thread::spawn(move || { + let _ = park_if_still_idle(&pool, 0, &idle); + }); + publisher.join().unwrap(); + worker.join().unwrap(); + }); +} diff --git a/crates/rbitcoin-consensus/tests/model/main.rs b/crates/rbitcoin-consensus/tests/model/main.rs new file mode 100644 index 0000000000..1baf57051b --- /dev/null +++ b/crates/rbitcoin-consensus/tests/model/main.rs @@ -0,0 +1,17 @@ +//! Model-checked script-pool claim order (loom) and idle turn (shuttle). +//! Empty of those tests unless the matching `--cfg` is set. +//! Run `./scripts/model-check.sh loom` or `./scripts/model-check.sh shuttle`. +//! +//! The index-release order (`enqueue_sh_pending` then `release_index_writebehind`) +//! is not a model here. Driving it needs a `Query` full of `std` locks. + +#[cfg(loom)] +mod claim; +#[cfg(any(loom, shuttle))] +mod engine; +#[cfg(shuttle)] +mod idle; + +#[cfg(not(any(loom, shuttle)))] +#[test] +fn model_cfg_required() {} diff --git a/crates/rbitcoin-store/Cargo.toml b/crates/rbitcoin-store/Cargo.toml index c7544be924..f216c68bc0 100644 --- a/crates/rbitcoin-store/Cargo.toml +++ b/crates/rbitcoin-store/Cargo.toml @@ -34,5 +34,12 @@ io-uring = "0.7" name = "suite" path = "tests/suite.rs" +[[test]] +name = "model" +path = "tests/model/main.rs" + +[target.'cfg(loom)'.dependencies] +loom = "0.7" + [lints] workspace = true diff --git a/crates/rbitcoin-store/tests/model/engine.rs b/crates/rbitcoin-store/tests/model/engine.rs new file mode 100644 index 0000000000..99949b104e --- /dev/null +++ b/crates/rbitcoin-store/tests/model/engine.rs @@ -0,0 +1,49 @@ +//! One `check` for loom. The bound is the PR budget. `LOOM_MAX_PREEMPTIONS` +//! overrides it. `MODEL_LOOM_EXHAUSTIVE=1` removes it. +//! +//! [`check_at_most`] caps a model. The sequence counter stays at bound 2. +//! Flush exclusion stays at bound 3. A higher nightly bound does not raise +//! either cap. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +const DEFAULT_PREEMPTIONS: usize = 2; + +pub fn check(f: impl Fn() + Send + Sync + 'static) { + check_limited(None, f); +} + +/// At most `cap` preemptions. A lower `LOOM_MAX_PREEMPTIONS` still wins. +pub fn check_at_most(cap: usize, f: impl Fn() + Send + Sync + 'static) { + check_limited(Some(cap), f); +} + +fn check_limited(cap: Option, f: impl Fn() + Send + Sync + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = match std::env::var("MODEL_LOOM_EXHAUSTIVE") + .ok() + .and_then(|v| v.parse::().ok()) + { + Some(1) => None, + _ => { + let bound = builder.preemption_bound.or(Some(DEFAULT_PREEMPTIONS)); + match (bound, cap) { + (Some(bound), Some(cap)) => Some(bound.min(cap)), + (bound, _) => bound, + } + } + }; + let runs = Arc::new(AtomicUsize::new(0)); + let start = std::time::Instant::now(); + let seen = Arc::clone(&runs); + builder.check(move || { + seen.fetch_add(1, Ordering::Relaxed); + f(); + }); + eprintln!( + "model: {} executions in {:.1?}", + runs.load(Ordering::Relaxed), + start.elapsed() + ); +} diff --git a/crates/rbitcoin-store/tests/model/flush.rs b/crates/rbitcoin-store/tests/model/flush.rs new file mode 100644 index 0000000000..cd9c05dc03 --- /dev/null +++ b/crates/rbitcoin-store/tests/model/flush.rs @@ -0,0 +1,54 @@ +//! Flush exclusion. A store that returns Ok is the slot on disk when that +//! flush left the mark clean. `get_sample` does not cover this handshake. + +use rbitcoin_primitives::TableKind; +use rbitcoin_store::array_table::{ArrayTable, FlushStep}; +use rbitcoin_store::ClassCWrite; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +#[test] +fn a_store_that_passes_the_flush_is_the_slot_on_disk() { + super::engine::check_at_most(3, || { + let dir = std::env::temp_dir().join("rbtc-flush-model"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let table = + Arc::new(ArrayTable::create(dir.join("slots.body"), TableKind::ArrayLink).unwrap()); + table.set(&ClassCWrite::for_test(), 0, 5).unwrap(); + let entered = Arc::new(AtomicBool::new(false)); + let sampled = Arc::new(AtomicU64::new(u64::MAX)); + let writing = Arc::clone(&table); + let entered_w = Arc::clone(&entered); + let writer = loom::thread::spawn(move || { + if writing.set(&ClassCWrite::for_test(), 0, 99).is_ok() { + entered_w.store(true, Ordering::Relaxed); + } + }); + let flushing = Arc::clone(&table); + let sampled_f = Arc::clone(&sampled); + let flusher = loom::thread::spawn(move || { + flushing + .flush_dirty_between(|t, step| { + if step == FlushStep::Write { + sampled_f.store(t.live_slot(0), Ordering::Relaxed); + } + }) + .unwrap(); + }); + writer.join().unwrap(); + flusher.join().unwrap(); + if entered.load(Ordering::Relaxed) + && table.unstored_from.load(Ordering::Relaxed) == u64::MAX + { + let mut buf = [0u8; 8]; + table.file.read_at(ArrayTable::offset(0), &mut buf).unwrap(); + let disk = u64::from_le_bytes(buf); + let seen = sampled.load(Ordering::Relaxed); + if disk != 99 { + panic!("committed store lost: disk={disk} mark=CLEAN entered=true sampled={seen}"); + } + } + let _ = std::fs::remove_dir_all(&dir); + }); +} diff --git a/crates/rbitcoin-store/tests/model/main.rs b/crates/rbitcoin-store/tests/model/main.rs new file mode 100644 index 0000000000..351f9f02c2 --- /dev/null +++ b/crates/rbitcoin-store/tests/model/main.rs @@ -0,0 +1,15 @@ +//! Model-checked Class C sequence counter and UTXO view stamp. +//! Empty unless `--cfg loom`. Run `./scripts/model-check.sh loom`. + +#[cfg(loom)] +mod engine; +#[cfg(loom)] +mod flush; +#[cfg(loom)] +mod seq; +#[cfg(loom)] +mod utxo; + +#[cfg(not(loom))] +#[test] +fn model_cfg_required() {} diff --git a/crates/rbitcoin-store/tests/model/seq.rs b/crates/rbitcoin-store/tests/model/seq.rs new file mode 100644 index 0000000000..d10aea1190 --- /dev/null +++ b/crates/rbitcoin-store/tests/model/seq.rs @@ -0,0 +1,46 @@ +//! Class C sequence counter. One `get_sample` is one generation: a length that +//! covers the index has that store's slot. One `get_range` is one header pair. + +use rbitcoin_primitives::{Fk, TableKind}; +use rbitcoin_store::chain::HeaderTxsTable; +use rbitcoin_store::ClassCWrite; + +#[test] +fn len_belongs_to_the_slot_and_a_header_pair_does_not_mix() { + super::engine::check_at_most(2, || { + let dir = std::env::temp_dir().join("rbtc-seq-model"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let slots = std::sync::Arc::new( + rbitcoin_store::array_table::ArrayTable::create( + dir.join("slots.body"), + TableKind::ArrayLink, + ) + .unwrap(), + ); + let headers = std::sync::Arc::new(HeaderTxsTable::create(&dir).unwrap()); + let slots_w = std::sync::Arc::clone(&slots); + let headers_w = std::sync::Arc::clone(&headers); + let writer = loom::thread::spawn(move || { + let write = ClassCWrite::for_test(); + slots_w.set(&write, 3, 99).unwrap(); + headers_w.put_range(&write, Fk(1), Fk(10), 4).unwrap(); + headers_w.put_range(&write, Fk(1), Fk(50), 9).unwrap(); + }); + let reader = loom::thread::spawn(move || { + let (value, len) = slots.get_sample(3).unwrap(); + if len > 3 { + assert_eq!(value, 99, "new length with a slot from before the store"); + } else { + assert_eq!(value, 0); + } + match headers.get_range(Fk(1)).unwrap() { + None | Some((Fk(10), 4)) | Some((Fk(50), 9)) => {} + other => panic!("torn header range {other:?}"), + } + }); + writer.join().unwrap(); + reader.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + }); +} diff --git a/crates/rbitcoin-store/tests/model/utxo.rs b/crates/rbitcoin-store/tests/model/utxo.rs new file mode 100644 index 0000000000..5370965d9f --- /dev/null +++ b/crates/rbitcoin-store/tests/model/utxo.rs @@ -0,0 +1,36 @@ +//! UTXO view stamp. While [`Store::hold_utxo_view`] is held the word is odd +//! and [`Store::utxo_view_stamp`] is `None`. + +#[test] +fn stamp_is_absent_while_the_view_is_held() { + super::engine::check(|| { + let dir = std::env::temp_dir().join("rbtc-utxo-model"); + let _ = std::fs::remove_dir_all(&dir); + // `create_tiny` does not fit the model's default stack. The view word + // is still minted inside the model, on this thread. + let made = dir.clone(); + let store = loom::thread::Builder::new() + .stack_size(8 * 1024 * 1024) + .spawn(move || std::sync::Arc::new(rbitcoin_store::Store::create_tiny(&made).unwrap())) + .unwrap() + .join() + .unwrap(); + let held = std::sync::Arc::clone(&store); + let holder = loom::thread::spawn(move || { + let guard = held.hold_utxo_view(); + assert!( + held.utxo_view_stamp().is_none(), + "stamp published an even word during the hold" + ); + drop(guard); + }); + let reader = loom::thread::spawn(move || { + if let Some(stamp) = store.utxo_view_stamp() { + assert_eq!(stamp & 1, 0, "odd stamp returned as a generation"); + } + }); + holder.join().unwrap(); + reader.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + }); +} diff --git a/deny.toml b/deny.toml index e6644094e9..e6d3369847 100644 --- a/deny.toml +++ b/deny.toml @@ -54,6 +54,8 @@ skip = [ { name = "hex-conservative", version = "=0.3.2" }, # rand 0.8 (via noise_sv2) still needs getrandom 0.2. Direct deps are 0.4. { name = "getrandom", version = "=0.2.17" }, + # shuttle's corosensei still uses windows-sys 0.59. The rest of the tree is 0.61. + { name = "windows-sys", version = "=0.59.0" }, ] [sources] diff --git a/scripts/model-check.sh b/scripts/model-check.sh new file mode 100755 index 0000000000..91e026234a --- /dev/null +++ b/scripts/model-check.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Model-checked concurrency. loom explores claim order, the UTXO view stamp, +# the Class C sequence counter, and Class C flush exclusion. shuttle runs +# the script-pool idle turn. +# The engine is a `--cfg`, not a feature, so each engine builds in its own +# target dir (TESTING.md, Artifact silos). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +engine="${1:-}" +case "$engine" in + loom | shuttle) shift ;; + *) + echo "usage: $0 [extra cargo test args...]" >&2 + exit 2 + ;; +esac + +RUSTFLAGS="${RUSTFLAGS:-} --cfg $engine" +export RUSTFLAGS="${RUSTFLAGS# }" +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR_MODEL:-$ROOT/target/model-$engine}" +cmd=(cargo test -p rbitcoin-store -p rbitcoin-consensus --test model "$@") + +exec "${cmd[@]}" From 3bf195430d742673fe22703d257aa9350792fb03 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:13:19 -0700 Subject: [PATCH 05/20] ci: model job takes rustc from rust-toolchain.toml Drop the pinned toolchain action from the loom and shuttle job. The runner rustup installs the compiler named in rust-toolchain.toml. --- .github/workflows/model.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/model.yml b/.github/workflows/model.yml index 7133de94b8..78ef69d088 100644 --- a/.github/workflows/model.yml +++ b/.github/workflows/model.yml @@ -5,6 +5,10 @@ # nightly schedule raises the budget (TESTING.md) and seeds shuttle with the # run id. The index-release order is not a model: it needs a Query full of # std locks. +# +# rustc comes from rust-toolchain.toml (the nix-shell / crane baseline). The +# runner image's rustup auto-installs that toolchain and its components on +# first cargo. name: model @@ -45,7 +49,6 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@035fceb3c12fb0c6a9461b52775e16473133e591 # 1.95.0 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: shared-key: model-${{ matrix.engine }} From f671f6b63b0209eb5e2ba5893961aadd8649b15c Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:42:19 -0700 Subject: [PATCH 06/20] store: strong bits use try_update, and CI checks rustc 1.99 fetch_update is a denied deprecation on 1.99. try_update is the same call on 1.95 and on 1.99. The rust-forward job checks that channel beside pread, after qc and test. Product rustc stays 1.95. --- .agents/skills/ship-pr/SKILL.md | 12 +++++++----- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ CONTRIBUTING.md | 12 ++++++++---- TESTING.md | 3 +++ crates/rbitcoin-store/src/chain.rs | 2 +- docs/releases.md | 2 +- rust-forward.toml | 6 ++++++ scripts/pr-checks-watch.sh | 1 + scripts/pr-checks-watch.test.sh | 4 +++- 9 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 rust-forward.toml diff --git a/.agents/skills/ship-pr/SKILL.md b/.agents/skills/ship-pr/SKILL.md index 3055d90c7c..3ea51008c7 100644 --- a/.agents/skills/ship-pr/SKILL.md +++ b/.agents/skills/ship-pr/SKILL.md @@ -75,12 +75,14 @@ Coverage and native `windows` / `macos` stay GitHub Actions; ## Push and poll -Required jobs: `qc`, `test`, `windows`, `macos`, `coverage`, and `pread`. -`qc` is fmt, ast-grep, deny, the script self-tests, clippy, then -nixos-module-eval on one runner. `pread` is GitHub Actions only: +Required jobs: `qc`, `test`, `windows`, `macos`, `coverage`, `pread`, and +`rust-forward`. `qc` is fmt, ast-grep, deny, the script self-tests, clippy, +then nixos-module-eval on one runner. `pread` is GitHub Actions only: `RBITCOIN_IO=pread cargo test -p rbitcoin-store -p rbitcoin-query` after -`qc` and `test`, in parallel with `coverage`. Do not run it in the local -gate list. The `test` job leaves `RBITCOIN_IO` unset. Mutants run nightly +`qc` and `test`, in parallel with `coverage`. `rust-forward` is the same +wait: `cargo check --workspace --all-targets` on the channel in +`rust-forward.toml`. Do not run either in the local gate list. The `test` +job leaves `RBITCOIN_IO` unset and stays on rustc 1.95. Mutants run nightly with workspace tests (`mutants.yml`); they are not a required check. Structural scan is `./scripts/ast-grep.sh`. `windows` / `macos` are native store plus `--smoke`, not operator zips. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60909ea313..92b5d06346 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,33 @@ jobs: - name: cargo test store and query run: cargo test -p rbitcoin-store -p rbitcoin-query + # Forward rustc. Same wait as pread: a red `test` does not compile it. + # Product MSRV stays rust-toolchain.toml. This job checks the dialect still + # builds on the channel in rust-forward.toml, beside pread. + rust-forward: + name: rust-forward + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [qc, test] + env: + CARGO_TARGET_DIR: target/forward + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: install forward toolchain + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-forward.toml) + test -n "$channel" + rustup toolchain install --profile minimal "$channel" + echo "RUSTUP_TOOLCHAIN=$channel" >> "$GITHUB_ENV" + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + shared-key: ci-rust-forward + workspaces: ". -> target/forward" + - name: cargo check + run: cargo check --workspace --all-targets + # Line-coverage 91% floor + CRAP --fail-above 30 (allowlist in .cargo-crap.toml). # Slow; waits for qc and test so a red check does not start # this job. Required. Job+step timeouts so a hung llvm-cov test binary diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2d45716395..92f3f41714 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,9 @@ **Nix is not required** to clone, compile, run `--smoke`, or send a PR. CI pins **rustc 1.95.0**; [`rust-toolchain.toml`](./rust-toolchain.toml) makes [rustup](https://rustup.rs) install that channel (plus `rustfmt` and `clippy`) -the first time you run `cargo` in this tree. +the first time you run `cargo` in this tree. [`rust-forward.toml`](./rust-forward.toml) +is the newer compiler the `rust-forward` job checks. It is not the compiler +you develop with. Later pulls move that channel forward. Operator Linux binaries stay the musl path in [`README.md`](./README.md) / [`OPERATOR.md`](./OPERATOR.md). This section is **developing the tree**. @@ -266,11 +268,13 @@ Do not edit [`CHANGELOG.md`](./CHANGELOG.md) on a feature branch. The release cut folds those files in. Owner: [`docs/releases.md`](./docs/releases.md). Required **jobs** match [`.github/workflows/ci.yml`](./.github/workflows/ci.yml) -(`qc`, `test`, `windows`, `macos`, `coverage`, `pread`). `qc` is fmt, ast-grep, deny, the script self-tests, +(`qc`, `test`, `windows`, `macos`, `coverage`, `pread`, `rust-forward`). `qc` is fmt, ast-grep, deny, the script self-tests, clippy, then nixos-module-eval, in that order on one runner. `pread` is the Linux libc backend for `rbitcoin-store` and `rbitcoin-query` -(`RBITCOIN_IO=pread`). It starts with `coverage`, after `qc` and `test`. -The `test` job does not set `RBITCOIN_IO`. +(`RBITCOIN_IO=pread`). `rust-forward` is `cargo check --workspace --all-targets` +on the channel in [`rust-forward.toml`](./rust-forward.toml). Both start with +`coverage`, after `qc` and `test`. The `test` job does not set `RBITCOIN_IO` +and stays on 1.95. Label **`nixos-module-runtime`** to run the NixOS module qemu test (not eval). That job is not required. diff --git a/TESTING.md b/TESTING.md index 05664a9a44..a33ccd384a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -42,6 +42,9 @@ The Linux `pread` job runs the same store and query suites with `RBITCOIN_IO=pread` after `qc` and `test`, in parallel with `coverage`. The `test` job does not set `RBITCOIN_IO`, so it stays on io_uring. `pread` is Actions-only. It is not part of the local gate list. +`rust-forward` is the same wait: `cargo check --workspace --all-targets` +on the channel in `rust-forward.toml`, beside `pread`. Product builds stay +on rustc 1.95. It is not part of the local gate list. When adding or folding a pin: diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index f1228459cd..8e4a2a3954 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -216,7 +216,7 @@ impl StrongTxTable { fn store_byte(words: &[AtomicU64], index: usize, value: u8) { let w = &words[index / 8]; let shift = (index % 8) * 8; - let _ = w.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { + let _ = w.try_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { Some((cur & !(0xffu64 << shift)) | (u64::from(value) << shift)) }); } diff --git a/docs/releases.md b/docs/releases.md index 00428c133c..4a99ca2b4d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -174,7 +174,7 @@ show as `core-functional / core-functional` and so on. | Check | Who | |-------|-----| -| `qc` `test` `windows` `macos` `coverage` `pread` | Every PR (`ci.yml`). `qc` runs fmt, ast-grep, unwrap-ratchet, deny, script self-tests, clippy, then nixos-module-eval. `pread` is `RBITCOIN_IO=pread cargo test -p rbitcoin-store -p rbitcoin-query` on Ubuntu. It `needs` `qc` and `test`, so it starts with `coverage`. The `test` job leaves `RBITCOIN_IO` unset (io_uring). | +| `qc` `test` `windows` `macos` `coverage` `pread` `rust-forward` | Every PR (`ci.yml`). `qc` runs fmt, ast-grep, unwrap-ratchet, deny, script self-tests, clippy, then nixos-module-eval. `pread` is `RBITCOIN_IO=pread cargo test -p rbitcoin-store -p rbitcoin-query` on Ubuntu. `rust-forward` is `cargo check --workspace --all-targets` on the channel in `rust-forward.toml`. Both `need` `qc` and `test`, so they start with `coverage`. The `test` job leaves `RBITCOIN_IO` unset (io_uring) and uses rustc 1.95. | | `mutants` | Nightly `47 0 * * *` (17:47 Pacific during PDT) and `workflow_dispatch` (`mutants.yml`). Workspace tests. New mutants until half of the first job's budget, then the backlog; the second job does not open another new window. 8 hour budget in two jobs. Not required. Cursor and `MISSED` lines are on the `mutants-state` branch. | | `core-functional` | Nightly, `workflow_dispatch`, label **`core-functional`**; via `release-gate.yml` on label **`release`** **or** ship version | | `overlay-functional` | Nightly (`42 6`), `workflow_dispatch`, label **`overlay-functional`**; via `release-gate.yml` on label **`release`** **or** ship version | diff --git a/rust-forward.toml b/rust-forward.toml new file mode 100644 index 0000000000..2337bd1f2d --- /dev/null +++ b/rust-forward.toml @@ -0,0 +1,6 @@ +# Forward compiler for CI job `rust-forward`. Product MSRV stays +# rust-toolchain.toml. This channel is the newest rustc the tree is held to. +# Later pulls move the channel and fix what that compiler names. +[toolchain] +channel = "1.99.0" +profile = "minimal" diff --git a/scripts/pr-checks-watch.sh b/scripts/pr-checks-watch.sh index fe5c885a0a..cbe178d979 100755 --- a/scripts/pr-checks-watch.sh +++ b/scripts/pr-checks-watch.sh @@ -18,6 +18,7 @@ REQUIRED=( macos coverage pread + rust-forward ) usage() { diff --git a/scripts/pr-checks-watch.test.sh b/scripts/pr-checks-watch.test.sh index 534382e7cf..81d9d45226 100755 --- a/scripts/pr-checks-watch.test.sh +++ b/scripts/pr-checks-watch.test.sh @@ -32,7 +32,8 @@ all_green="$( windows pass 1m \ macos pass 50s \ coverage pass 2m \ - pread pass 2m + pread pass 2m \ + rust-forward pass 2m )" rc=0 out="$(CI_PR_CHECKS_TEXT="$all_green" "$RUN" --once 2>&1)" || rc=$? @@ -47,6 +48,7 @@ analyze_pending_windows_green="$( macos pass 50s \ coverage pass 2m \ pread pass 2m \ + rust-forward pass 2m \ 'Analyze (rust)' pending 0 )" rc=0 From 81987cdce2f6ae5a649b390ee15945d3d7de540f Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:43:28 -0700 Subject: [PATCH 07/20] test: run /dev/full flush failures on Linux only Those tests open /dev/full. macOS has no such device, so the helper and both tests compile only on Linux. --- crates/rbitcoin-store/tests/array_table.rs | 2 +- crates/rbitcoin-store/tests/chain/strong_tests.rs | 2 +- crates/rbitcoin-store/tests/dev_full.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index 525bf9fdc4..4dc2198187 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -219,7 +219,7 @@ fn same_length_rewrite_publishes_logical_len() { /// A failed `write_at` leaves the recorded range dirty. The retry writes /// those slots and does not truncate the table. -#[cfg(unix)] +#[cfg(target_os = "linux")] #[test] fn failed_write_keeps_the_range_dirty() { let path = tmp_path(); diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index 5b836b14c0..f0152647b3 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -86,7 +86,7 @@ fn strong_flush_writes_live_bytes_and_refuses_a_store() { } /// A failed bit-image `write_at` leaves that range dirty. The retry writes it. -#[cfg(unix)] +#[cfg(target_os = "linux")] #[test] fn strong_failed_write_keeps_the_range_dirty() { use rbitcoin_store::array_table::FlushStep; diff --git a/crates/rbitcoin-store/tests/dev_full.rs b/crates/rbitcoin-store/tests/dev_full.rs index f718755b4d..adf5bac747 100644 --- a/crates/rbitcoin-store/tests/dev_full.rs +++ b/crates/rbitcoin-store/tests/dev_full.rs @@ -1,11 +1,11 @@ /// Points an open file descriptor at `/dev/full` until dropped. -#[cfg(unix)] +#[cfg(target_os = "linux")] pub struct DevFull { fd: std::os::fd::RawFd, saved: std::os::fd::RawFd, } -#[cfg(unix)] +#[cfg(target_os = "linux")] impl DevFull { pub fn on(file: &std::fs::File) -> Self { use std::os::fd::AsRawFd; @@ -25,7 +25,7 @@ impl DevFull { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] impl Drop for DevFull { fn drop(&mut self) { // SAFETY: `saved` is the duplicate from `on`. Restore `fd`, then close that duplicate. From a1b3f2fac2d59a09838966fe216c38c2e69176e5 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:31:41 -0700 Subject: [PATCH 08/20] store: reject a sparse array set above the length set and set_many return corrupt when an index above the length is missing, so a truncate cannot republish bytes the length no longer names. Header ranges use set_zero_gap, which stores zeros for a header that was stamped and then rejected. --- crates/rbitcoin-query/tests/spend_sync.rs | 3 +- .../tests/structure_rule_tests.rs | 11 +- crates/rbitcoin-store/src/array_table.rs | 109 ++++++++++++++---- crates/rbitcoin-store/src/chain.rs | 12 +- crates/rbitcoin-store/tests/array_table.rs | 64 +++++++--- crates/rbitcoin-store/tests/model/seq.rs | 4 +- crates/rbitcoin-store/tests/spend_durable.rs | 18 ++- 7 files changed, 161 insertions(+), 60 deletions(-) diff --git a/crates/rbitcoin-query/tests/spend_sync.rs b/crates/rbitcoin-query/tests/spend_sync.rs index c898006ab8..27af0cc060 100644 --- a/crates/rbitcoin-query/tests/spend_sync.rs +++ b/crates/rbitcoin-query/tests/spend_sync.rs @@ -7,9 +7,10 @@ use rbitcoin_query::spend_sync::SpendSync; #[test] fn shutdown_publishes_the_snapshot() { let (dir, q) = rbitcoin_query::testutil::tiny_query_labeled("spend-sync-stop"); + let pairs: Vec<_> = (0..=4u32).map(|h| (Height(h), Fk(1))).collect(); q.store() .confirmed - .set(&rbitcoin_store::ClassCWrite::for_test(), Height(4), Fk(1)) + .set_many(&rbitcoin_store::ClassCWrite::for_test(), &pairs) .unwrap(); q.store().note_spend_snapshot(4); let q = Arc::new(q); diff --git a/crates/rbitcoin-query/tests/structure_rule_tests.rs b/crates/rbitcoin-query/tests/structure_rule_tests.rs index 8ae9e07916..19c50f0710 100644 --- a/crates/rbitcoin-query/tests/structure_rule_tests.rs +++ b/crates/rbitcoin-query/tests/structure_rule_tests.rs @@ -495,13 +495,14 @@ fn plant_bip34_ancestor(q: &rbitcoin_query::Query, hash: [u8; 32]) { ..HeaderRecord::default() }) .unwrap(); + let tip = main.btc.bip34_height; + let pairs: Vec<_> = (0..tip) + .map(|h| (Height(h), rbitcoin_primitives::Fk(1))) + .chain(std::iter::once((Height(tip), hfk))) + .collect(); q.store() .confirmed - .set( - &rbitcoin_store::ClassCWrite::for_test(), - Height(main.btc.bip34_height), - hfk, - ) + .set_many(&rbitcoin_store::ClassCWrite::for_test(), &pairs) .unwrap(); q.clear_bip34_ancestor(); } diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 89d8249771..ac32da43b5 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -404,32 +404,63 @@ impl ArrayTable { Some((slots[index as usize].load(order), len)) } - pub fn set(&self, _write: &ClassCWrite, index: u64, value: u64) -> Result<(), StoreError> { - let _odd = self.begin_store()?; - if !self.in_ram { - let len = self.len.load(Ordering::Acquire); - if index >= len { - for i in len..index { - self.file.write_at(Self::offset(i), &0u64.to_le_bytes())?; - } - } - self.file - .write_at(Self::offset(index), &value.to_le_bytes())?; - if index >= len { - self.len.store(index + 1, Ordering::Release); - } - return Ok(()); - } - self.store_ram(index, index, &[(index, value)]); - Ok(()) + /// Overwrite `index < len`, or append `index == len`. A higher index is + /// corrupt: the buffer can still hold bytes above `len` after a truncate. + pub fn set(&self, write: &ClassCWrite, index: u64, value: u64) -> Result<(), StoreError> { + self.write_slots(write, index, index, &[(index, value)], false) + } + + /// Like [`Self::set`], and indexes from `len` up to `index` that this call + /// does not name become zero. Header ranges use this. A failed header can + /// sit between two bodies; that slot is empty, not the bytes truncate left. + pub fn set_zero_gap( + &self, + write: &ClassCWrite, + index: u64, + value: u64, + ) -> Result<(), StoreError> { + self.write_slots(write, index, index, &[(index, value)], true) + } + + pub fn set_many(&self, write: &ClassCWrite, pairs: &[(u64, u64)]) -> Result<(), StoreError> { + self.write_many(write, pairs, false) + } + + /// Like [`Self::set_many`]. Slots from `len` through `max` that `pairs` + /// does not name become zero. + pub fn set_many_zero_gap( + &self, + write: &ClassCWrite, + pairs: &[(u64, u64)], + ) -> Result<(), StoreError> { + self.write_many(write, pairs, true) } - pub fn set_many(&self, _write: &ClassCWrite, pairs: &[(u64, u64)]) -> Result<(), StoreError> { + fn write_many( + &self, + write: &ClassCWrite, + pairs: &[(u64, u64)], + zero_gap: bool, + ) -> Result<(), StoreError> { if pairs.is_empty() { return Ok(()); } let max_idx = pairs.iter().map(|(i, _)| *i).max().unwrap(); let min_idx = pairs.iter().map(|(i, _)| *i).min().unwrap(); + self.write_slots(write, min_idx, max_idx, pairs, zero_gap) + } + + fn write_slots( + &self, + _write: &ClassCWrite, + min_idx: u64, + max_idx: u64, + pairs: &[(u64, u64)], + zero_gap: bool, + ) -> Result<(), StoreError> { + if !zero_gap && !self.extension_is_dense(max_idx, pairs) { + return Err(StoreError::Corrupt("array set skips slots")); + } let _odd = self.begin_store()?; if !self.in_ram { let len = self.len.load(Ordering::Acquire); @@ -454,21 +485,53 @@ impl ArrayTable { } return Ok(()); } - self.store_ram(min_idx, max_idx, pairs); + self.store_ram(min_idx, max_idx, pairs, zero_gap); Ok(()) } + /// An append must name every index from `len` through `max`. Overwrites + /// below `len` may skip. A single next-slot append is dense by itself. + /// A skip would publish buffer bytes the length no longer names. + fn extension_is_dense(&self, max_idx: u64, pairs: &[(u64, u64)]) -> bool { + let len = self.len.load(Ordering::Acquire); + if max_idx <= len { + return true; + } + let width = max_idx - len + 1; + if width > pairs.len() as u64 { + return false; + } + let mut seen = vec![false; width as usize]; + for &(index, _) in pairs { + if index >= len { + seen[(index - len) as usize] = true; + } + } + seen.iter().all(|slot| *slot) + } + /// Release-store each slot, then the length. A relaxed slot store can move /// one header column while that pair's counter still looks even. - fn store_ram(&self, min_idx: u64, max_idx: u64, pairs: &[(u64, u64)]) { + fn store_ram(&self, min_idx: u64, max_idx: u64, pairs: &[(u64, u64)], zero_gap: bool) { self.slots.grow(max_idx as usize + 1); let slots = self.slots.slots(Ordering::Relaxed); + let len_now = self.len.load(Ordering::Relaxed); + let mark = if zero_gap { + if max_idx >= len_now { + for i in len_now..=max_idx { + slots[i as usize].store(0, Ordering::Release); + } + } + min_idx.min(len_now) + } else { + min_idx + }; for &(index, value) in pairs { slots[index as usize].store(value, Ordering::Release); } - let len = self.len.load(Ordering::Relaxed).max(max_idx + 1); + let len = len_now.max(max_idx + 1); self.len.store(len, Ordering::Relaxed); - self.mark_unstored(min_idx); + self.mark_unstored(mark); } pub fn truncate(&self, _write: &ClassCWrite, new_len: u64) -> Result<(), StoreError> { diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index 8e4a2a3954..5c914b6e17 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -532,8 +532,8 @@ impl HeaderTxsTable { return Err(StoreError::InvalidFk); } let _pair = seq_odd(&self.seq); - self.first.set(write, id - 1, first_tx_fk.0)?; - self.count.set(write, id - 1, u64::from(n))?; + self.first.set_zero_gap(write, id - 1, first_tx_fk.0)?; + self.count.set_zero_gap(write, id - 1, u64::from(n))?; Ok(()) } @@ -555,8 +555,8 @@ impl HeaderTxsTable { count_pairs.push((id - 1, u64::from(n))); } let _pair = seq_odd(&self.seq); - self.first.set_many(write, &first_pairs)?; - self.count.set_many(write, &count_pairs)?; + self.first.set_many_zero_gap(write, &first_pairs)?; + self.count.set_many_zero_gap(write, &count_pairs)?; Ok(()) } @@ -580,8 +580,8 @@ impl HeaderTxsTable { count_pairs.push((id - 1, u64::from(n))); } let _pair = seq_odd(&self.seq); - self.first.set_many(write, &first_pairs)?; - self.count.set_many(write, &count_pairs)?; + self.first.set_many_zero_gap(write, &first_pairs)?; + self.count.set_many_zero_gap(write, &count_pairs)?; Ok(()) } diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index 4dc2198187..b79e0d93cd 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -25,7 +25,10 @@ fn get_racing_truncate_does_not_panic() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = Arc::new(ArrayTable::create(&path, TableKind::Confirmed).unwrap()); - t.set(&rbitcoin_store::ClassCWrite::for_test(), 1023, 7) + let wide: Vec<_> = (0..1024u64) + .map(|i| (i, if i == 1023 { 7 } else { 0 })) + .collect(); + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &wide) .unwrap(); assert!(t.l2_resident_bytes() > 0, "test needs the in-RAM image"); let stop = Arc::new(AtomicBool::new(false)); @@ -44,7 +47,7 @@ fn get_racing_truncate_does_not_panic() { while std::time::Instant::now() < deadline { t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 0) .unwrap(); - t.set(&rbitcoin_store::ClassCWrite::for_test(), 1023, 7) + t.set_many(&rbitcoin_store::ClassCWrite::for_test(), &wide) .unwrap(); } stop.store(true, Ordering::Relaxed); @@ -304,8 +307,11 @@ fn array_set_get_grow_set_many_truncate_flush() { let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); assert_eq!(t.len(), 0); assert_eq!(t.get(0).unwrap(), 0); - t.set(&rbitcoin_store::ClassCWrite::for_test(), 3, 30) - .unwrap(); + t.set_many( + &rbitcoin_store::ClassCWrite::for_test(), + &[(0, 0), (1, 0), (2, 0), (3, 30)], + ) + .unwrap(); assert_eq!(t.len(), 4); assert_eq!(t.get(0).unwrap(), 0); assert_eq!(t.get(3).unwrap(), 30); @@ -316,7 +322,7 @@ fn array_set_get_grow_set_many_truncate_flush() { .unwrap(); t.set_many( &rbitcoin_store::ClassCWrite::for_test(), - &[(10, 100), (5, 50)], + &[(4, 0), (5, 50), (6, 0), (7, 0), (8, 0), (9, 0), (10, 100)], ) .unwrap(); assert_eq!(t.len(), 11); @@ -456,20 +462,13 @@ fn append_only_flush_extends_without_losing_prefix() { } #[test] -fn set_gap_zeros_prefix_and_env_cap_and_flush_async() { +fn contiguous_image_flush_async_and_env_cap() { let path = tmp_path(); let _ = std::fs::remove_file(&path); let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); - t.set(&rbitcoin_store::ClassCWrite::for_test(), 3, 42) - .unwrap(); - assert_eq!(t.len(), 4); - assert_eq!(t.get(0).unwrap(), 0); - assert_eq!(t.get(1).unwrap(), 0); - assert_eq!(t.get(2).unwrap(), 0); - assert_eq!(t.get(3).unwrap(), 42); t.set_many( &rbitcoin_store::ClassCWrite::for_test(), - &[(1, 11), (5, 55), (3, 33)], + &[(0, 0), (1, 11), (2, 0), (3, 33), (4, 0), (5, 55)], ) .unwrap(); assert_eq!(t.len(), 6); @@ -542,3 +541,40 @@ fn store_on_another_thread_during_flush_is_refused() { assert_eq!(t.get(0).unwrap(), 9); let _ = std::fs::remove_file(&path); } + +/// Truncate keeps the logical length. A later store may overwrite a slot +/// below that length or append the next index. It may not skip ahead: the +/// bytes above the length are still in the buffer, and a skip would publish +/// them again. +#[test] +fn sparse_set_after_truncate_is_corrupt() { + let path = tmp_path(); + let _ = std::fs::remove_file(&path); + let t = ArrayTable::create(&path, TableKind::Confirmed).unwrap(); + let write = rbitcoin_store::ClassCWrite::for_test(); + t.set_many(&write, &[(0, 10), (1, 11), (2, 12), (3, 13)]) + .unwrap(); + t.truncate(&write, 1).unwrap(); + assert!(matches!(t.set(&write, 3, 99), Err(StoreError::Corrupt(_)))); + assert_eq!(t.len(), 1); + assert_eq!(t.get(0).unwrap(), 10); + t.set(&write, 1, 77).unwrap(); + assert_eq!((t.len(), t.get(0).unwrap(), t.get(1).unwrap()), (2, 10, 77)); + t.set_many(&write, &[(2, 20), (3, 21)]).unwrap(); + assert_eq!(t.len(), 4); + assert_eq!(t.get(3).unwrap(), 21); + t.set(&write, 0, 1).unwrap(); + assert_eq!(t.get(0).unwrap(), 1); + assert!(matches!( + t.set_many(&write, &[(5, 50)]), + Err(StoreError::Corrupt(_)) + )); + assert_eq!(t.len(), 4); + t.set_zero_gap(&write, 6, 60).unwrap(); + assert_eq!(t.len(), 7); + assert_eq!(t.get(4).unwrap(), 0); + assert_eq!(t.get(5).unwrap(), 0); + assert_eq!(t.get(6).unwrap(), 60); + assert_eq!(t.get(0).unwrap(), 1); + let _ = std::fs::remove_file(&path); +} diff --git a/crates/rbitcoin-store/tests/model/seq.rs b/crates/rbitcoin-store/tests/model/seq.rs index d10aea1190..4d7e98bf66 100644 --- a/crates/rbitcoin-store/tests/model/seq.rs +++ b/crates/rbitcoin-store/tests/model/seq.rs @@ -23,7 +23,9 @@ fn len_belongs_to_the_slot_and_a_header_pair_does_not_mix() { let headers_w = std::sync::Arc::clone(&headers); let writer = loom::thread::spawn(move || { let write = ClassCWrite::for_test(); - slots_w.set(&write, 3, 99).unwrap(); + slots_w + .set_many(&write, &[(0, 0), (1, 0), (2, 0), (3, 99)]) + .unwrap(); headers_w.put_range(&write, Fk(1), Fk(10), 4).unwrap(); headers_w.put_range(&write, Fk(1), Fk(50), 9).unwrap(); }); diff --git a/crates/rbitcoin-store/tests/spend_durable.rs b/crates/rbitcoin-store/tests/spend_durable.rs index 8355d56a8a..6a5f2c0bea 100644 --- a/crates/rbitcoin-store/tests/spend_durable.rs +++ b/crates/rbitcoin-store/tests/spend_durable.rs @@ -37,12 +37,11 @@ fn eight_notes_do_not_publish_and_checkpoint_keeps_the_snapshot() { )); let _ = std::fs::remove_dir_all(&dir); let s = rbitcoin_store::Store::create_tiny(&dir).unwrap(); + let pairs: Vec<_> = (0..=7u32) + .map(|h| (rbitcoin_primitives::Height(h), rbitcoin_primitives::Fk(1))) + .collect(); s.confirmed - .set( - &rbitcoin_store::ClassCWrite::for_test(), - rbitcoin_primitives::Height(7), - rbitcoin_primitives::Fk(1), - ) + .set_many(&rbitcoin_store::ClassCWrite::for_test(), &pairs) .unwrap(); for _ in 0..8 { s.note_spend_snapshot(4); @@ -153,12 +152,11 @@ fn checkpoint_stops_below_a_pending_annotate() { )); let _ = std::fs::remove_dir_all(&dir); let s = rbitcoin_store::Store::create_tiny(&dir).unwrap(); + let pairs: Vec<_> = (0..=7u32) + .map(|h| (rbitcoin_primitives::Height(h), rbitcoin_primitives::Fk(1))) + .collect(); s.confirmed - .set( - &rbitcoin_store::ClassCWrite::for_test(), - rbitcoin_primitives::Height(7), - rbitcoin_primitives::Fk(1), - ) + .set_many(&rbitcoin_store::ClassCWrite::for_test(), &pairs) .unwrap(); s.note_spend_snapshot(7); s.note_spend_annotate_pending(5); From 2c37ccbef654fb880fbc079909c534d3c6b8527d Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:50:46 -0700 Subject: [PATCH 09/20] ibd: drop the coordinator lock across an archived-body clear A mutated reject still waits for the confirm-write thread to clear the body. That wait no longer holds the coordinator state lock. After the reply, the hash is marked missing only when it still occupies the height. --- crates/rbitcoin-ibd/src/ibd/events/mod.rs | 203 +++++++++++++----- crates/rbitcoin-ibd/src/ibd/mod.rs | 149 +++++++++---- .../tests/ibd/events/confirm_reject_tests.rs | 112 ++++++++++ crates/rbitcoin-query/src/lib.rs | 4 + docs/concurrency.md | 2 +- 5 files changed, 384 insertions(+), 86 deletions(-) diff --git a/crates/rbitcoin-ibd/src/ibd/events/mod.rs b/crates/rbitcoin-ibd/src/ibd/events/mod.rs index 3e09b827ab..739f213514 100644 --- a/crates/rbitcoin-ibd/src/ibd/events/mod.rs +++ b/crates/rbitcoin-ibd/src/ibd/events/mod.rs @@ -1164,6 +1164,31 @@ pub fn apply_confirm_events( last_progress: &mut Instant, feed: Option<&super::confirm::ConfirmFeed>, ) { + let pending = take_confirm_events( + st, + hub, + rx, + archive_write_next, + max_ready_shared, + last_progress, + feed, + ); + debug_assert!( + pending.is_empty(), + "coordinator must release shared around an archived clear" + ); +} + +pub(crate) fn take_confirm_events( + st: &mut IbdWorkState, + hub: &ChainHub, + rx: &std::sync::mpsc::Receiver, + archive_write_next: &AtomicU32, + max_ready_shared: &AtomicU32, + last_progress: &mut Instant, + feed: Option<&super::confirm::ConfirmFeed>, +) -> Vec { + let mut deferred = Vec::new(); while let Ok(ev) = rx.try_recv() { match ev { super::confirm::ConfirmEvent::Accepted { hash } => { @@ -1188,7 +1213,7 @@ pub fn apply_confirm_events( sender, parent, } => { - apply_confirm_reject( + if let Some(item) = apply_confirm_reject_pending( st, height, hash, @@ -1200,7 +1225,9 @@ pub fn apply_confirm_events( feed, sender, parent, - ); + ) { + deferred.push(item); + } } } } @@ -1210,6 +1237,7 @@ pub fn apply_confirm_events( } st.confirm_quiesce = false; } + deferred } #[allow(clippy::too_many_arguments)] // call-site args stay unbundled @@ -1226,6 +1254,29 @@ pub fn apply_confirm_reject( sender: Option, parent: super::confirm::RejectParent, ) { + let pending = apply_confirm_reject_pending( + st, height, hash, class, err, query, hub, batch_len, feed, sender, parent, + ); + debug_assert!( + pending.is_none(), + "coordinator must release shared around an archived clear" + ); +} + +#[allow(clippy::too_many_arguments)] // call-site args stay unbundled +pub(crate) fn apply_confirm_reject_pending( + st: &mut IbdWorkState, + height: u32, + hash: BlockHash, + class: super::confirm::ConfirmRejectClass, + err: &str, + query: Option<&rbitcoin_query::Query>, + hub: Option<&rbitcoin_net::chain::ChainHub>, + batch_len: usize, + feed: Option<&super::confirm::ConfirmFeed>, + sender: Option, + parent: super::confirm::RejectParent, +) -> Option { use super::confirm::ConfirmRejectClass; let sent = class; let class = if class == ConfirmRejectClass::ConsensusInvalid { @@ -1245,8 +1296,7 @@ pub fn apply_confirm_reject( if super::reorg::is_bad_prev_err(err) && bad_prev_parent(st, hub, hash, height) == BadPrevParent::OffPath { - apply_soft_wire_reject(st, height, hash, err, query, hub, sender); - return; + return apply_soft_wire_reject(st, height, hash, err, query, hub, sender); } if class == ConfirmRejectClass::Cascade && batch_len > 1 { if let Some(f) = feed { @@ -1255,8 +1305,7 @@ pub fn apply_confirm_reject( } } if class.is_soft() { - apply_soft_wire_reject(st, height, hash, err, query, hub, sender); - return; + return apply_soft_wire_reject(st, height, hash, err, query, hub, sender); } match class { ConfirmRejectClass::Cancelled => { @@ -1276,56 +1325,53 @@ pub fn apply_confirm_reject( apply_consensus_invalid_reject(st, height, hash, err, query, hub); } } + None } -#[doc(hidden)] -pub fn apply_soft_wire_reject( - st: &mut IbdWorkState, +/// Mutated clear that must wait until the coordinator has dropped `shared`. +pub(crate) struct DeferredArchivedClear { + pub height: u32, + pub hash: BlockHash, + pub err: String, + pub sender: Option, + pub parent: BadPrevParent, + pub retry_tip_child: bool, + pub bad_prev: bool, + /// This hash occupied `height` after the pre-clear updates. + pub was_occupant: bool, +} + +pub(crate) fn note_archived_body_clear( height: u32, hash: BlockHash, - err: &str, - query: Option<&rbitcoin_query::Query>, - hub: Option<&rbitcoin_net::chain::ChainHub>, - sender: Option, + result: Result, ) { - let bad_prev = super::reorg::is_bad_prev_err(err); - // A BadPrev whose parent is the current tip is a missing header slot, - // and one whose parent is still committing is a stale stamp. Neither is - // a competing fork. Keep the hash so densify asks for it again. - let parent = if bad_prev { - bad_prev_parent(st, hub, hash, height) - } else { - BadPrevParent::OffPath - }; - let retry_tip_child = parent != BadPrevParent::OffPath; - if bad_prev { - st.headers_done = false; - if let Some(h) = hub { - st.reorg - .register_explore(std::iter::empty::(), Some(hash)); - let rewound = super::reorg::maybe_rewind_to_best_work(st, h).unwrap_or(false); - if rewound { - return; - } - } - if !retry_tip_child && st.height_to_hash.get(&height) == Some(&hash) { - let q = query.or_else(|| hub.map(|h| h.query.as_ref())); - evict_off_path_suffix(st, height, q); + match result { + Ok(true) => { + warn!("ibd: cleared corrupt Class A body for {hash} @{height} (mutated body)") } + Ok(false) => {} + Err(e) => warn!("ibd: clear Class A body {hash} @{height}: {e}"), } - clear_hash_inflight(&mut st.slots, &mut st.inflight, hash); - if let Some(q) = query { - let _ = q.block_queue_dequeue_height(height); - if rbitcoin_net::chain::reject_is_mutated(err) { - match q.clear_archived_body(hash.as_byte_array()) { - Ok(true) => { - warn!("ibd: cleared corrupt Class A body for {hash} @{height} (mutated body)") - } - Ok(false) => {} - Err(e) => warn!("ibd: clear Class A body {hash} @{height}: {e}"), - } - } +} + +/// Post-clear reject updates. Skips them when `height` changed hands while +/// the coordinator lock was down, so the new occupant is not marked missing. +pub(crate) fn finish_deferred_archived_clear( + st: &mut IbdWorkState, + hub: Option<&rbitcoin_net::chain::ChainHub>, + item: &DeferredArchivedClear, +) { + if item.was_occupant && st.height_to_hash.get(&item.height) != Some(&item.hash) { + return; } + let height = item.height; + let hash = item.hash; + let err = item.err.as_str(); + let sender = item.sender; + let parent = item.parent; + let retry_tip_child = item.retry_tip_child; + let bad_prev = item.bad_prev; if !bad_prev || retry_tip_child { // `isolate_if_batched` leaves SoftWire only for a one-block wave, so // `hash` is the block that failed and `sender` sent that copy. The @@ -1374,6 +1420,67 @@ pub fn apply_soft_wire_reject( } } +#[doc(hidden)] +pub(crate) fn apply_soft_wire_reject( + st: &mut IbdWorkState, + height: u32, + hash: BlockHash, + err: &str, + query: Option<&rbitcoin_query::Query>, + hub: Option<&rbitcoin_net::chain::ChainHub>, + sender: Option, +) -> Option { + let bad_prev = super::reorg::is_bad_prev_err(err); + // A BadPrev whose parent is the current tip is a missing header slot, + // and one whose parent is still committing is a stale stamp. Neither is + // a competing fork. Keep the hash so densify asks for it again. + let parent = if bad_prev { + bad_prev_parent(st, hub, hash, height) + } else { + BadPrevParent::OffPath + }; + let retry_tip_child = parent != BadPrevParent::OffPath; + if bad_prev { + st.headers_done = false; + if let Some(h) = hub { + st.reorg + .register_explore(std::iter::empty::(), Some(hash)); + let rewound = super::reorg::maybe_rewind_to_best_work(st, h).unwrap_or(false); + if rewound { + return None; + } + } + if !retry_tip_child && st.height_to_hash.get(&height) == Some(&hash) { + let q = query.or_else(|| hub.map(|h| h.query.as_ref())); + evict_off_path_suffix(st, height, q); + } + } + clear_hash_inflight(&mut st.slots, &mut st.inflight, hash); + if let Some(q) = query { + let _ = q.block_queue_dequeue_height(height); + } + let item = DeferredArchivedClear { + height, + hash, + err: err.to_string(), + sender, + parent, + retry_tip_child, + bad_prev, + was_occupant: st.height_to_hash.get(&height) == Some(&hash), + }; + if query.is_some() && rbitcoin_net::chain::reject_is_mutated(err) { + if rbitcoin_store::coord_shared_lock_held() { + return Some(item); + } + if let Some(q) = query { + note_archived_body_clear(height, hash, q.clear_archived_body(hash.as_byte_array())); + } + } + finish_deferred_archived_clear(st, hub, &item); + None +} + /// Drop the work path and body queue from `height` upward. /// /// One BadPrev names one block. Later blocks of that fork stay queued, and diff --git a/crates/rbitcoin-ibd/src/ibd/mod.rs b/crates/rbitcoin-ibd/src/ibd/mod.rs index 3c5bb0907d..f7f65e3bfa 100644 --- a/crates/rbitcoin-ibd/src/ibd/mod.rs +++ b/crates/rbitcoin-ibd/src/ibd/mod.rs @@ -73,9 +73,9 @@ use dial::{ redial_want, release_peer_block_work, }; pub use events::drain_ready_peer_and_body_events; -use events::{apply_confirm_events, apply_peer_event, disconnect_all_peers, update_confirm_lag}; #[cfg(debug_assertions)] pub use events::{apply_confirm_reject, apply_headers_event}; +use events::{apply_peer_event, disconnect_all_peers, update_confirm_lag}; use exit::{ all_peers_dead_action, best_chain_remainder, dark_empty_after, empty_path_header_fan, header_lag_behind_peers, ibd_caught_up, may_spawn_redial, path_drained, @@ -193,6 +193,72 @@ fn settle_disconnected_heavier<'a>( HeldShared::lock(shared) } +/// Apply confirm events, dropping `shared` while a mutated body clear waits +/// on the confirm-write thread. The post-clear updates run after the reply, +/// and only while this hash still occupies its height. +#[allow(clippy::too_many_arguments)] // call-site args stay unbundled +fn release_archived_clears<'a>( + mut st_guard: HeldShared<'a>, + shared: &'a std::sync::Mutex, + hub: &ChainHub, + rx: &std::sync::mpsc::Receiver, + archive_write_next: &AtomicU32, + max_ready_shared: &AtomicU32, + last_progress: &mut Instant, + feed: Option<&confirm::ConfirmFeed>, +) -> HeldShared<'a> { + let pending = events::take_confirm_events( + &mut st_guard, + hub, + rx, + archive_write_next, + max_ready_shared, + last_progress, + feed, + ); + if pending.is_empty() { + return st_guard; + } + drop(st_guard); + let noted: Vec<_> = pending + .into_iter() + .inspect(|item| { + let result = hub + .query + .clear_archived_body(bitcoin::hashes::Hash::as_byte_array(&item.hash)); + events::note_archived_body_clear(item.height, item.hash, result); + }) + .collect(); + st_guard = HeldShared::lock(shared); + for item in noted { + events::finish_deferred_archived_clear(&mut st_guard, Some(hub), &item); + } + st_guard +} + +/// Drive confirm events the way the coordinator loop does. +#[doc(hidden)] +pub fn apply_confirm_events_for_test( + shared: &std::sync::Mutex, + hub: &ChainHub, + rx: &std::sync::mpsc::Receiver, +) { + let write_next = AtomicU32::new(0); + let max_ready = AtomicU32::new(0); + let mut progress = Instant::now(); + let st_guard = HeldShared::lock(shared); + let _ = release_archived_clears( + st_guard, + shared, + hub, + rx, + &write_next, + &max_ready, + &mut progress, + None, + ); +} + /// Same drop, header walk, and commit as the coordinator assign turn. #[doc(hidden)] pub fn settle_disconnected_heavier_for_test( @@ -718,18 +784,19 @@ pub async fn ibd_cancellable( let pending_redial = { let mut st_guard = HeldShared::lock(&shared); let now_cadence = Instant::now(); + let tip_before_confirm = hub.tip_height(); + st_guard = release_archived_clears( + st_guard, + &shared, + hub.as_ref(), + &confirm_ev_rx, + &archive_write_next, + &max_ready_shared, + &mut last_progress, + Some(confirm_feed.as_ref()), + ); { let st = &mut *st_guard; - let tip_before_confirm = hub.tip_height(); - apply_confirm_events( - st, - hub.as_ref(), - &confirm_ev_rx, - &archive_write_next, - &max_ready_shared, - &mut last_progress, - Some(confirm_feed.as_ref()), - ); if hub.tip_height() < tip_before_confirm { confirm_feed.clear(); } @@ -802,19 +869,22 @@ pub async fn ibd_cancellable( cadence.mark_assign(now_cadence); } - let st = &mut *st_guard; - offer_confirm_ready( - &confirm_feed, - &st.height_to_hash, - &st.path_prev, - &mut st.body, - hub.as_ref(), - &mut st.max_ready_height, - &max_ready_shared, - ); - update_confirm_lag(&confirm_lag, hub.tip_height(), st.max_ready_height); - apply_confirm_events( - st, + { + let st = &mut *st_guard; + offer_confirm_ready( + &confirm_feed, + &st.height_to_hash, + &st.path_prev, + &mut st.body, + hub.as_ref(), + &mut st.max_ready_height, + &max_ready_shared, + ); + update_confirm_lag(&confirm_lag, hub.tip_height(), st.max_ready_height); + } + st_guard = release_archived_clears( + st_guard, + &shared, hub.as_ref(), &confirm_ev_rx, &archive_write_next, @@ -822,6 +892,7 @@ pub async fn ibd_cancellable( &mut last_progress, Some(confirm_feed.as_ref()), ); + let st = &mut *st_guard; if !drain_ready_peer_and_body_events( st, hub.as_ref(), @@ -1263,19 +1334,22 @@ pub async fn ibd_cancellable( break; } let mut st_guard = HeldShared::lock(&shared); - let st = &mut *st_guard; - offer_confirm_ready( - &confirm_feed, - &st.height_to_hash, - &st.path_prev, - &mut st.body, - hub.as_ref(), - &mut st.max_ready_height, - &max_ready_shared, - ); - update_confirm_lag(&confirm_lag, hub.tip_height(), st.max_ready_height); - apply_confirm_events( - st, + { + let st = &mut *st_guard; + offer_confirm_ready( + &confirm_feed, + &st.height_to_hash, + &st.path_prev, + &mut st.body, + hub.as_ref(), + &mut st.max_ready_height, + &max_ready_shared, + ); + update_confirm_lag(&confirm_lag, hub.tip_height(), st.max_ready_height); + } + st_guard = release_archived_clears( + st_guard, + &shared, hub.as_ref(), &confirm_ev_rx, &archive_write_next, @@ -1283,6 +1357,7 @@ pub async fn ibd_cancellable( &mut last_progress, Some(confirm_feed.as_ref()), ); + let st = &mut *st_guard; if let Some(msg) = st.halt.take() { warn!("ibd: engine fault halt: {msg}"); halt_err = Some(msg); diff --git a/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs index 445b405b76..8bfd9dd274 100644 --- a/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs @@ -3388,6 +3388,118 @@ fn batched_scripts_reject_offers_the_wave_back() { rig.finish(); } +/// A mutated soft reject clears the archived body without the coordinator +/// lock, and only then marks that hash missing. +#[test] +fn mutated_clear_drops_the_coordinator_lock_before_mark_missing() { + use rbitcoin_ibd::ibd::confirm::ConfirmEvent; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + let (dir, hub) = rbitcoin_net::chain::tiny_regtest_hub_labeled("clear-drops-coord-lock"); + hub.ensure_genesis().unwrap(); + let tip = hub.tip_hash().unwrap(); + let height = hub.tip_height().unwrap() + 1; + let hash = h(0x44); + let mut st = IbdWorkState::new(Vec::new(), Some(tip), Some(height - 1)); + st.record_height(hash, height); + st.body.mark_archived(hash); + let (tx, rx) = std::sync::mpsc::sync_channel(0); + hub.query.attach_confirm_write(tx); + let shared = Arc::new(Mutex::new(st)); + let missing_during = Arc::new(AtomicBool::new(false)); + let shared_w = Arc::clone(&shared); + let missing_w = Arc::clone(&missing_during); + let worker = std::thread::spawn(move || { + let Ok(msg) = rx.recv_timeout(std::time::Duration::from_secs(2)) else { + return; + }; + let rbitcoin_query::ConfirmWriteMsg::ClearArchived { reply, .. } = msg else { + panic!("clear must be the write-channel item"); + }; + let st = shared_w.lock().unwrap_or_else(|e| e.into_inner()); + missing_w.store(st.body.is_missing(&hash), Ordering::Relaxed); + drop(st); + reply.send(Ok(false)).unwrap(); + }); + let (ev_tx, ev_rx) = std::sync::mpsc::channel(); + ev_tx + .send(ConfirmEvent::Reject { + height, + hash, + class: ConfirmRejectClass::SoftWire, + err: "consensus: bad-txnmrklroot".to_string(), + batch_len: 1, + sender: None, + parent: RejectParent::default(), + }) + .unwrap(); + rbitcoin_ibd::apply_confirm_events_for_test(&shared, &hub, &ev_rx); + worker.join().unwrap(); + let st = shared.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + !missing_during.load(Ordering::Relaxed), + "mark_missing follows the clear" + ); + assert!(st.body.is_missing(&hash)); + hub.query.detach_confirm_write(); + let _ = std::fs::remove_dir_all(dir); +} + +/// A header that takes the height while the clear waits is not marked missing. +#[test] +fn mutated_clear_skips_mark_missing_when_the_height_changes_hands() { + use rbitcoin_ibd::ibd::confirm::ConfirmEvent; + use std::sync::{Arc, Mutex}; + + let (dir, hub) = rbitcoin_net::chain::tiny_regtest_hub_labeled("clear-height-replaced"); + hub.ensure_genesis().unwrap(); + let tip = hub.tip_hash().unwrap(); + let height = hub.tip_height().unwrap() + 1; + let hash = h(0x44); + let other = h(0x45); + let mut st = IbdWorkState::new(Vec::new(), Some(tip), Some(height - 1)); + st.record_height(hash, height); + st.body.mark_archived(hash); + let (tx, rx) = std::sync::mpsc::sync_channel(0); + hub.query.attach_confirm_write(tx); + let shared = Arc::new(Mutex::new(st)); + let shared_w = Arc::clone(&shared); + let worker = std::thread::spawn(move || { + let msg = rx + .recv_timeout(std::time::Duration::from_secs(2)) + .expect("clear"); + let rbitcoin_query::ConfirmWriteMsg::ClearArchived { reply, .. } = msg else { + panic!("clear must be the write-channel item"); + }; + let mut st = shared_w.lock().unwrap_or_else(|e| e.into_inner()); + assert!(!st.body.is_missing(&hash), "mark_missing follows the clear"); + st.record_height(other, height); + drop(st); + reply.send(Ok(false)).unwrap(); + }); + let (ev_tx, ev_rx) = std::sync::mpsc::channel(); + ev_tx + .send(ConfirmEvent::Reject { + height, + hash, + class: ConfirmRejectClass::SoftWire, + err: "consensus: bad-txnmrklroot".to_string(), + batch_len: 1, + sender: None, + parent: RejectParent::default(), + }) + .unwrap(); + rbitcoin_ibd::apply_confirm_events_for_test(&shared, &hub, &ev_rx); + worker.join().unwrap(); + let st = shared.lock().unwrap_or_else(|e| e.into_inner()); + assert!(!st.body.is_missing(&hash)); + assert!(!st.body.is_missing(&other)); + assert_eq!(st.height_to_hash.get(&height), Some(&other)); + hub.query.detach_confirm_write(); + let _ = std::fs::remove_dir_all(dir); +} + /// Peer 2 serves a mutated tip+1 in one wave with tip+2. The wave reject /// is isolated, and the confirm engine retakes tip+1 and rejects it alone /// before IBD applies either reject. Applied together, the mutated sender diff --git a/crates/rbitcoin-query/src/lib.rs b/crates/rbitcoin-query/src/lib.rs index 08b33679f2..e59ee8b451 100644 --- a/crates/rbitcoin-query/src/lib.rs +++ b/crates/rbitcoin-query/src/lib.rs @@ -2452,6 +2452,10 @@ impl Query { } pub fn clear_archived_body(&self, hash: &[u8; 32]) -> Result { + debug_assert!( + !rbitcoin_store::coord_shared_lock_held(), + "archived clear waits while the coordinator holds shared" + ); let tx = self .confirm_write .lock() diff --git a/docs/concurrency.md b/docs/concurrency.md index f0dbd47585..bdbfab3303 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -10,7 +10,7 @@ Short map of who may write which tables. **Format is unstable until 1.0.** | Confirm **lookup** | 1 OS thread | pack/hold from BQ stamped Σ inputs; emit: decode BQ raw + TipOnly `head_fk` + **`take_raw` onto loadq** (`Arc` + `TxPrecompute`). Does **not** plan_batch / structure | | Confirm **load** | 1 OS thread | `confirm_wire_lookup_stamp` (structure with **loadq `pres`**, plan_batch binds carried BQ keys, skeleton bind) + pin `txout` + assemble | | Confirm **scripts** | 1 OS thread (`ibd-confirm`) publishes waves + `rbtc-scripts` steal (lock-free claim). The interpreter runs here, not on a peer IO task (`verify_tx_scripts_detached`) | **none** — pure CPU | -| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` + one `rbtc-tip-flush` for the confirm engine (the Class C barrier) | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C (`ClassCWrite`, minted on this thread, not `Send`) + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. Archived-body clear is an item on this thread's channel: it joins the in-flight tip flush, then clears and flushes, before the next Class C store. Header range puts on this path wait for that same join, including when a later phase rejects the batch. Class A body append still overlaps the flush. Class C still flushes once per confirm batch. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era on a clean run; no archive-ahead DONTNEED). A crash inside the era can leave it above the tip ([crash-recovery.md](./crash-recovery.md#class-a-archive)). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | +| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` + one `rbtc-tip-flush` for the confirm engine (the Class C barrier) | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C (`ClassCWrite`, minted on this thread, not `Send`) + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. Archived-body clear is an item on this thread's channel: it joins the in-flight tip flush, then clears and flushes, before the next Class C store. The coordinator drops its state lock for that wait, then retakes the lock and marks the hash missing only when that hash still occupies the height. Header range puts on this path wait for that same join, including when a later phase rejects the batch. Class A body append still overlaps the flush. Class C still flushes once per confirm batch. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era on a clean run; no archive-ahead DONTNEED). A crash inside the era can leave it above the tip ([crash-recovery.md](./crash-recovery.md#class-a-archive)). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | | IBD main loop | 1 tokio task | none (orchestration only; archived-body clear is sent to confirm-write, which stores it). Event drain + confirm-offer every turn; getdata assign ≤50 ms (immediate if inflight empty); main-loop `getheaders`/`locator_hashes` ≤500 ms (empty path immediate); peer stall/relative-slow and work-path hygiene ≤1 s. Full-batch header continuation stays on the Headers event. Tip hash is the snapshot confirm published with `confirmed[]` (`published_tip`); each admitted header's parent stays on the work state. Those two reads are not store IO. The write holds `tip_commit` from the tip-height check through `flush_class_c_tip`; in IBD, through starting it on `rbtc-tip-flush`, and the next `tip_commit` taker joins that flush first. IBD rewind tries that lock, and treats a flush still running as held, and retries on the next pass. | **IoSession TLS:** one completion session per OS thread (`with_thread_local`). Harvest / poison / drain / do-not-flatten: [`io-modality.md`](./io-modality.md). `RBITCOIN_IO=pread` disables the session. SH k-way merge submits 256 KiB ahead preads on that TLS session and waits only when promote needs a page that has not completed. From a69c011d009dc1f40c71ffde1e1cccf5a2715bbe Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:00:25 -0700 Subject: [PATCH 10/20] store: commit the strong bit image in one reservation strong_tx reserves the 256 MiB cap once and commits the used prefix. A grow does not allocate a second buffer. Slot tables still keep the previous buffer until the table is dropped. The logical body meter is unchanged. --- changelog.d/class-c-snapshot.md | 6 +- crates/rbitcoin-store/src/array_table.rs | 253 +++++++++++++++++- crates/rbitcoin-store/src/chain.rs | 23 +- .../tests/chain/strong_tests.rs | 48 ++++ docs/concurrency.md | 2 +- docs/ibd-memory.md | 2 +- 6 files changed, 312 insertions(+), 22 deletions(-) diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index 0c03119131..56a8ae6d1b 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -22,8 +22,10 @@ Changed flag is sequentially consistent in both directions, with a sequentially consistent fence between the odd bit and the flag, so a store cannot pass the flag and still be marked clean. The flush does not bump the - counter. Growth keeps the previous buffer. A get retries when the - length it sampled does not fit that buffer. + counter. Slot-table growth keeps the previous buffer. A get retries when + the length it sampled does not fit that buffer. `strong_tx` is one + anonymous 256 MiB reservation; a grow commits the used prefix and does + not keep a retired copy. - **A script-pool claim release-publishes `next` after `in_wave`.** `is_complete` acquire-loads `next`, so it does not drop the wave under a claim that still applies. diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index ac32da43b5..5f01d00fad 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -1,7 +1,8 @@ //! Growable dense array of little-endian `u64` values, 0-based. //! //! `confirmed` and `header_txs_*` load into RAM up to [`class_c_inram_max_bytes`]; -//! `strong_tx` is always in RAM. Stores update live slots. `flush` writes those +//! `strong_tx` is one anonymous reservation of that cap and commits the used +//! prefix. Stores update live slots. `flush` writes those //! slots in [`FLUSH_SCRATCH`] chunks and clears the unstored mark only after the //! persist succeeds. A clean table returns before new stores are excluded. Over //! the cap, tables stay fd write-through. In-RAM gets take no reader-writer lock. @@ -91,13 +92,26 @@ struct Buf { prev: Option>, } -/// In-RAM `u64` slots. Growth keeps the previous buffer on the new box. +/// In-RAM `u64` slots. A slot table keeps the previous buffer on the new box. +/// `strong_tx` uses [`BitReserve`] instead: one mapping, no retired copy. pub(crate) struct RamSlots { ptr: AtomicPtr, + reserve: Option, +} + +/// Anonymous reservation for the strong-bit image. The base does not move. +struct BitReserve { + base: AtomicPtr, + cap: usize, + committed: AtomicUsize, } impl Drop for RamSlots { fn drop(&mut self) { + if let Some(reserve) = self.reserve.take() { + release_reserve(reserve.base.load(Ordering::Relaxed), reserve.cap); + return; + } let p = self.ptr.load(Ordering::Relaxed); if p.is_null() { return; @@ -112,6 +126,7 @@ impl RamSlots { pub(crate) fn empty() -> Self { Self { ptr: AtomicPtr::new(std::ptr::null_mut()), + reserve: None, } } @@ -122,17 +137,35 @@ impl RamSlots { }); Self { ptr: AtomicPtr::new(Box::into_raw(buf)), + reserve: None, } } - pub(crate) fn grow(&self, need: usize) { + /// Reserve `cap` bytes. Nothing is committed until [`Self::grow`]. + pub(crate) fn reserve(cap: usize) -> Result { + let cap = cap.max(page_size()); + let base = map_reserve(cap)?; + Ok(Self { + ptr: AtomicPtr::new(std::ptr::null_mut()), + reserve: Some(BitReserve { + base: AtomicPtr::new(base), + cap, + committed: AtomicUsize::new(0), + }), + }) + } + + pub(crate) fn grow(&self, need: usize) -> Result<(), StoreError> { + if let Some(reserve) = &self.reserve { + return reserve.commit_words(need); + } let cur = self.ptr.load(Ordering::Relaxed); // SAFETY: `cur` is null or a box this writer published. The copy finishes // before `from_raw` moves that box onto the new buffer. let old = unsafe { cur.as_ref().map(|buf| &buf.slots[..]) }; let cap = old.map_or(0, |slots| slots.len()); if old.is_some() && need <= cap { - return; + return Ok(()); } let new_cap = if cap == 0 { need.max(1).next_power_of_two() @@ -153,18 +186,40 @@ impl RamSlots { }); let buf = Box::into_raw(Box::new(Buf { slots, prev })); self.ptr.store(buf, Ordering::Release); + Ok(()) } /// Writer (Relaxed) or flush (Acquire), after the buffer covers the index. pub(crate) fn slots(&self, order: Ordering) -> &[AtomicU64] { + if let Some(reserve) = &self.reserve { + return reserve.words(order); + } let ptr = self.ptr.load(order); // SAFETY: caller indexes a buffer this table still owns. unsafe { &(*ptr).slots } } + /// Committed bytes of a strong-bit reservation, or the published box. + pub(crate) fn committed_bytes(&self) -> u64 { + if let Some(reserve) = &self.reserve { + return reserve.committed.load(Ordering::Relaxed) as u64; + } + let ptr = self.ptr.load(Ordering::Relaxed); + if ptr.is_null() { + return 0; + } + // SAFETY: `ptr` is a box this table published. + let slots = unsafe { &(*ptr).slots }; + slots.len() as u64 * 8 + } + /// `None` when the published buffer is missing or shorter than `need`. /// A torn length can name a slot this buffer does not have. pub(crate) fn try_slots(&self, need: usize) -> Option<&[AtomicU64]> { + if let Some(reserve) = &self.reserve { + let words = reserve.words(Ordering::Acquire); + return (words.len() >= need).then_some(words); + } let ptr = self.ptr.load(Ordering::Acquire); // SAFETY: growth keeps every published buffer until this table drops. let buf = unsafe { ptr.as_ref() }?; @@ -172,6 +227,175 @@ impl RamSlots { } } +impl BitReserve { + fn commit_words(&self, need_words: usize) -> Result<(), StoreError> { + let page = page_size(); + let bytes = need_words.saturating_mul(8); + let bytes = bytes.div_ceil(page).saturating_mul(page); + let cur = self.committed.load(Ordering::Relaxed); + if bytes <= cur { + return Ok(()); + } + if bytes > self.cap { + return Err(StoreError::Corrupt( + "strong bit image exceeds the reservation", + )); + } + commit_prefix(self.base.load(Ordering::Relaxed), bytes)?; + self.committed.store(bytes, Ordering::Release); + Ok(()) + } + + fn words(&self, order: Ordering) -> &[AtomicU64] { + let n = self.committed.load(order) / 8; + if n == 0 { + return &[]; + } + let base = self.base.load(Ordering::Relaxed) as *const AtomicU64; + // SAFETY: `commit_words` published `n` aligned words at `base`. The + // mapping lives until this table drops, and the base does not move. + unsafe { std::slice::from_raw_parts(base, n) } + } +} + +/// Page size the strong-bit reservation commits in. +#[doc(hidden)] +pub fn class_c_commit_quantum() -> u64 { + page_size() as u64 +} + +fn page_size() -> usize { + static PAGE: std::sync::OnceLock = std::sync::OnceLock::new(); + *PAGE.get_or_init(|| { + #[cfg(unix)] + { + // SAFETY: `sysconf` takes a constant and returns the page size. + let n = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if n <= 0 { + 4096 + } else { + n as usize + } + } + #[cfg(not(unix))] + { + 4096 + } + }) +} + +#[cfg(unix)] +fn map_reserve(cap: usize) -> Result<*mut u8, StoreError> { + // SAFETY: anonymous private mapping. Pages stay inaccessible until commit. + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + cap, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + if ptr == libc::MAP_FAILED { + return Err(StoreError::Corrupt("strong bit reservation failed")); + } + Ok(ptr.cast()) +} + +#[cfg(unix)] +fn commit_prefix(base: *mut u8, bytes: usize) -> Result<(), StoreError> { + if bytes == 0 { + return Ok(()); + } + // SAFETY: `base`/`bytes` is a page-aligned prefix of this reservation. + let rc = unsafe { libc::mprotect(base.cast(), bytes, libc::PROT_READ | libc::PROT_WRITE) }; + if rc != 0 { + return Err(StoreError::Corrupt("strong bit commit failed")); + } + Ok(()) +} + +#[cfg(unix)] +fn release_reserve(base: *mut u8, cap: usize) { + if base.is_null() || cap == 0 { + return; + } + // SAFETY: `base`/`cap` came from `map_reserve` for this table. + unsafe { + libc::munmap(base.cast(), cap); + } +} + +#[cfg(windows)] +fn map_reserve(cap: usize) -> Result<*mut u8, StoreError> { + // SAFETY: reserve the address range. Commit happens in `commit_prefix`. + let ptr = unsafe { VirtualAlloc(std::ptr::null_mut(), cap, MEM_RESERVE, PAGE_NOACCESS) }; + if ptr.is_null() { + return Err(StoreError::Corrupt("strong bit reservation failed")); + } + Ok(ptr.cast()) +} + +#[cfg(windows)] +fn commit_prefix(base: *mut u8, bytes: usize) -> Result<(), StoreError> { + if bytes == 0 { + return Ok(()); + } + // SAFETY: `base` is inside the reservation. `bytes` is page-aligned. + let ptr = unsafe { VirtualAlloc(base.cast(), bytes, MEM_COMMIT, PAGE_READWRITE) }; + if ptr.is_null() { + return Err(StoreError::Corrupt("strong bit commit failed")); + } + Ok(()) +} + +#[cfg(windows)] +fn release_reserve(base: *mut u8, _cap: usize) { + if base.is_null() { + return; + } + // SAFETY: `base` came from `VirtualAlloc` with `MEM_RESERVE`. + unsafe { + VirtualFree(base.cast(), 0, MEM_RELEASE); + } +} + +#[cfg(windows)] +const MEM_COMMIT: u32 = 0x1000; +#[cfg(windows)] +const MEM_RESERVE: u32 = 0x2000; +#[cfg(windows)] +const MEM_RELEASE: u32 = 0x8000; +#[cfg(windows)] +const PAGE_NOACCESS: u32 = 0x01; +#[cfg(windows)] +const PAGE_READWRITE: u32 = 0x04; + +#[cfg(windows)] +extern "system" { + fn VirtualAlloc( + addr: *mut std::ffi::c_void, + size: usize, + alloc_type: u32, + protect: u32, + ) -> *mut std::ffi::c_void; + fn VirtualFree(addr: *mut std::ffi::c_void, size: usize, free_type: u32) -> i32; +} + +#[cfg(not(any(unix, windows)))] +fn map_reserve(_cap: usize) -> Result<*mut u8, StoreError> { + Err(StoreError::Corrupt("strong bit reservation failed")) +} + +#[cfg(not(any(unix, windows)))] +fn commit_prefix(_base: *mut u8, _bytes: usize) -> Result<(), StoreError> { + Err(StoreError::Corrupt("strong bit commit failed")) +} + +#[cfg(not(any(unix, windows)))] +fn release_reserve(_base: *mut u8, _cap: usize) {} + /// Evens the counter on drop. The raw pointer keeps the guard from holding /// a field borrow across the slot stores. An fd-only store holds `None`. pub(crate) struct SeqOdd(*const AtomicU64); @@ -318,8 +542,11 @@ impl ArrayTable { } /// In-RAM bit image. `bytes` is the body length and is not rounded to a word. - pub(crate) fn bit_image(file: TableFile, bytes: u64) -> Self { - Self::new(file, None, bytes, true) + /// The mapping is the 256 MiB cap. A grow commits the used prefix only. + pub(crate) fn bit_image(file: TableFile, bytes: u64) -> Result { + let mut table = Self::new(file, None, bytes, true); + table.slots = RamSlots::reserve(DEFAULT_CLASS_C_INRAM_MAX_BYTES as usize)?; + Ok(table) } fn new(file: TableFile, data: Option>, n: u64, byte_body: bool) -> Self { @@ -485,8 +712,7 @@ impl ArrayTable { } return Ok(()); } - self.store_ram(min_idx, max_idx, pairs, zero_gap); - Ok(()) + self.store_ram(min_idx, max_idx, pairs, zero_gap) } /// An append must name every index from `len` through `max`. Overwrites @@ -512,8 +738,14 @@ impl ArrayTable { /// Release-store each slot, then the length. A relaxed slot store can move /// one header column while that pair's counter still looks even. - fn store_ram(&self, min_idx: u64, max_idx: u64, pairs: &[(u64, u64)], zero_gap: bool) { - self.slots.grow(max_idx as usize + 1); + fn store_ram( + &self, + min_idx: u64, + max_idx: u64, + pairs: &[(u64, u64)], + zero_gap: bool, + ) -> Result<(), StoreError> { + self.slots.grow(max_idx as usize + 1)?; let slots = self.slots.slots(Ordering::Relaxed); let len_now = self.len.load(Ordering::Relaxed); let mark = if zero_gap { @@ -532,6 +764,7 @@ impl ArrayTable { let len = len_now.max(max_idx + 1); self.len.store(len, Ordering::Relaxed); self.mark_unstored(mark); + Ok(()) } pub fn truncate(&self, _write: &ClassCWrite, new_len: u64) -> Result<(), StoreError> { diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index 5c914b6e17..110a4f82b1 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -146,7 +146,7 @@ impl StrongTxTable { pub fn create(dir: &Path) -> Result { let file = TableFile::create(dir.join("strong_tx.body"), TableKind::StrongTx)?; Ok(Self { - table: ArrayTable::bit_image(file, 0), + table: ArrayTable::bit_image(file, 0)?, n_bits: AtomicU64::new(0), }) } @@ -158,9 +158,9 @@ impl StrongTxTable { if body > 0 { bits.read_at(FILE_HEADER_LEN as u64, &mut bytes)?; } - let table = ArrayTable::bit_image(bits, body); + let table = ArrayTable::bit_image(bits, body)?; if body > 0 { - table.slots.grow((body as usize).div_ceil(8)); + table.slots.grow((body as usize).div_ceil(8))?; Self::copy_bytes_in(table.slots.slots(Ordering::Relaxed), &bytes); } Ok(Self { @@ -169,14 +169,15 @@ impl StrongTxTable { }) } - fn ensure_held(&self, need_bits: u64) { + fn ensure_held(&self, need_bits: u64) -> Result<(), StoreError> { if need_bits <= self.n_bits.load(Ordering::Relaxed) { - return; + return Ok(()); } let need_bytes = need_bits.div_ceil(8) as usize; - self.table.slots.grow(need_bytes.div_ceil(8)); + self.table.slots.grow(need_bytes.div_ceil(8))?; self.table.len.store(need_bytes as u64, Ordering::Relaxed); self.n_bits.store(need_bits, Ordering::Relaxed); + Ok(()) } fn for_each_mask(start: u64, end: u64, mut f: impl FnMut(usize, u64)) { @@ -240,7 +241,7 @@ impl StrongTxTable { return Ok(()); } let _odd = self.table.begin_store()?; - self.ensure_held(need_bits); + self.ensure_held(need_bits)?; Ok(()) } @@ -272,6 +273,12 @@ impl StrongTxTable { self.table.l2_resident_bytes() } + /// Bytes committed for the bit image. The logical body can be shorter. + #[doc(hidden)] + pub fn committed_bytes(&self) -> u64 { + self.table.slots.committed_bytes() + } + /// Set one live body byte without the sequence counter. Flush tests use /// this to show the persist reads the image the writer would read. #[doc(hidden)] @@ -325,7 +332,7 @@ impl StrongTxTable { return Ok(()); } let _odd = self.table.begin_store()?; - self.ensure_held(end); + self.ensure_held(end)?; self.apply_bits(start, end, on); self.mark_dirty_bit(start); Ok(()) diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index f0152647b3..81f0c7928b 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -15,6 +15,54 @@ fn tmp() -> std::path::PathBuf { p } +/// Growing the bit image keeps one committed prefix. Earlier bits stay. +#[test] +fn strong_commit_is_the_used_prefix_not_a_second_image() { + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + let write = rbitcoin_store::ClassCWrite::for_test(); + t.set_bit(&write, 0, true).unwrap(); + t.set_bit(&write, 1_000, true).unwrap(); + let bytes = 4 * 1024 * 1024 + 100; + t.ensure_bits(&write, bytes * 8).unwrap(); + assert!(t.get_bit(0).unwrap()); + assert!(t.get_bit(1_000).unwrap()); + let page = rbitcoin_store::array_table::class_c_commit_quantum(); + let words = bytes.div_ceil(8); + let expect = words.saturating_mul(8).div_ceil(page) * page; + let committed = t.committed_bytes(); + assert_eq!(committed, expect, "committed {committed} prefix {expect}"); + let pow2 = words.next_power_of_two() * 8; + assert_ne!(committed, pow2, "prefix is not the next power of two"); + assert!( + t.l2_resident_bytes() == bytes, + "meter stays the logical body" + ); + drop(t); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A bit past the 256 MiB reservation is corrupt and does not commit that range. +#[test] +fn strong_bit_past_the_reservation_is_corrupt() { + use rbitcoin_store::array_table::DEFAULT_CLASS_C_INRAM_MAX_BYTES; + let dir = tmp(); + let t = StrongTxTable::create(&dir).unwrap(); + let err = t + .ensure_bits( + &rbitcoin_store::ClassCWrite::for_test(), + (DEFAULT_CLASS_C_INRAM_MAX_BYTES + 1) * 8, + ) + .unwrap_err(); + assert!( + matches!(err, StoreError::Corrupt(msg) if msg.contains("reservation")), + "{err:?}" + ); + assert_eq!(t.committed_bytes(), 0); + drop(t); + let _ = std::fs::remove_dir_all(&dir); +} + /// A bit store after the flush has noticed dirt, and before new stores are /// excluded, is still in this flush. #[test] diff --git a/docs/concurrency.md b/docs/concurrency.md index bdbfab3303..fc78d040e6 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -279,7 +279,7 @@ and a release store of the flag do not exclude each other, and a `SeqCst` access is not that fence. The flag clears with `SeqCst` after the disk length and the clean mark, which stay release stores. The flush does not bump the counter, so a get during the flush does not retry because of the flush. Readers stay up -during IBD. Growth keeps the previous buffer until the table is dropped. A +during IBD. Slot-table growth keeps the previous buffer until the table is dropped. `strong_tx` is one anonymous reservation of the 256 MiB cap: a grow commits the used prefix and does not keep a second image. A get retries when the length it sampled does not fit the buffer it loaded. A bulk bit scan rechecks the counter once at the end. See **[io-modality.md](./io-modality.md)** for operator IO levers. diff --git a/docs/ibd-memory.md b/docs/ibd-memory.md index eded100497..d687d4005b 100644 --- a/docs/ibd-memory.md +++ b/docs/ibd-memory.md @@ -173,7 +173,7 @@ intentional: | **Open `tx.head` map** | **0 heap**; 128 MiB `file=` per unsealed segment (one; two while a seal is in flight) | Read-only `MAP_SHARED` view of the slot file; probes and inserts hop it, so a probe costs no syscall. Kernel reclaim may drop its pages; the next hop faults the file. A meta v1 head gets its slot fingerprints written from `txid.body` on first open (one mix per row, `rebuild_workers` threads, once). `head_map_bytes` in the sizes line only; it is not heap and not in the in-flight retain budget. | | **Sealed BDZ `g`** | **0 heap** | Header only; 4 KiB `g` pages via uring stream (`KIND_MPHF_G`). Hot pages are kernel `RssFile`. FdOnly because a mapped miss serializes one fault per lookup thread; the ring keeps 128 pages in flight. Do not mmap packed `g`. | | **SH BDZ3 occupancy** | **supers heap** (`mphf_occ=`); ~150 MiB `file=` at ~1 B SH keys | Read-only map of the `NN.mphf` prefix through occ (not tags). Rank popcounts mapped bytes. | -| **Class C L2 `strong_tx`** | **~177 MiB** | 1 bit/create, under the 256 MiB in-RAM cap. Stays a process-resident bit image: `MAP_SHARED` would write before the tip barrier; `MAP_PRIVATE` COWs `set_bit` back into anon. | +| **Class C L2 `strong_tx`** | **~177 MiB** logical; RSS is that prefix rounded up to pages | 1 bit/create. One anonymous 256 MiB reservation, commit the used prefix. No retired copy. `class_c_l2=` stays the logical body. Not a map of `strong_tx.body`: `MAP_SHARED` would write before the tip barrier, and `MAP_PRIVATE` of the file would COW each `set_bit`. | | **Mempool schema 4** | Slot table ~6 MiB at 128k + body weight | InRam `slots`/`body` Vecs + `pwrite`. Live set is also the graph. Not mapped. | | **`height_by_hash`** | **~60 MiB** | In-process confirmed hash→height map. Incremental on tip extend/shrink; full `0..=tip` walk on open / invalidate only. | | **mimalloc arenas** | **anon − accounted** after IBD | Product bins are global mimalloc. IBD allocates/frees GiB-class transients; `free` is not `munmap`. After tip catch-up, `anon=` can stay fat while `accounted=` collapses. That is allocator residue, not a store leak. Fuse mmap does not mean RSS equals fuse. Do not `malloc_trim` a mimalloc process. Optional operator: `MIMALLOC_PURGE_DELAY=0`. The `mimalloc` crate does not expose `mi_collect` on `MiMalloc` (no in-process purge hook). | From eeab2c13949000b5ebcc8343f54286ccc700de13 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:56:19 -0700 Subject: [PATCH 11/20] store: one Class C writer wins the even-to-odd exchange Two threads can each mint ClassCWrite. The sequence counter now moves even to odd with a compare-exchange, so the second store waits. Both SeqCst fences stay. The strong-bit flush takes the same flush lock as the slot tables. Loom cannot run two waiters on that lock, so the overlap check is a normal thread test. --- crates/rbitcoin-store/src/array_table.rs | 35 +++++++++++---- crates/rbitcoin-store/src/chain.rs | 9 ++-- .../tests/chain/strong_tests.rs | 44 +++++++++++++++++++ docs/concurrency.md | 6 ++- 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 5f01d00fad..f701965f5d 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -16,9 +16,11 @@ use std::marker::PhantomData; use std::path::Path; use std::sync::Mutex; -/// One Class C writer (`!Send` + `!Sync`). A shared [`crate::Store`] cannot carry -/// it onto another thread. Confirm-write, tip-accept, and startup each mint one -/// on the storing thread. The flush thread reads live slots and does not take it. +/// Minted on the storing thread (`!Send` + `!Sync`). A shared store cannot +/// carry this value onto another thread. It does not stop that thread from +/// minting its own. The sequence counter's even-to-odd exchange is what keeps +/// two stores out of one image. Confirm-write, tip-accept, and startup each +/// mint one. The flush thread reads live slots and does not take it. #[derive(Default)] pub struct ClassCWrite(PhantomData<*const ()>); @@ -413,7 +415,7 @@ pub(crate) fn seq_begin(seq: &AtomicU64, flushing: &AtomicBool) -> Result Result SeqOdd { - seq.fetch_add(1, Ordering::Relaxed); + claim_odd(seq); atomic::fence(Ordering::Release); SeqOdd(seq as *const AtomicU64) } +/// Even to odd. A loser waits. The release fence after this is what publishes +/// the slot stores; this exchange does not. +fn claim_odd(seq: &AtomicU64) { + loop { + let cur = seq.load(Ordering::SeqCst); + if cur & 1 == 1 { + crate::sync::spin(); + continue; + } + if seq + .compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + return; + } + } +} + pub(crate) fn seq_arm(seq: &AtomicU64, flushing: &AtomicBool) { loop { let g1 = seq.load(Ordering::SeqCst); @@ -816,9 +836,8 @@ impl ArrayTable { &self, mut between: impl FnMut(&Self, FlushStep), ) -> Result<(), StoreError> { - // `strong_tx` has one flusher. Slot tables keep the lock. - let _one = - (!self.byte_body).then(|| self.flush_lock.lock().unwrap_or_else(|e| e.into_inner())); + // One flusher, including the strong-bit image. + let _one = self.flush_lock.lock().unwrap_or_else(|e| e.into_inner()); if !self.in_ram { return Ok(()); } diff --git a/crates/rbitcoin-store/src/chain.rs b/crates/rbitcoin-store/src/chain.rs index 110a4f82b1..210cb6e78f 100644 --- a/crates/rbitcoin-store/src/chain.rs +++ b/crates/rbitcoin-store/src/chain.rs @@ -217,9 +217,12 @@ impl StrongTxTable { fn store_byte(words: &[AtomicU64], index: usize, value: u8) { let w = &words[index / 8]; let shift = (index % 8) * 8; - let _ = w.try_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { - Some((cur & !(0xffu64 << shift)) | (u64::from(value) << shift)) - }); + let update = |cur: u64| Some((cur & !(0xffu64 << shift)) | (u64::from(value) << shift)); + // loom 0.7 still names this `fetch_update`. std renamed it `try_update`. + #[cfg(loom)] + let _ = w.fetch_update(Ordering::Relaxed, Ordering::Relaxed, update); + #[cfg(not(loom))] + let _ = w.try_update(Ordering::Relaxed, Ordering::Relaxed, update); } fn copy_bytes_in(words: &[AtomicU64], src: &[u8]) { diff --git a/crates/rbitcoin-store/tests/chain/strong_tests.rs b/crates/rbitcoin-store/tests/chain/strong_tests.rs index 81f0c7928b..3b3f0d4db0 100644 --- a/crates/rbitcoin-store/tests/chain/strong_tests.rs +++ b/crates/rbitcoin-store/tests/chain/strong_tests.rs @@ -63,6 +63,50 @@ fn strong_bit_past_the_reservation_is_corrupt() { let _ = std::fs::remove_dir_all(&dir); } +/// Two flushes of the bit image and one store all return. Loom cannot run +/// this: the flush lock is a real mutex, and a second waiter blocks the +/// model thread. A store that returned is on disk once the mark is clean. +#[test] +fn two_strong_flushes_keep_a_store_that_returned() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + let dir = tmp(); + let table = Arc::new(StrongTxTable::create(&dir).unwrap()); + table + .set_bit(&rbitcoin_store::ClassCWrite::for_test(), 0, true) + .unwrap(); + table.flush().unwrap(); + let stored = Arc::new(AtomicBool::new(false)); + let writing = Arc::clone(&table); + let stored_w = Arc::clone(&stored); + let writer = std::thread::spawn(move || { + if writing + .set_bit(&rbitcoin_store::ClassCWrite::for_test(), 8, true) + .is_ok() + { + stored_w.store(true, Ordering::Relaxed); + } + }); + let first = Arc::clone(&table); + let second = Arc::clone(&table); + let flush_a = std::thread::spawn(move || first.table.flush_dirty().unwrap()); + let flush_b = std::thread::spawn(move || second.table.flush_dirty().unwrap()); + writer.join().unwrap(); + flush_a.join().unwrap(); + flush_b.join().unwrap(); + if stored.load(Ordering::Relaxed) + && table.table.unstored_from.load(Ordering::Relaxed) == u64::MAX + { + let opened = StrongTxTable::open(&dir).unwrap(); + assert!( + opened.get_bit(8).unwrap(), + "store that returned is not on disk" + ); + } + drop(table); + let _ = std::fs::remove_dir_all(&dir); +} + /// A bit store after the flush has noticed dirt, and before new stores are /// excluded, is still in this flush. #[test] diff --git a/docs/concurrency.md b/docs/concurrency.md index fc78d040e6..b3060e2b48 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -262,8 +262,10 @@ write-behind (barrier `flush_class_c_tip`); large tables stay L0. In-RAM gets of `confirmed`, `header_txs`, and `strong_tx` take no reader-writer lock. `strong_tx` packs its bits into the same slot buffer and flush as those arrays. Its length is the body in bytes and is not rounded up to a word. -The writer advances a sequence counter to odd, release-fences, stores -the slots, and advances the counter to even. The odd advance does not publish +The writer advances a sequence counter from even to odd with a +compare-exchange, so a second writer waits. The `ClassCWrite` token does +not do that exclusion. It then release-fences, stores the slots, and +advances the counter to even. The odd advance does not publish those stores: a release on it would publish only the writes that came before it, and a `SeqCst` increment does not either. A get loads the counter, loads the slots, acquire-fences, and keeps the sample only when the counter is still From a22b82cd5c6a73ed10640bd4a5951726bd16769f Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:41:51 -0700 Subject: [PATCH 12/20] query: join the tip flush before a standalone header range put The Class A commit helper stored header_txs while the previous IBD tip flush could still be excluding stores. pread keeps that flush in the window, and the next block's put then returns Rejected. Class A append still overlaps the flush. The put waits on the same tip join as confirm-write. A debug hold pauses the tip flush after it excludes stores, so the test observes the flag. --- changelog.d/class-c-snapshot.md | 5 +-- crates/rbitcoin-query/src/archive.rs | 9 ++++-- crates/rbitcoin-query/src/connect.rs | 7 +++-- crates/rbitcoin-query/tests/catchup.rs | 40 +++++++++++++++++++++++- crates/rbitcoin-store/src/array_table.rs | 37 ++++++++++++++++++++++ crates/rbitcoin-store/src/lib.rs | 3 ++ 6 files changed, 92 insertions(+), 9 deletions(-) diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index 56a8ae6d1b..db5d5ff0f3 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -3,8 +3,9 @@ Changed - **Class C stores take a single-writer capability.** `ClassCWrite` is not `Send`. Confirm-write, tip-accept, and startup mint it on the thread that stores. Archived-body clear runs on - `ibd-confirm-write` after the in-flight tip flush joins. On that path, - header range puts wait for the same join. Class A body append still + `ibd-confirm-write` after the in-flight tip flush joins. Header range + puts on that path, and on the standalone Class A commit, wait for the + same join. Class A body append still overlaps the tip flush. A structural reject after that append still stores the header ranges after the join, so a retry sees the body. Class A device sync stays on `rbtc-spend-sync` about every 10 minutes. diff --git a/crates/rbitcoin-query/src/archive.rs b/crates/rbitcoin-query/src/archive.rs index e7fa6ce5b3..cf744f3fa9 100644 --- a/crates/rbitcoin-query/src/archive.rs +++ b/crates/rbitcoin-query/src/archive.rs @@ -1720,6 +1720,9 @@ impl Query { ) -> Result<(bool, Vec), QueryError> { let (committed, loc, ranges) = self.archive_commit_plan_defer_head_parents(plan)?; if !ranges.is_empty() { + // Class A append above overlaps the in-flight tip flush. This put + // is Class C, so it waits until that flush stops excluding stores. + let _commit = self.lock_tip_commit()?; let t = std::time::Instant::now(); let write = rbitcoin_store::ClassCWrite::new(); self.store.header_txs.put_ranges_batch(&write, &ranges)?; @@ -1734,9 +1737,9 @@ impl Query { /// planner). A non-empty plan whose vectors do not match `packed` is /// corrupt. This function does not walk prevouts. It does not store /// header ranges: the returned list is Class C. [`Self::archive_commit_plan_defer_head`] - /// stores it immediately. The confirm-write path stores it after - /// `tip_commit` joins the in-flight tip flush, including when a later - /// phase of that write rejects the batch. `header_txs` is what a retry + /// stores it after `tip_commit` joins the in-flight tip flush. The + /// confirm-write path does the same put after that join, including when a + /// later phase of that write rejects the batch. `header_txs` is what a retry /// uses to skip a second body append. pub fn archive_commit_plan_defer_head_parents( &self, diff --git a/crates/rbitcoin-query/src/connect.rs b/crates/rbitcoin-query/src/connect.rs index b206dbf3c6..88f9458fac 100644 --- a/crates/rbitcoin-query/src/connect.rs +++ b/crates/rbitcoin-query/src/connect.rs @@ -279,11 +279,12 @@ impl Query { .name("rbtc-tip-flush".into()) .spawn(move || { while let Ok(job) = rx.recv() { + // Pause after the flush excludes stores, not before it starts. #[cfg(debug_assertions)] - while hold.load(Ordering::Acquire) { - std::thread::sleep(std::time::Duration::from_millis(1)); - } + rbitcoin_store::bind_tip_flush_hold(Some(Arc::clone(&hold))); let result = store.flush_class_c_tip(); + #[cfg(debug_assertions)] + rbitcoin_store::bind_tip_flush_hold(None); let _ = job.reply.send(result); job.done.store(true, Ordering::Release); } diff --git a/crates/rbitcoin-query/tests/catchup.rs b/crates/rbitcoin-query/tests/catchup.rs index 440d6129d2..d0758bcba2 100644 --- a/crates/rbitcoin-query/tests/catchup.rs +++ b/crates/rbitcoin-query/tests/catchup.rs @@ -6,10 +6,11 @@ use rbitcoin_query::Query; use rbitcoin_store::StoreError; use rbitcoin_store::{HeaderRecord, InputRecord, OutputRecord, TxRecord}; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::Ordering; use std::sync::mpsc; use std::sync::Arc; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; fn coinbase_block( h: u32, @@ -170,6 +171,43 @@ fn finalize_sh_runs_empty_head_leftover_seal_is_wiped() { let _ = std::fs::remove_dir_all(&dir); } +/// The next Class A commit stores `header_txs` while the previous tip flush +/// can still be excluding stores. That put has to wait the flush out. +#[test] +fn class_a_header_store_waits_out_the_tip_flush() { + let (dir, q) = rbitcoin_query::testutil::tiny_query_labeled("q-sh-flush-wait"); + q.enter_direct_index_mode().unwrap(); + let (h0, ta0) = coinbase_block(0, Fk::NULL, None); + let prev = q.connect_block(Height(0), &h0, &[ta0]).unwrap(); + let _commit = q.lock_tip_commit().unwrap(); + drop(_commit); + let hold = q.hold_tip_flushes(); + let (h1, ta1) = coinbase_block(1, prev, Some(h0.hash)); + let prev = q.commit_class_a_only(&h1, &[ta1]).unwrap(); + q.flush_tip(Some(Height(0))).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while !q.store.header_txs.first.flushing.load(Ordering::SeqCst) { + assert!( + Instant::now() < deadline, + "tip flush did not exclude header_txs stores" + ); + thread::sleep(Duration::from_millis(1)); + } + let hold2 = Arc::clone(&hold); + let releaser = thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + hold2.store(false, Ordering::SeqCst); + }); + let (h2, ta2) = coinbase_block(2, prev, Some(h1.hash)); + let h2_fk = q + .commit_class_a_only(&h2, &[ta2]) + .expect("header range put waits for the tip flush"); + releaser.join().unwrap(); + assert!(q.store.header_txs.has_body(h2_fk).unwrap()); + drop(q); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn force_rebuild_recollects_class_a_not_empty_materialize() { let (dir, q) = rbitcoin_query::testutil::tiny_query_labeled("q-sh-force-recol"); diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index f701965f5d..2615747c90 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -16,6 +16,40 @@ use std::marker::PhantomData; use std::path::Path; use std::sync::Mutex; +/// Debug builds pause the IBD tip-flush thread here when a test armed the hold. +/// The pause is after `seq_arm`, so a store on another thread sees `flushing`. +#[cfg(debug_assertions)] +mod flush_hold { + use std::cell::RefCell; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + thread_local! { + static HOLD: RefCell>> = const { RefCell::new(None) }; + } + + pub(super) fn bind(hold: Option>) { + HOLD.with(|slot| *slot.borrow_mut() = hold); + } + + pub(super) fn wait() { + let Some(hold) = HOLD.with(|slot| slot.borrow().clone()) else { + return; + }; + while hold.load(Ordering::Acquire) { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } +} + +/// Install the IBD tip-flush hold on this thread. [`ArrayTable::flush`] waits +/// after it excludes stores. Release builds have no hold. +#[doc(hidden)] +#[cfg(debug_assertions)] +pub fn bind_tip_flush_hold(hold: Option>) { + flush_hold::bind(hold); +} + /// Minted on the storing thread (`!Send` + `!Sync`). A shared store cannot /// carry this value onto another thread. It does not stop that thread from /// minting its own. The sequence counter's even-to-odd exchange is what keeps @@ -847,6 +881,9 @@ impl ArrayTable { } between(self, FlushStep::Admit); seq_arm(&self.seq, &self.flushing); + // After stores are excluded. A test hold pauses the tip flush here. + #[cfg(debug_assertions)] + flush_hold::wait(); let from = self.unstored_from.load(Ordering::Acquire); let n = self.len.load(Ordering::Acquire); let disk = self.disk_len.load(Ordering::Acquire); diff --git a/crates/rbitcoin-store/src/lib.rs b/crates/rbitcoin-store/src/lib.rs index 0f71222d48..67e137654e 100644 --- a/crates/rbitcoin-store/src/lib.rs +++ b/crates/rbitcoin-store/src/lib.rs @@ -123,6 +123,9 @@ pub mod uring_session; #[doc(hidden)] pub mod var_table; +#[cfg(debug_assertions)] +#[doc(hidden)] +pub use crate::array_table::bind_tip_flush_hold; pub use crate::array_table::ClassCWrite; pub use crate::chain::HeaderTxsTable; pub use crate::compact::output_flags; From 461d08eab21d60b538ccef15ee048bb403d73be1 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:54:01 -0700 Subject: [PATCH 13/20] test: declare the log suite close symbol unsafe rustc 1.99 denies a safe extern close because that name is a runtime symbol. The stderr test now calls the unsafe declaration. --- crates/rbitcoin-log/tests/root.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rbitcoin-log/tests/root.rs b/crates/rbitcoin-log/tests/root.rs index 5e0fb94711..caaf481c0c 100644 --- a/crates/rbitcoin-log/tests/root.rs +++ b/crates/rbitcoin-log/tests/root.rs @@ -175,7 +175,8 @@ fn init_from_env_off_and_level() { unsafe extern "C" { safe fn dup(fd: i32) -> i32; safe fn dup2(oldfd: i32, newfd: i32) -> i32; - safe fn close(fd: i32) -> i32; + // `close` is a runtime symbol. rustc 1.99 rejects a safe declaration. + unsafe fn close(fd: i32) -> i32; } #[cfg(unix)] @@ -200,7 +201,8 @@ fn stderr_line_is_plain_when_not_a_terminal() { log_at_style(Level::Info, Style::Plain, format_args!("hidden-info")); let _ = io::stderr().flush(); assert_eq!(dup2(saved, 2), 2); - close(saved); + // SAFETY: `saved` is the fd from `dup` above, and stderr is already restored. + unsafe { close(saved) }; let body = std::fs::read_to_string(&path).unwrap(); let _ = std::fs::remove_file(&path); init(Level::Info); From 10d19bb5a1e7e879744db20117dd31358d533135 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:01:12 -0700 Subject: [PATCH 14/20] test: split the Class C sequence loom checks One closure ran a four-slot store and two header puts in the same schedule, so bound 2 took about 15 minutes. A slot set against one get_sample, and one header put_range against a pair stored before the threads start, still fail a relaxed even publish. --- TESTING.md | 2 +- crates/rbitcoin-store/tests/model/seq.rs | 52 ++++++++++++++++-------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/TESTING.md b/TESTING.md index a33ccd384a..274ef6117e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -235,7 +235,7 @@ frontier reassignment after EOF without waiting for header timeout intervals. | loom | Class C sequence counter (`get_sample`, `get_range`) | a new length arrives with the slot from before that store, or a header first/count pair mixes two generations | | loom | Class C flush exclusion (`set`, `flush_dirty_between`) | a store returns Ok, the flush marks the table clean, and the slot it sampled is the one left on disk | -A `Relaxed` counter, or publishing a length before its slot stores, fails the sequence-counter model. One counter covers a header first/count pair. Header pairs take a relaxed odd step and a release fence before the column stores. Array and `strong_tx` stores take a sequentially consistent odd step, then that same release fence. The reader's second counter load sits behind an acquire fence. That model stays at preemption bound 2: the schedule is already millions of executions, so the nightly bound does not raise it. The bound does not explore every aarch64 reorder of a release on the odd advance, so the fence order is part of the protocol and not something this model is claimed to exhaust. It also does not flush, so it does not cover exclusion. +A `Relaxed` even publish fails both sequence closures. `get_sample` then sees a new length with the slot from before that store. `get_range` mixes two header generations (`(Fk(50), 4)`). Storing the length before the slots, while the even step stays a release, still passes: readers wait for that even value. The slot closure is one `set` against one `get_sample`. The header closure stores `(Fk(10), 4)` before either thread starts, then races one `put_range` of `(Fk(50), 9)` against one `get_range`. Each stays at preemption bound 2. A higher nightly bound does not raise that cap. The bound does not explore every aarch64 reorder of a release on the odd advance, so the fence order is part of the protocol and not something this model is claimed to exhaust. Neither closure flushes, so they do not cover exclusion. The flush model calls `ArrayTable::set` and `flush_dirty_between`. A relaxed odd increment and a release store of `flushing` fail it: the store returns Ok, the mark is clean, and disk still holds the slot the flush sampled (`disk=5`, `sampled=5`). The odd bit and `flushing` are `SeqCst` in both directions. A `SeqCst` fence between the odd store and the flag load, and between the flag store and the counter load, is the order this model checks: loom 0.7 does not order `SeqCst` accesses with each other. `check_at_most(3)` keeps the model at preemption bound 3 when the nightly job sets `LOOM_MAX_PREEMPTIONS=4`. The pull-request budget stays 2, which is enough for that lost store. `strong_tx` uses that same flush. Its body length is in bytes. diff --git a/crates/rbitcoin-store/tests/model/seq.rs b/crates/rbitcoin-store/tests/model/seq.rs index 4d7e98bf66..c8f058acf1 100644 --- a/crates/rbitcoin-store/tests/model/seq.rs +++ b/crates/rbitcoin-store/tests/model/seq.rs @@ -1,14 +1,17 @@ //! Class C sequence counter. One `get_sample` is one generation: a length that //! covers the index has that store's slot. One `get_range` is one header pair. +//! The two checks are separate closures. use rbitcoin_primitives::{Fk, TableKind}; use rbitcoin_store::chain::HeaderTxsTable; use rbitcoin_store::ClassCWrite; +/// One slot store against one sample. The length that covers the index has +/// that store's value. #[test] -fn len_belongs_to_the_slot_and_a_header_pair_does_not_mix() { +fn slot_length_carries_its_slot() { super::engine::check_at_most(2, || { - let dir = std::env::temp_dir().join("rbtc-seq-model"); + let dir = std::env::temp_dir().join("rbtc-seq-slot"); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let slots = std::sync::Arc::new( @@ -18,28 +21,45 @@ fn len_belongs_to_the_slot_and_a_header_pair_does_not_mix() { ) .unwrap(), ); - let headers = std::sync::Arc::new(HeaderTxsTable::create(&dir).unwrap()); let slots_w = std::sync::Arc::clone(&slots); - let headers_w = std::sync::Arc::clone(&headers); let writer = loom::thread::spawn(move || { - let write = ClassCWrite::for_test(); - slots_w - .set_many(&write, &[(0, 0), (1, 0), (2, 0), (3, 99)]) - .unwrap(); - headers_w.put_range(&write, Fk(1), Fk(10), 4).unwrap(); - headers_w.put_range(&write, Fk(1), Fk(50), 9).unwrap(); + slots_w.set(&ClassCWrite::for_test(), 0, 99).unwrap(); }); let reader = loom::thread::spawn(move || { - let (value, len) = slots.get_sample(3).unwrap(); - if len > 3 { + let (value, len) = slots.get_sample(0).unwrap(); + if len > 0 { assert_eq!(value, 99, "new length with a slot from before the store"); } else { assert_eq!(value, 0); } - match headers.get_range(Fk(1)).unwrap() { - None | Some((Fk(10), 4)) | Some((Fk(50), 9)) => {} - other => panic!("torn header range {other:?}"), - } + }); + writer.join().unwrap(); + reader.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + }); +} + +/// One later header pair against a pair stored before either thread starts. +/// A mix of the two generations panics. +#[test] +fn header_pair_is_one_generation() { + super::engine::check_at_most(2, || { + let dir = std::env::temp_dir().join("rbtc-seq-header"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let headers = std::sync::Arc::new(HeaderTxsTable::create(&dir).unwrap()); + headers + .put_range(&ClassCWrite::for_test(), Fk(1), Fk(10), 4) + .unwrap(); + let headers_w = std::sync::Arc::clone(&headers); + let writer = loom::thread::spawn(move || { + headers_w + .put_range(&ClassCWrite::for_test(), Fk(1), Fk(50), 9) + .unwrap(); + }); + let reader = loom::thread::spawn(move || match headers.get_range(Fk(1)).unwrap() { + None | Some((Fk(10), 4)) | Some((Fk(50), 9)) => {} + other => panic!("torn header range {other:?}"), }); writer.join().unwrap(); reader.join().unwrap(); From d328a3e9eceba8849048e6447554629863f66532 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:47:58 -0700 Subject: [PATCH 15/20] store: say the header-pair odd step is SeqCst Header pairs call the same compare-exchange as a slot store, and they have no flush flag. The release fence after that exchange publishes the pair. --- changelog.d/class-c-snapshot.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index db5d5ff0f3..8c06bbccd3 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -13,9 +13,10 @@ Changed - **In-RAM Class C gets take no reader-writer lock.** One sequence counter covers a store. The odd bit is sequentially consistent, a release fence runs before the slot stores, and the even advance publishes them. Header - pairs, which have no flush flag, still take a relaxed odd step and that - same release fence. A get acquire-fences before it checks the counter - again. `header_txs` reads first and count under one counter. A header + pairs have no flush flag. Their odd step is that same sequentially + consistent compare-exchange, then the release fence. A get acquire-fences + before it checks the counter again. `header_txs` reads first and count + under one counter. A header column load acquires that store, then the same fence checks the counter. `strong_tx` keeps its bits in that slot buffer. One flush serves the arrays and the bit image. The bit image's length is the body in bytes. From ae78634893bcd474f33f5245b652d42e3d0288a8 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:46:16 -0700 Subject: [PATCH 16/20] ibd: defer every mutated archived-body clear The classifier returns the clear and does not send it. The coordinator drops its state lock, then the caller sends the clear and applies the reject. The test entry points send the same way. --- changelog.d/class-c-snapshot.md | 3 +- crates/rbitcoin-ibd/src/ibd/events/mod.rs | 48 +++++++++----- crates/rbitcoin-ibd/src/ibd/mod.rs | 14 ++-- .../tests/ibd/events/confirm_reject_tests.rs | 65 +++++++++++++++++++ 4 files changed, 101 insertions(+), 29 deletions(-) diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index 8c06bbccd3..1fa406dfe7 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -3,7 +3,8 @@ Changed - **Class C stores take a single-writer capability.** `ClassCWrite` is not `Send`. Confirm-write, tip-accept, and startup mint it on the thread that stores. Archived-body clear runs on - `ibd-confirm-write` after the in-flight tip flush joins. Header range + `ibd-confirm-write` after the in-flight tip flush joins. The coordinator + drops its state lock before that wait. Header range puts on that path, and on the standalone Class A commit, wait for the same join. Class A body append still overlaps the tip flush. A structural reject after that append still diff --git a/crates/rbitcoin-ibd/src/ibd/events/mod.rs b/crates/rbitcoin-ibd/src/ibd/events/mod.rs index 739f213514..8d7c27b393 100644 --- a/crates/rbitcoin-ibd/src/ibd/events/mod.rs +++ b/crates/rbitcoin-ibd/src/ibd/events/mod.rs @@ -1173,13 +1173,16 @@ pub fn apply_confirm_events( last_progress, feed, ); - debug_assert!( - pending.is_empty(), - "coordinator must release shared around an archived clear" - ); + for item in &pending { + send_deferred_archived_clear(hub.query.as_ref(), item); + } + for item in &pending { + finish_deferred_archived_clear(st, Some(hub), item); + } } -pub(crate) fn take_confirm_events( +#[doc(hidden)] +pub fn take_confirm_events( st: &mut IbdWorkState, hub: &ChainHub, rx: &std::sync::mpsc::Receiver, @@ -1254,13 +1257,15 @@ pub fn apply_confirm_reject( sender: Option, parent: super::confirm::RejectParent, ) { - let pending = apply_confirm_reject_pending( + let Some(item) = apply_confirm_reject_pending( st, height, hash, class, err, query, hub, batch_len, feed, sender, parent, - ); - debug_assert!( - pending.is_none(), - "coordinator must release shared around an archived clear" - ); + ) else { + return; + }; + if let Some(q) = query { + send_deferred_archived_clear(q, &item); + } + finish_deferred_archived_clear(st, hub, &item); } #[allow(clippy::too_many_arguments)] // call-site args stay unbundled @@ -1329,7 +1334,8 @@ pub(crate) fn apply_confirm_reject_pending( } /// Mutated clear that must wait until the coordinator has dropped `shared`. -pub(crate) struct DeferredArchivedClear { +#[doc(hidden)] +pub struct DeferredArchivedClear { pub height: u32, pub hash: BlockHash, pub err: String, @@ -1341,6 +1347,17 @@ pub(crate) struct DeferredArchivedClear { pub was_occupant: bool, } +pub(crate) fn send_deferred_archived_clear( + query: &rbitcoin_query::Query, + item: &DeferredArchivedClear, +) { + note_archived_body_clear( + item.height, + item.hash, + query.clear_archived_body(item.hash.as_byte_array()), + ); +} + pub(crate) fn note_archived_body_clear( height: u32, hash: BlockHash, @@ -1470,12 +1487,7 @@ pub(crate) fn apply_soft_wire_reject( was_occupant: st.height_to_hash.get(&height) == Some(&hash), }; if query.is_some() && rbitcoin_net::chain::reject_is_mutated(err) { - if rbitcoin_store::coord_shared_lock_held() { - return Some(item); - } - if let Some(q) = query { - note_archived_body_clear(height, hash, q.clear_archived_body(hash.as_byte_array())); - } + return Some(item); } finish_deferred_archived_clear(st, hub, &item); None diff --git a/crates/rbitcoin-ibd/src/ibd/mod.rs b/crates/rbitcoin-ibd/src/ibd/mod.rs index f7f65e3bfa..54b7edf377 100644 --- a/crates/rbitcoin-ibd/src/ibd/mod.rs +++ b/crates/rbitcoin-ibd/src/ibd/mod.rs @@ -220,17 +220,11 @@ fn release_archived_clears<'a>( return st_guard; } drop(st_guard); - let noted: Vec<_> = pending - .into_iter() - .inspect(|item| { - let result = hub - .query - .clear_archived_body(bitcoin::hashes::Hash::as_byte_array(&item.hash)); - events::note_archived_body_clear(item.height, item.hash, result); - }) - .collect(); + for item in &pending { + events::send_deferred_archived_clear(hub.query.as_ref(), item); + } st_guard = HeldShared::lock(shared); - for item in noted { + for item in pending { events::finish_deferred_archived_clear(&mut st_guard, Some(hub), &item); } st_guard diff --git a/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs index 8bfd9dd274..01e0e1dc4b 100644 --- a/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-ibd/tests/ibd/events/confirm_reject_tests.rs @@ -3388,6 +3388,71 @@ fn batched_scripts_reject_offers_the_wave_back() { rig.finish(); } +/// A mutated clear is handed back before any confirm-write send. The debug +/// lock flag is not set on this path. +#[test] +fn mutated_clear_is_deferred_without_the_debug_lock_flag() { + use rbitcoin_ibd::ibd::confirm::ConfirmEvent; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + use std::sync::Arc; + use std::time::Instant; + + let (dir, hub) = rbitcoin_net::chain::tiny_regtest_hub_labeled("clear-deferred-no-flag"); + hub.ensure_genesis().unwrap(); + let tip = hub.tip_hash().unwrap(); + let height = hub.tip_height().unwrap() + 1; + let hash = h(0x46); + let mut st = IbdWorkState::new(Vec::new(), Some(tip), Some(height - 1)); + st.record_height(hash, height); + st.body.mark_archived(hash); + let (tx, rx_clear) = std::sync::mpsc::sync_channel(0); + hub.query.attach_confirm_write(tx); + let saw = Arc::new(AtomicBool::new(false)); + let saw_w = Arc::clone(&saw); + let worker = std::thread::spawn(move || { + let Ok(msg) = rx_clear.recv_timeout(std::time::Duration::from_secs(2)) else { + return; + }; + saw_w.store(true, Ordering::Relaxed); + let rbitcoin_query::ConfirmWriteMsg::ClearArchived { reply, .. } = msg else { + return; + }; + let _ = reply.send(Ok(false)); + }); + let (ev_tx, ev_rx) = std::sync::mpsc::channel(); + ev_tx + .send(ConfirmEvent::Reject { + height, + hash, + class: ConfirmRejectClass::SoftWire, + err: "consensus: bad-txnmrklroot".to_string(), + batch_len: 1, + sender: None, + parent: RejectParent::default(), + }) + .unwrap(); + let write_next = AtomicU32::new(0); + let max_ready = AtomicU32::new(0); + let mut progress = Instant::now(); + let pending = rbitcoin_ibd::ibd::events::take_confirm_events( + &mut st, + &hub, + &ev_rx, + &write_next, + &max_ready, + &mut progress, + None, + ); + assert_eq!(pending.len(), 1, "mutated clear is deferred"); + assert!( + !saw.load(Ordering::Relaxed), + "clear is not sent inside take_confirm_events" + ); + worker.join().unwrap(); + hub.query.detach_confirm_write(); + let _ = std::fs::remove_dir_all(dir); +} + /// A mutated soft reject clears the archived body without the coordinator /// lock, and only then marks that hash missing. #[test] From 84068187038388146005689340dcd99547e1d0fe Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:03:28 -0700 Subject: [PATCH 17/20] store: publish a disk-only Class C append after its bytes An fd-only extension pwrites the new slots, then the file length, then len. A zero gap is only the unnamed slots, and it does not publish the file end. An index already inside len stays a direct write. Loom branches before the length store, and the pwrite is not a loom access, so the model yields once len covers the new slots. The append under test is the next index of a pre-sized file: an empty create is in RAM, and index 0 of that file is an overwrite. --- changelog.d/class-c-snapshot.md | 3 + crates/rbitcoin-store/src/array_table.rs | 99 ++++++++++++++++---- crates/rbitcoin-store/tests/model/fd_only.rs | 56 +++++++++++ crates/rbitcoin-store/tests/model/main.rs | 2 + docs/concurrency.md | 5 +- 5 files changed, 145 insertions(+), 20 deletions(-) create mode 100644 crates/rbitcoin-store/tests/model/fd_only.rs diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index 1fa406dfe7..d3af1c623e 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -32,6 +32,9 @@ Changed - **A script-pool claim release-publishes `next` after `in_wave`.** `is_complete` acquire-loads `next`, so it does not drop the wave under a claim that still applies. +- **A disk-only Class C append publishes its length after the slot bytes.** + The file length follows those bytes. `len` follows the file length. A + reader that sees the new length does not read a zero fill. - **Loom and shuttle model the claim order, the idle turn, the UTXO view stamp, the Class C sequence counter, and Class C flush exclusion.** They are not required PR checks. The index-release order is not a model: diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 2615747c90..71c08779bc 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -5,7 +5,8 @@ //! prefix. Stores update live slots. `flush` writes those //! slots in [`FLUSH_SCRATCH`] chunks and clears the unstored mark only after the //! persist succeeds. A clean table returns before new stores are excluded. Over -//! the cap, tables stay fd write-through. In-RAM gets take no reader-writer lock. +//! the cap, tables stay fd write-through: an append pwrites the new slots, then +//! the file length, then `len`. In-RAM gets take no reader-writer lock. //! Counter order is the Class C paragraph of `docs/concurrency.md`. use crate::error::StoreError; @@ -744,29 +745,89 @@ impl ArrayTable { } let _odd = self.begin_store()?; if !self.in_ram { - let len = self.len.load(Ordering::Acquire); - if max_idx >= len { - let new_len = max_idx + 1; - let gap = new_len - len; - const CHUNK: usize = 1024 * 1024; - let mut offset = Self::offset(len); - let mut remaining = gap * ELEM; - let zeros = vec![0u8; CHUNK]; - while remaining > 0 { - let n = remaining.min(zeros.len() as u64) as usize; - self.file.write_at(offset, &zeros[..n])?; - offset += n as u64; - remaining -= n as u64; - } - self.len.store(new_len, Ordering::Release); - } - for &(index, value) in pairs { + self.write_fd(pairs, max_idx, zero_gap)?; + return Ok(()); + } + self.store_ram(min_idx, max_idx, pairs, zero_gap) + } + + /// Bytes, then the file length, then `len`. `write_at` would publish the + /// file end during a zero fill. An index already inside `len` stays a + /// direct write. + fn write_fd( + &self, + pairs: &[(u64, u64)], + max_idx: u64, + zero_gap: bool, + ) -> Result<(), StoreError> { + let len = self.len.load(Ordering::Acquire); + for &(index, value) in pairs { + if index < len { self.file .write_at(Self::offset(index), &value.to_le_bytes())?; } + } + if max_idx < len { return Ok(()); } - self.store_ram(min_idx, max_idx, pairs, zero_gap) + let new_len = max_idx + 1; + let end = Self::offset(new_len); + self.file.ensure_capacity(end)?; + if zero_gap { + self.pwrite_zero_holes(len, new_len, pairs)?; + } + for &(index, value) in pairs { + if index >= len { + self.file + .pwrite_all(Self::offset(index), &value.to_le_bytes())?; + } + } + self.file.set_logical_len(end)?; + self.len.store(new_len, Ordering::Release); + // Loom branches before the store. The pwrite is not a loom access, + // so the model yields once `len` covers the new slots. + #[cfg(loom)] + loom::thread::yield_now(); + Ok(()) + } + + /// Unnamed slots in `[from, to)` are the gap. Named slots are written after. + fn pwrite_zero_holes( + &self, + from: u64, + to: u64, + pairs: &[(u64, u64)], + ) -> Result<(), StoreError> { + let mut named: Vec = pairs + .iter() + .map(|(index, _)| *index) + .filter(|index| *index >= from && *index < to) + .collect(); + named.sort_unstable(); + named.dedup(); + let mut cursor = from; + for index in named { + if cursor < index { + self.pwrite_zeros(Self::offset(cursor), (index - cursor) * ELEM)?; + } + cursor = index + 1; + } + if cursor < to { + self.pwrite_zeros(Self::offset(cursor), (to - cursor) * ELEM)?; + } + Ok(()) + } + + fn pwrite_zeros(&self, mut offset: u64, mut remaining: u64) -> Result<(), StoreError> { + const CHUNK: usize = 1024 * 1024; + let zeros = vec![0u8; CHUNK.min(remaining as usize)]; + while remaining > 0 { + let n = remaining.min(zeros.len() as u64) as usize; + self.file.pwrite_all(offset, &zeros[..n])?; + offset += n as u64; + remaining -= n as u64; + } + Ok(()) } /// An append must name every index from `len` through `max`. Overwrites diff --git a/crates/rbitcoin-store/tests/model/fd_only.rs b/crates/rbitcoin-store/tests/model/fd_only.rs new file mode 100644 index 0000000000..5ca0dc087d --- /dev/null +++ b/crates/rbitcoin-store/tests/model/fd_only.rs @@ -0,0 +1,56 @@ +//! Disk-only Class C. An append's length covers that store's slot. A get +//! that races a truncate reads as absent. +//! +//! The file is pre-sized so `open` stays fd-only under a zero RAM cap. An +//! empty `create` is in RAM even when that cap is 0. Index 0 of a one-slot +//! file is an overwrite. The append is the next index. + +use rbitcoin_primitives::TableKind; +use rbitcoin_store::array_table::ArrayTable; +use rbitcoin_store::file::TableFile; +use rbitcoin_store::ClassCWrite; +use std::path::Path; + +fn open_fd_only(dir: &Path, slots: u64) -> ArrayTable { + let path = dir.join("slots.body"); + let file = TableFile::create(&path, TableKind::ArrayLink).unwrap(); + let zeros = vec![0u8; (slots * 8) as usize]; + if slots > 0 { + file.write_at(ArrayTable::offset(0), &zeros).unwrap(); + } + drop(file); + let prev = std::env::var_os("RBITCOIN_CLASS_C_INRAM_MAX_MB"); + std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", "0"); + let table = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); + match prev { + Some(v) => std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", v), + None => std::env::remove_var("RBITCOIN_CLASS_C_INRAM_MAX_MB"), + } + assert_eq!(table.l2_resident_bytes(), 0, "disk-only table"); + table +} + +/// One append against one sample. A length that covers the new index has +/// that store's value. +#[test] +fn fd_only_append_publishes_its_value() { + super::engine::check_at_most(2, || { + let dir = std::env::temp_dir().join("rbtc-fd-append"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let slots = std::sync::Arc::new(open_fd_only(&dir, 1)); + let slots_w = std::sync::Arc::clone(&slots); + let writer = loom::thread::spawn(move || { + slots_w.set(&ClassCWrite::for_test(), 1, 99).unwrap(); + }); + let reader = loom::thread::spawn(move || { + let (value, len) = slots.get_sample(1).unwrap(); + if len > 1 { + assert_eq!(value, 99, "new length with a zero fill"); + } + }); + writer.join().unwrap(); + reader.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + }); +} diff --git a/crates/rbitcoin-store/tests/model/main.rs b/crates/rbitcoin-store/tests/model/main.rs index 351f9f02c2..e77a0d65cf 100644 --- a/crates/rbitcoin-store/tests/model/main.rs +++ b/crates/rbitcoin-store/tests/model/main.rs @@ -4,6 +4,8 @@ #[cfg(loom)] mod engine; #[cfg(loom)] +mod fd_only; +#[cfg(loom)] mod flush; #[cfg(loom)] mod seq; diff --git a/docs/concurrency.md b/docs/concurrency.md index b3060e2b48..eda591761a 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -283,7 +283,10 @@ and the clean mark, which stay release stores. The flush does not bump the counter, so a get during the flush does not retry because of the flush. Readers stay up during IBD. Slot-table growth keeps the previous buffer until the table is dropped. `strong_tx` is one anonymous reservation of the 256 MiB cap: a grow commits the used prefix and does not keep a second image. A get retries when the length it sampled does not fit the buffer it loaded. A -bulk bit scan rechecks the counter once at the end. +bulk bit scan rechecks the counter once at the end. A table past the RAM cap +is fd write-through and has no sequence counter. An append pwrites the new +slots, then the file length, then `len`, so a get that sees the new length +has those bytes. See **[io-modality.md](./io-modality.md)** for operator IO levers. ### Confirm load read pipeline From edb87fb91a4433ecdd36b236ca0a696f36b4a1aa Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:13:16 -0700 Subject: [PATCH 18/20] store: a disk-only Class C get treats a truncate race as absent The get loads len, reads, then loads len again. A slot past that second length is empty. A past-end pread is empty only when the length no longer covers the index. Any other error is unchanged. Loom branches before the length load and the pread is not a loom access, so the model yields after len covers the index and before the read. --- changelog.d/class-c-snapshot.md | 6 +++- crates/rbitcoin-store/src/array_table.rs | 36 ++++++++++++++++---- crates/rbitcoin-store/tests/array_table.rs | 27 +++++++++++++++ crates/rbitcoin-store/tests/model/fd_only.rs | 31 +++++++++++++++++ docs/concurrency.md | 4 ++- 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/changelog.d/class-c-snapshot.md b/changelog.d/class-c-snapshot.md index d3af1c623e..36bfa4ca3d 100644 --- a/changelog.d/class-c-snapshot.md +++ b/changelog.d/class-c-snapshot.md @@ -35,8 +35,12 @@ Changed - **A disk-only Class C append publishes its length after the slot bytes.** The file length follows those bytes. `len` follows the file length. A reader that sees the new length does not read a zero fill. +- **A disk-only Class C get that races a truncate reads as absent.** + After the read, a slot past the reloaded length is empty. A past-end + pread is empty only when that length no longer covers the index. - **Loom and shuttle model the claim order, the idle turn, the UTXO view - stamp, the Class C sequence counter, and Class C flush exclusion.** + stamp, the Class C sequence counter, disk-only append and truncate, and + Class C flush exclusion.** They are not required PR checks. The index-release order is not a model: it needs a `Query` full of `std` locks. The job shape follows phlip9's #1214: cfg-gated model tests, a separate target directory, and the same diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 71c08779bc..9349bb6882 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -662,19 +662,41 @@ impl ArrayTable { #[doc(hidden)] pub fn get_sample(&self, index: u64) -> Result<(u64, u64), StoreError> { if !self.in_ram { - let len = self.len.load(Ordering::Acquire); - if index >= len { - return Ok((0, len)); - } - let mut buf = [0u8; 8]; - self.file.read_at(Self::offset(index), &mut buf)?; - return Ok((u64::from_le_bytes(buf), len)); + return self.get_sample_fd(index); } Ok(seq_read(&self.seq, || { self.sample_ram(index, Ordering::Relaxed) })) } + /// A truncate may shorten `len` and the file after the first load. The + /// slot is empty when the reloaded length no longer covers it. A past-end + /// pread whose length still covers the index stays an error. + fn get_sample_fd(&self, index: u64) -> Result<(u64, u64), StoreError> { + let len = self.len.load(Ordering::Acquire); + if index >= len { + return Ok((0, len)); + } + // Loom branches before the load. `pread` is not a loom access, so the + // model yields after `len` has covered `index` and before the read. + #[cfg(loom)] + loom::thread::yield_now(); + let mut buf = [0u8; 8]; + let value = match self.file.read_at(Self::offset(index), &mut buf) { + Ok(()) => Some(u64::from_le_bytes(buf)), + Err(StoreError::Corrupt("pread past logical end")) => None, + Err(e) => return Err(e), + }; + let len = self.len.load(Ordering::Acquire); + if index >= len { + return Ok((0, len)); + } + match value { + Some(value) => Ok((value, len)), + None => Err(StoreError::Corrupt("pread past logical end")), + } + } + /// `None` when the published buffer is shorter than `index`. Past `len` is /// `(0, len)`, not a missing slot. fn sample_ram(&self, index: u64, order: Ordering) -> Option<(u64, u64)> { diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index b79e0d93cd..d5692a9750 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -57,6 +57,33 @@ fn get_racing_truncate_does_not_panic() { let _ = std::fs::remove_file(&path); } +/// A disk-only table returns an empty slot after truncate, without a read. +#[test] +fn fd_only_truncate_then_get_is_absent() { + let dir = tmp_path(); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("slots.body"); + let file = rbitcoin_store::file::TableFile::create(&path, TableKind::ArrayLink).unwrap(); + file.write_at(ArrayTable::offset(0), &[0u8; 16]).unwrap(); + drop(file); + let prev = std::env::var_os("RBITCOIN_CLASS_C_INRAM_MAX_MB"); + std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", "0"); + let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); + match &prev { + Some(v) => std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", v), + None => std::env::remove_var("RBITCOIN_CLASS_C_INRAM_MAX_MB"), + } + assert_eq!(t.l2_resident_bytes(), 0, "disk-only table"); + t.set(&rbitcoin_store::ClassCWrite::for_test(), 1, 22) + .unwrap(); + t.truncate(&rbitcoin_store::ClassCWrite::for_test(), 1) + .unwrap(); + assert_eq!(t.get_sample(1).unwrap(), (0, 1)); + drop(t); + let _ = std::fs::remove_dir_all(&dir); +} + /// A multi-slot store publishes one generation. Each `get` is the previous /// value or the new one for that index. #[test] diff --git a/crates/rbitcoin-store/tests/model/fd_only.rs b/crates/rbitcoin-store/tests/model/fd_only.rs index 5ca0dc087d..9eca172ee3 100644 --- a/crates/rbitcoin-store/tests/model/fd_only.rs +++ b/crates/rbitcoin-store/tests/model/fd_only.rs @@ -12,6 +12,10 @@ use rbitcoin_store::ClassCWrite; use std::path::Path; fn open_fd_only(dir: &Path, slots: u64) -> ArrayTable { + // `set_var` is process-global. The model binary runs these tests in + // parallel, so one open must not restore the cap under the other. + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _lock = LOCK.lock().unwrap_or_else(|err| err.into_inner()); let path = dir.join("slots.body"); let file = TableFile::create(&path, TableKind::ArrayLink).unwrap(); let zeros = vec![0u8; (slots * 8) as usize]; @@ -54,3 +58,30 @@ fn fd_only_append_publishes_its_value() { let _ = std::fs::remove_dir_all(&dir); }); } + +/// One truncate against one sample of a primed slot. Absent, or the value +/// with a length that still covers the index. Not a past-end pread. +#[test] +fn fd_only_get_during_truncate_is_absent_or_present() { + super::engine::check_at_most(2, || { + let dir = std::env::temp_dir().join("rbtc-fd-trunc"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let slots = std::sync::Arc::new(open_fd_only(&dir, 2)); + slots.set(&ClassCWrite::for_test(), 0, 11).unwrap(); + slots.set(&ClassCWrite::for_test(), 1, 22).unwrap(); + let slots_w = std::sync::Arc::clone(&slots); + let truncator = loom::thread::spawn(move || { + slots_w.truncate(&ClassCWrite::for_test(), 0).unwrap(); + }); + let reader = loom::thread::spawn(move || match slots.get_sample(1) { + Ok((0, len)) => assert!(len <= 1, "zero slot while length covers it: {len}"), + Ok((22, len)) if len > 1 => {} + Ok((value, len)) => panic!("sample {value} at len {len}"), + Err(e) => panic!("pread raced truncate: {e}"), + }); + truncator.join().unwrap(); + reader.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + }); +} diff --git a/docs/concurrency.md b/docs/concurrency.md index eda591761a..679e6597fe 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -286,7 +286,9 @@ get retries when the length it sampled does not fit the buffer it loaded. A bulk bit scan rechecks the counter once at the end. A table past the RAM cap is fd write-through and has no sequence counter. An append pwrites the new slots, then the file length, then `len`, so a get that sees the new length -has those bytes. +has those bytes. A get that sampled a length a truncate has since shortened +reads as an empty slot. A past-end pread is that empty slot only when the +reloaded length no longer covers the index. See **[io-modality.md](./io-modality.md)** for operator IO levers. ### Confirm load read pipeline From eaae07738db4c9399a7874e8eac5c2f271eeadf1 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:54:24 -0700 Subject: [PATCH 19/20] store: a disk-only zero gap fills one span A zero gap pwrites zeros across the new span, then the named slots. Walking each hole was a second write of the same bytes, and that helper was uncovered. The append still publishes the file length after those bytes. A unit test reads the value, the holes, and the slot that was already in range. --- crates/rbitcoin-store/src/array_table.rs | 52 ++++++++-------------- crates/rbitcoin-store/tests/array_table.rs | 31 +++++++++++++ docs/concurrency.md | 3 +- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/crates/rbitcoin-store/src/array_table.rs b/crates/rbitcoin-store/src/array_table.rs index 9349bb6882..9d2442164a 100644 --- a/crates/rbitcoin-store/src/array_table.rs +++ b/crates/rbitcoin-store/src/array_table.rs @@ -5,8 +5,9 @@ //! prefix. Stores update live slots. `flush` writes those //! slots in [`FLUSH_SCRATCH`] chunks and clears the unstored mark only after the //! persist succeeds. A clean table returns before new stores are excluded. Over -//! the cap, tables stay fd write-through: an append pwrites the new slots, then -//! the file length, then `len`. In-RAM gets take no reader-writer lock. +//! the cap, tables stay fd write-through: an append pwrites the new slots +//! (a zero gap fills that span first), then the file length, then `len`. +//! In-RAM gets take no reader-writer lock. //! Counter order is the Class C paragraph of `docs/concurrency.md`. use crate::error::StoreError; @@ -773,9 +774,8 @@ impl ArrayTable { self.store_ram(min_idx, max_idx, pairs, zero_gap) } - /// Bytes, then the file length, then `len`. `write_at` would publish the - /// file end during a zero fill. An index already inside `len` stays a - /// direct write. + /// An index already inside `len` is a direct write. An extension publishes + /// its length only after the new bytes. fn write_fd( &self, pairs: &[(u64, u64)], @@ -792,11 +792,24 @@ impl ArrayTable { if max_idx < len { return Ok(()); } + self.append_fd(len, max_idx, pairs, zero_gap) + } + + /// Bytes, then the file length, then `len`. `write_at` would publish the + /// file end during a zero fill. A zero gap fills the new span first; the + /// named slots then overwrite their places in that span. + fn append_fd( + &self, + len: u64, + max_idx: u64, + pairs: &[(u64, u64)], + zero_gap: bool, + ) -> Result<(), StoreError> { let new_len = max_idx + 1; let end = Self::offset(new_len); self.file.ensure_capacity(end)?; if zero_gap { - self.pwrite_zero_holes(len, new_len, pairs)?; + self.pwrite_zeros(Self::offset(len), (new_len - len) * ELEM)?; } for &(index, value) in pairs { if index >= len { @@ -813,33 +826,6 @@ impl ArrayTable { Ok(()) } - /// Unnamed slots in `[from, to)` are the gap. Named slots are written after. - fn pwrite_zero_holes( - &self, - from: u64, - to: u64, - pairs: &[(u64, u64)], - ) -> Result<(), StoreError> { - let mut named: Vec = pairs - .iter() - .map(|(index, _)| *index) - .filter(|index| *index >= from && *index < to) - .collect(); - named.sort_unstable(); - named.dedup(); - let mut cursor = from; - for index in named { - if cursor < index { - self.pwrite_zeros(Self::offset(cursor), (index - cursor) * ELEM)?; - } - cursor = index + 1; - } - if cursor < to { - self.pwrite_zeros(Self::offset(cursor), (to - cursor) * ELEM)?; - } - Ok(()) - } - fn pwrite_zeros(&self, mut offset: u64, mut remaining: u64) -> Result<(), StoreError> { const CHUNK: usize = 1024 * 1024; let zeros = vec![0u8; CHUNK.min(remaining as usize)]; diff --git a/crates/rbitcoin-store/tests/array_table.rs b/crates/rbitcoin-store/tests/array_table.rs index d5692a9750..7bb0b11ad0 100644 --- a/crates/rbitcoin-store/tests/array_table.rs +++ b/crates/rbitcoin-store/tests/array_table.rs @@ -57,6 +57,37 @@ fn get_racing_truncate_does_not_panic() { let _ = std::fs::remove_file(&path); } +/// A disk-only append publishes the named slots, and a zero gap publishes +/// zeros in the holes, before a reader can see the new length. +#[test] +fn fd_only_append_publishes_values_and_zero_gap() { + let dir = tmp_path(); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("slots.body"); + let file = rbitcoin_store::file::TableFile::create(&path, TableKind::ArrayLink).unwrap(); + file.write_at(ArrayTable::offset(0), &[0u8; 8]).unwrap(); + drop(file); + let prev = std::env::var_os("RBITCOIN_CLASS_C_INRAM_MAX_MB"); + std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", "0"); + let t = ArrayTable::open(&path, TableKind::ArrayLink).unwrap(); + match &prev { + Some(v) => std::env::set_var("RBITCOIN_CLASS_C_INRAM_MAX_MB", v), + None => std::env::remove_var("RBITCOIN_CLASS_C_INRAM_MAX_MB"), + } + assert_eq!(t.l2_resident_bytes(), 0, "disk-only table"); + let write = rbitcoin_store::ClassCWrite::for_test(); + t.set(&write, 0, 7).unwrap(); + assert_eq!(t.get_sample(0).unwrap(), (7, 1)); + t.set_zero_gap(&write, 3, 9).unwrap(); + assert_eq!(t.get_sample(0).unwrap(), (7, 4)); + assert_eq!(t.get_sample(1).unwrap(), (0, 4)); + assert_eq!(t.get_sample(2).unwrap(), (0, 4)); + assert_eq!(t.get_sample(3).unwrap(), (9, 4)); + drop(t); + let _ = std::fs::remove_dir_all(&dir); +} + /// A disk-only table returns an empty slot after truncate, without a read. #[test] fn fd_only_truncate_then_get_is_absent() { diff --git a/docs/concurrency.md b/docs/concurrency.md index 679e6597fe..d9ddf5e386 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -286,7 +286,8 @@ get retries when the length it sampled does not fit the buffer it loaded. A bulk bit scan rechecks the counter once at the end. A table past the RAM cap is fd write-through and has no sequence counter. An append pwrites the new slots, then the file length, then `len`, so a get that sees the new length -has those bytes. A get that sampled a length a truncate has since shortened +has those bytes. A zero gap pwrites zeros across that new span before the +named slots. A get that sampled a length a truncate has since shortened reads as an empty slot. A past-end pread is that empty slot only when the reloaded length no longer covers the index. See **[io-modality.md](./io-modality.md)** for operator IO levers. From 6357fb5e82ed0cd821c59575a79580fc4dddf711 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:54:24 -0700 Subject: [PATCH 20/20] test: a tip flush counts its own thread The pread suite counts every task named rbtc-tip-flush. Another query's barrier in that process made the count two, or left one after drop. The worker records its own live count and Linux tid. The test checks that thread's name and that drop joins it. --- crates/rbitcoin-query/src/connect.rs | 65 +++++++++++++++++++++- crates/rbitcoin-query/tests/tip_flush.rs | 71 ++++++++++++------------ 2 files changed, 99 insertions(+), 37 deletions(-) diff --git a/crates/rbitcoin-query/src/connect.rs b/crates/rbitcoin-query/src/connect.rs index 88f9458fac..16a3c5adb7 100644 --- a/crates/rbitcoin-query/src/connect.rs +++ b/crates/rbitcoin-query/src/connect.rs @@ -77,6 +77,12 @@ pub struct TipFlushJob { pub struct TipFlushWorker { tx: std::sync::Mutex>>, thread: std::sync::Mutex>>, + /// Live threads of this worker. A second spawn would leave this above one. + #[cfg(debug_assertions)] + alive: std::sync::Arc, + /// Linux tid of this worker. Zero until the thread publishes it. + #[cfg(all(debug_assertions, target_os = "linux"))] + os_tid: std::sync::Arc, } impl Default for TipFlushWorker { @@ -84,10 +90,39 @@ impl Default for TipFlushWorker { Self { tx: std::sync::Mutex::new(None), thread: std::sync::Mutex::new(None), + #[cfg(debug_assertions)] + alive: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), + #[cfg(all(debug_assertions, target_os = "linux"))] + os_tid: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), } } } +/// Decrements the worker's live count when the barrier thread exits. +#[cfg(debug_assertions)] +struct TipFlushAlive(std::sync::Arc); + +#[cfg(debug_assertions)] +impl TipFlushAlive { + fn arm(alive: std::sync::Arc) -> Self { + alive.fetch_add(1, Ordering::Release); + Self(alive) + } +} + +#[cfg(debug_assertions)] +impl Drop for TipFlushAlive { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Release); + } +} + +#[cfg(all(debug_assertions, target_os = "linux"))] +fn linux_task_id() -> Option { + let path = std::fs::read_link("/proc/thread-self").ok()?; + path.file_name()?.to_str()?.parse().ok() +} + impl Query { /// Tip hash from the last confirm or disconnect, if it still matches `height`. pub fn published_tip(&self) -> Option { @@ -275,9 +310,19 @@ impl Query { let store = Arc::clone(&self.store); #[cfg(debug_assertions)] let hold = Arc::clone(&self.tip_flush_hold); + #[cfg(debug_assertions)] + let alive = Arc::clone(&self.tip_flush_worker.alive); + #[cfg(all(debug_assertions, target_os = "linux"))] + let os_tid = Arc::clone(&self.tip_flush_worker.os_tid); let handle = std::thread::Builder::new() .name("rbtc-tip-flush".into()) .spawn(move || { + #[cfg(debug_assertions)] + let _alive = TipFlushAlive::arm(alive); + #[cfg(all(debug_assertions, target_os = "linux"))] + if let Some(tid) = linux_task_id() { + os_tid.store(tid, Ordering::Release); + } while let Ok(job) = rx.recv() { // Pause after the flush excludes stores, not before it starts. #[cfg(debug_assertions)] @@ -323,7 +368,25 @@ impl Query { false } - /// IBD tip flush threads wait before writing until the returned flag is cleared. + /// Live `rbtc-tip-flush` threads of this query. Other queries do not count. + #[cfg(debug_assertions)] + #[doc(hidden)] + pub fn tip_flush_alive(&self) -> usize { + self.tip_flush_worker + .alive + .load(std::sync::atomic::Ordering::Acquire) + } + + /// Linux tid of this query's barrier thread, or zero before it publishes. + #[cfg(all(debug_assertions, target_os = "linux"))] + #[doc(hidden)] + pub fn tip_flush_os_tid(&self) -> u64 { + self.tip_flush_worker + .os_tid + .load(std::sync::atomic::Ordering::Acquire) + } + + /// IBD tip flush threads pause after they exclude stores, until this flag clears. #[cfg(debug_assertions)] #[doc(hidden)] pub fn hold_tip_flushes(&self) -> Arc { diff --git a/crates/rbitcoin-query/tests/tip_flush.rs b/crates/rbitcoin-query/tests/tip_flush.rs index 053aca4446..7b7783e0b8 100644 --- a/crates/rbitcoin-query/tests/tip_flush.rs +++ b/crates/rbitcoin-query/tests/tip_flush.rs @@ -1,6 +1,7 @@ //! IBD Class C barrier thread. //! -//! The OS name is read from `/proc`, so this file is Linux-only. +//! The OS name is read from `/proc`, so this file is Linux-only. The count is +//! this query's thread: other tests in the process also name `rbtc-tip-flush`. #![cfg(target_os = "linux")] @@ -13,50 +14,48 @@ fn two_ibd_flushes_share_one_named_thread() { let (dir, q) = rbitcoin_query::testutil::tiny_query_labeled("tip-flush-one"); assert!(q.index_mode().unwrap().is_tip()); q.flush_tip(None).unwrap(); - assert_eq!(comm_now("rbtc-tip-flush"), 0, "tip mode flushes inline"); + assert_eq!(q.tip_flush_alive(), 0, "tip mode flushes inline"); q.enter_direct_index_mode_sh(false).unwrap(); let hold = q.hold_tip_flushes(); q.spawn_tip_flush().unwrap(); q.spawn_tip_flush().unwrap(); - let n = comm_max("rbtc-tip-flush", Duration::from_millis(200)); + let deadline = Instant::now() + Duration::from_millis(200); + let mut tid = 0u64; + while Instant::now() < deadline { + let alive = q.tip_flush_alive(); + let published = q.tip_flush_os_tid(); + if tid == 0 { + assert!(alive <= 1, "two IBD barriers use one rbtc-tip-flush thread"); + if alive == 1 && published != 0 { + tid = published; + } + } else { + assert_eq!(alive, 1, "two IBD barriers use one rbtc-tip-flush thread"); + assert_eq!(published, tid, "the second barrier starts another thread"); + } + std::thread::sleep(Duration::from_millis(10)); + } + assert_ne!(tid, 0, "rbtc-tip-flush published its tid"); + assert_eq!( + task_comm(tid).as_deref(), + Some("rbtc-tip-flush"), + "IBD barrier thread name" + ); hold.store(false, Ordering::SeqCst); q.join_tip_flush().unwrap(); + assert_eq!( + q.tip_flush_alive(), + 1, + "the barrier thread outlives one flush" + ); + assert_eq!(q.tip_flush_os_tid(), tid); drop(q); drop(dir); - assert_eq!(n, 1, "two IBD barriers use one rbtc-tip-flush thread"); - assert_eq!(comm_now("rbtc-tip-flush"), 0, "drop joins rbtc-tip-flush"); -} - -fn comm_now(want: &str) -> usize { - linux_comms() - .iter() - .filter(|name| name.as_str() == want) - .count() -} - -fn comm_max(want: &str, window: Duration) -> usize { - let deadline = Instant::now() + window; - let mut n = 0; - loop { - n = n.max(comm_now(want)); - if Instant::now() >= deadline { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } - n + assert!(task_comm(tid).is_none(), "drop joins rbtc-tip-flush"); } -fn linux_comms() -> Vec { - let mut names = Vec::new(); - let Ok(tasks) = std::fs::read_dir("/proc/self/task") else { - return names; - }; - for task in tasks.flatten() { - if let Ok(raw) = std::fs::read_to_string(task.path().join("comm")) { - names.push(raw.trim_end_matches('\n').to_string()); - } - } - names +fn task_comm(tid: u64) -> Option { + let raw = std::fs::read_to_string(format!("/proc/self/task/{tid}/comm")).ok()?; + Some(raw.trim_end_matches('\n').to_string()) }