fix(startup): resume from existing DB without a checkpoint URL

bin/ethlambda/Cargo.toml

@@ -49,5 +49,11 @@ tikv-jemallocator = { workspace = true, optional = true }
 
 libc.workspace = true
 
+[dev-dependencies]
+# `test-util` for `#[tokio::test(start_paused = true)]`: the checkpoint-sync
+# tests would otherwise wait out the real retry backoff. Dev-only, so the
+# feature never reaches the shipped binary.
+tokio = { workspace = true, features = ["test-util"] }
+
 [build-dependencies]
 vergen-git2.workspace = true

bin/ethlambda/src/cli.rs

@@ -37,11 +37,17 @@ pub(crate) struct CliOptions {
     #[arg(long)]
     pub(crate) node_id: String,
     /// Base URL(s) of checkpoint-sync peer API servers (e.g., http://peer:5052).
-    /// When set, skips genesis initialization and fetches the finalized state
-    /// and block from each peer's `/lean/v0/states/finalized` and
-    /// `/lean/v0/blocks/finalized` endpoints. For backward compatibility, a
-    /// URL ending in `/lean/v0/states/finalized` is accepted and the trailing
-    /// path is stripped.
+    /// When set, fetches the finalized state and block from each peer's
+    /// `/lean/v0/states/finalized` and `/lean/v0/blocks/finalized` endpoints.
+    /// For backward compatibility, a URL ending in
+    /// `/lean/v0/states/finalized` is accepted and the trailing path is
+    /// stripped.
+    ///
+    /// This is a fallback, not a precedence: state already in the data
+    /// directory always wins, so these URLs are only used when there is no
+    /// resumable state on disk (or it has fallen too far behind the current
+    /// slot). With neither resumable state nor URLs, the node starts from
+    /// genesis.
     ///
     /// Multiple URLs may be supplied for redundancy, either comma-separated
     /// (`--checkpoint-sync-url u1,u2`) or by repeating the flag

bin/ethlambda/src/main.rs

@@ -641,11 +641,26 @@ fn read_hex_file_bytes(path: impl AsRef<Path>) -> eyre::Result<Vec<u8>> {
 
 /// Fetch the initial state for the node.
 ///
-/// If `checkpoint_urls` is empty, creates a genesis state from the local
-/// genesis configuration. Otherwise performs checkpoint sync by downloading
-/// and verifying the finalized state AND signed block from a peer. URLs are
-/// tried in order: the first peer that succeeds wins, and failures fall over
-/// to the next URL. Startup only aborts if every URL fails.
+/// State already on disk wins: a previous run's DB is resumed from whenever it
+/// exists and belongs to this network, whether or not `checkpoint_urls` is
+/// supplied. `checkpoint_urls` is the fallback for when there is nothing
+/// resumable on disk, or when what is there has fallen too far behind the
+/// current slot to be worth catching up over P2P
+/// ([`MAX_RESUMABLE_DB_STATE_AGE`]).
+///
+/// With no resumable DB state, a non-empty `checkpoint_urls` performs checkpoint
+/// sync by downloading and verifying the finalized state AND signed block from a
+/// peer. URLs are tried in order: the first peer that succeeds wins, and
+/// failures fall over to the next URL. Startup only aborts if every URL fails.
+/// An empty `checkpoint_urls` creates a genesis state from the local genesis
+/// configuration.
+///
+/// Aborting when every URL fails is deliberate, and applies even when a stale
+/// resumable DB is in hand: an operator who configured a checkpoint URL asked
+/// for a specific anchor, so an unreachable one is a misconfiguration to
+/// surface at boot rather than paper over by silently starting a node that is
+/// hours behind. Dropping the flag is the way to say "resume whatever is on
+/// disk"; that path never aborts.
 ///
 /// Fetching the matching signed block lets the local store serve a valid
 /// anchor via the `BlocksByRoot` req-resp protocol; without it, peers
@@ -669,20 +684,10 @@ async fn fetch_initial_state(
 ) -> Result<Store, checkpoint_sync::CheckpointSyncError> {
     let validators = genesis.validators();
 
-    if checkpoint_urls.is_empty() {
-        info!("No checkpoint sync URL provided, initializing from genesis state");
-        let genesis_state = State::from_genesis(genesis.genesis_time, validators);
-        return Ok(Store::from_anchor_state(backend, genesis_state));
-    };
-
-    // Checkpoint sync path: try URLs in order, fail over to the next on error.
-    info!(
-        url_count = checkpoint_urls.len(),
-        "Starting checkpoint sync"
-    );
-    // Checkpoint sync path
-
-    // Prefer resuming from a fresh on-disk state to avoid re-downloading what we already have.
+    // Prefer resuming from on-disk state to avoid re-downloading what we already
+    // have. Tried before the checkpoint-sync and genesis paths so that a restart
+    // without `--checkpoint-sync-url` keeps the chain instead of writing a
+    // slot-0 anchor over it.
     if let Ok(Some(store)) = Store::from_db_state(backend.clone(), genesis.genesis_time) {
         let now_ms = SystemTime::UNIX_EPOCH
             .elapsed()
@@ -693,18 +698,30 @@ async fn fetch_initial_state(
         let head_slot = store.head_slot();
         let gap = current_slot.saturating_sub(head_slot);
         if gap <= MAX_RESUMABLE_DB_STATE_AGE {
-            info!(
-                head_slot,
-                current_slot, gap, "Resuming from existing DB state"
-            );
+            info!(head_slot, current_slot, gap, "Resuming from existing DB");
             return Ok(store);
         }
-        warn!(
-            head_slot,
-            current_slot, gap, "Existing DB state is stale; falling through to checkpoint sync"
-        );
+        // No checkpoint URL was configured, so just run the node against the
+        // data directory it was given: that is the setup asked for, and there
+        // is no anchor to switch to. The warning is the point of this arm,
+        // since the DB is known to be stale and range sync may not be able to
+        // close a gap this large: peers prune block signatures past
+        // `SIGNATURE_PRUNING_RANGE`, so beyond that horizon they cannot serve
+        // the history the node is missing.
+        if checkpoint_urls.is_empty() {
+            warn!(head_slot, current_slot, gap, "DB is stale; resuming anyway");
+            return Ok(store);
+        }
+        warn!(head_slot, current_slot, gap, "DB is stale; checkpoint sync");
+    }
+
+    if checkpoint_urls.is_empty() {
+        info!("No checkpoint sync URL provided, initializing from genesis state");
+        let genesis_state = State::from_genesis(genesis.genesis_time, validators);
+        return Ok(Store::from_anchor_state(backend, genesis_state));
     }
 
+    // Checkpoint sync path: try URLs in order, fail over to the next on error.
     info!(?checkpoint_urls, "Starting checkpoint sync");
 
     let (state, signed_block) = checkpoint_sync::fetch_anchor_with_retry(
@@ -740,6 +757,8 @@ async fn fetch_initial_state(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use ethlambda_storage::backend::InMemoryBackend;
+    use ethlambda_types::genesis::GenesisValidatorEntry;
 
     /// Validator-config snippet matching `lean-quickstart`'s ansible-devnet
     /// where networks share a non-default committee count.
@@ -828,4 +847,157 @@ validators:
             .unwrap_or(1);
         assert_eq!(resolved, 1);
     }
+
+    /// Slot of the anchor seeded into the test DB. Any non-zero slot works: a
+    /// genesis re-initialization always anchors at slot 0, so a non-zero head
+    /// slot is what distinguishes "resumed from disk" from "started over".
+    const SEEDED_HEAD_SLOT: u64 = 12;
+
+    /// Loopback port 1 refuses connections immediately, so the checkpoint-sync
+    /// path fails fast and deterministically without reaching the network.
+    const UNREACHABLE_CHECKPOINT_URL: &str = "http://127.0.0.1:1";
+
+    fn now_secs() -> u64 {
+        SystemTime::UNIX_EPOCH
+            .elapsed()
+            .expect("already past the unix epoch")
+            .as_secs()
+    }
+
+    /// A `genesis_time` placing the current slot exactly `gap` slots ahead of
+    /// [`SEEDED_HEAD_SLOT`], so a test picks which side of
+    /// [`MAX_RESUMABLE_DB_STATE_AGE`] the seeded DB lands on.
+    ///
+    /// `current_slot` is derived from the wall clock inside
+    /// [`fetch_initial_state`], so `genesis_time` is the only knob and no clock
+    /// injection is needed. Sub-second truncation here only ever *shortens* the
+    /// elapsed time, and a whole slot of it would have to pass between this
+    /// call and the read inside the function to shift the gap.
+    fn genesis_time_for_gap(gap: u64) -> u64 {
+        let seconds_per_slot = MILLISECONDS_PER_SLOT / 1_000;
+        now_secs() - (SEEDED_HEAD_SLOT + gap) * seconds_per_slot
+    }
+
+    /// Single-validator genesis config. The pubkeys are placeholders; none of
+    /// the paths under test verify signatures.
+    fn test_genesis(genesis_time: u64) -> GenesisConfig {
+        GenesisConfig {
+            genesis_time,
+            genesis_validators: vec![GenesisValidatorEntry {
+                attestation_pubkey: [1u8; 52],
+                proposal_pubkey: [2u8; 52],
+            }],
+        }
+    }
+
+    /// Write an anchor at [`SEEDED_HEAD_SLOT`] into `backend`, standing in for a
+    /// previous run's persisted chain state.
+    fn seed_db(backend: Arc<dyn StorageBackend>, genesis: &GenesisConfig) {
+        let mut anchor = State::from_genesis(genesis.genesis_time, genesis.validators());
+        anchor.slot = SEEDED_HEAD_SLOT;
+        anchor.latest_block_header.slot = SEEDED_HEAD_SLOT;
+        Store::from_anchor_state(backend, anchor);
+    }
+
+    #[tokio::test]
+    async fn initializes_from_genesis_when_db_is_empty() {
+        let genesis = test_genesis(now_secs());
+        let backend = Arc::new(InMemoryBackend::default());
+
+        let store = fetch_initial_state(&[], &genesis, backend).await.unwrap();
+
+        assert_eq!(store.head_slot(), 0);
+    }
+
+    #[tokio::test]
+    async fn resumes_from_fresh_db_without_checkpoint_url() {
+        let genesis = test_genesis(genesis_time_for_gap(MAX_RESUMABLE_DB_STATE_AGE / 2));
+        let backend = Arc::new(InMemoryBackend::default());
+        seed_db(backend.clone(), &genesis);
+
+        let store = fetch_initial_state(&[], &genesis, backend).await.unwrap();
+
+        assert_eq!(store.head_slot(), SEEDED_HEAD_SLOT);
+    }
+
+    /// With no checkpoint URL to fall back to, resuming a stale DB beats
+    /// clobbering it with a slot-0 genesis anchor: P2P forward-sync can close
+    /// the gap, a genesis re-init cannot.
+    #[tokio::test]
+    async fn resumes_from_stale_db_without_checkpoint_url() {
+        let genesis = test_genesis(genesis_time_for_gap(MAX_RESUMABLE_DB_STATE_AGE + 100));
+        let backend = Arc::new(InMemoryBackend::default());
+        seed_db(backend.clone(), &genesis);
+
+        let store = fetch_initial_state(&[], &genesis, backend).await.unwrap();
+
+        assert_eq!(store.head_slot(), SEEDED_HEAD_SLOT);
+    }
+
+    /// A DB inside the resume window wins over a checkpoint URL: the store
+    /// comes back even though the URL is unreachable, so nothing was dialed.
+    ///
+    /// This and [`falls_through_to_checkpoint_sync_when_db_is_stale`] are what
+    /// pin the [`MAX_RESUMABLE_DB_STATE_AGE`] comparison. The no-URL tests
+    /// cannot: both of their branches return the same store, so inverting the
+    /// threshold leaves them green. The gap is exactly the window bound here,
+    /// so an off-by-one to `<` also fails this test.
+    #[tokio::test(start_paused = true)]
+    async fn resumes_from_fresh_db_with_checkpoint_url() {
+        let genesis = test_genesis(genesis_time_for_gap(MAX_RESUMABLE_DB_STATE_AGE));
+        let backend = Arc::new(InMemoryBackend::default());
+        seed_db(backend.clone(), &genesis);
+
+        let urls = [UNREACHABLE_CHECKPOINT_URL.to_string()];
+        let store = fetch_initial_state(&urls, &genesis, backend).await.unwrap();
+
+        assert_eq!(store.head_slot(), SEEDED_HEAD_SLOT);
+    }
+
+    /// Past the resume window a checkpoint URL takes over, so an unreachable
+    /// one surfaces as a startup error rather than a silent stale resume.
+    ///
+    /// Paused time collapses the `CHECKPOINT_RETRY_BACKOFF` sleeps between
+    /// attempts; the connection refusal itself is immediate.
+    #[tokio::test(start_paused = true)]
+    async fn falls_through_to_checkpoint_sync_when_db_is_stale() {
+        let genesis = test_genesis(genesis_time_for_gap(MAX_RESUMABLE_DB_STATE_AGE + 1));
+        let backend = Arc::new(InMemoryBackend::default());
+        seed_db(backend.clone(), &genesis);
+
+        let urls = [UNREACHABLE_CHECKPOINT_URL.to_string()];
+        // `Store` is not `Debug`, so unwrap the error by pattern rather than
+        // with `expect_err`.
+        let Err(err) = fetch_initial_state(&urls, &genesis, backend).await else {
+            panic!("unreachable checkpoint URL must abort startup");
+        };
+
+        assert!(
+            matches!(err, checkpoint_sync::CheckpointSyncError::Http(_)),
+            "expected a transport error, got {err:?}"
+        );
+    }
+
+    /// A DB from another network is not resumable, so the no-URL path still
+    /// falls back to genesis.
+    ///
+    /// This pins current behavior, not a desired one. `from_db_state` treats a
+    /// `GENESIS_TIME` mismatch as an empty DB and only warns, so with no
+    /// checkpoint URL the node writes a genesis anchor over a populated
+    /// foreign-network directory: the same data loss the resume ordering
+    /// removes everywhere else. Left as-is deliberately, and this test is here
+    /// to make the change visible when someone fixes it.
+    #[tokio::test]
+    async fn initializes_from_genesis_when_db_genesis_time_differs() {
+        let seeded_genesis = test_genesis(now_secs());
+        let backend = Arc::new(InMemoryBackend::default());
+        seed_db(backend.clone(), &seeded_genesis);
+
+        let other_genesis = test_genesis(seeded_genesis.genesis_time + 1);
+        let store = fetch_initial_state(&[], &other_genesis, backend)
+            .await
+            .unwrap();
+
+        assert_eq!(store.head_slot(), 0);
+    }
 }

docs/checkpoint_sync.md

@@ -24,7 +24,7 @@ ethlambda \
 
 Where `<URL>` is the address of a checkpoint source (see [Checkpoint Sources](#checkpoint-sources) below).
 
-When `--checkpoint-sync-url` is omitted, the node initializes from genesis.
+State already on disk takes precedence over both checkpoint sync and genesis: if the data directory holds a previous run's chain state for this network, the node resumes from it. `--checkpoint-sync-url` is the fallback for when there is nothing resumable on disk, or when what is there has fallen too far behind (see [Restarts and Existing State](#restarts-and-existing-state)). With no resumable state and no URL, the node initializes from genesis.
 
 ## Checkpoint Sources
 
@@ -56,7 +56,25 @@ If any step fails (network error, decoding error, verification failure), the nod
 
 After successful initialization, the node starts normally: it connects to the P2P network and begins participating from the checkpoint slot.
 
-If the data directory (`./data`) already contains state from a previous run, checkpoint sync writes the new anchor state on top without clearing existing data. For a clean checkpoint sync, remove the data directory first.
+## Restarts and Existing State
+
+A node restarted against a populated data directory resumes from disk rather than re-initializing, so no flag is needed to preserve the chain across a redeploy. The decision is made before any download:
+
+| State in data directory | `--checkpoint-sync-url` | Result |
+| ------------------------- | ------------------------- | -------- |
+| None, or from another network (`GENESIS_TIME` differs) | omitted | Initialize from genesis |
+| None, or from another network | set | Checkpoint sync |
+| Present, head within the resume window | either | Resume from disk (no download) |
+| Present, head beyond the resume window | set | Checkpoint sync |
+| Present, head beyond the resume window | omitted | Resume from disk anyway, with a warning |
+
+The resume window is `MAX_RESUMABLE_DB_STATE_AGE` (450 slots, ~30 minutes at 4-second slots) measured as `current_slot - head_slot`. Staleness is measured against the head, not the finalized checkpoint, so a node whose head is current still resumes during a finality stall.
+
+Beyond that window the node prefers a checkpoint when one is offered, since catching up over P2P costs more than downloading a recent state. With no URL configured there is no anchor to switch to, so the node simply runs against the data directory it was given: that is the setup that was asked for. The warning is there because range sync may not be able to close a gap this large. Peers prune block signatures past `SIGNATURE_PRUNING_RANGE` (21600 slots, ~1 day), so beyond that horizon they cannot serve the history the node is missing and it needs a checkpoint URL to catch up at all. The warning logs the gap so this is visible in the boot log.
+
+When a checkpoint URL *is* set and every URL fails, the node exits rather than falling back to the stale state on disk. This is intentional: configuring the flag asks for a specific anchor, so an unreachable source is a misconfiguration worth surfacing at boot instead of quietly starting a node that is hours behind. Omitting the flag is how you ask for "resume whatever is on disk"; that path never exits.
+
+To deliberately discard existing state and start over from genesis or from a checkpoint, remove the data directory first. Checkpoint sync itself writes its anchor state on top without clearing existing data.
 
 ## Verification Checks