server-zd: default the log deletion delay to 1 hour in single-node modes

docs/reference/server/configuration.mdx

@@ -175,6 +175,31 @@ authenticators, it logs a warning at startup. Configure authentication before
 you expose it. Set **metrics.listen** in the overrides to enable
 [metrics](/reference/server/metrics).
 
+### Disk use {#single-node-local-disk}
+
+**dev** and **local** default **snapshot.logDeletionDelay** to 1 hour.
+**serve** defaults it to 24 hours. An explicit value in the overrides always
+wins, and it must be greater than zero. A log object is deleted once a snapshot
+that covers it is this old. The age counts from the snapshot's creation time,
+before its upload finishes. A snapshot beyond **snapshot.retainedCount** is
+deleted once the newer snapshot that displaced it is this old. A longer delay
+keeps older snapshots and their log for fallback and lets a stopped node catch
+up from the log, at the cost of disk space. The delay must also exceed the
+longest snapshot upload. Changed in 0.1.21: before, **dev** and **local** also
+defaulted to 24 hours.
+
+The embedded object store shares the disk with the local store. Plan for the
+size of **store**, plus the object store: the snapshots that
+**snapshot.retainedCount** keeps, and the log and displaced snapshots written
+during **snapshot.logDeletionDelay**. Together, the object store can reach two
+to three times the data written during the delay.
+
+A Flow that rewrites a large value on every Step needs more disk. Its history
+records wait for a newer [checkpoint](#history-checkpoints) before they can be
+deleted. For such Flows, lower **history.checkpointCommandThreshold** from its
+default of 1000. Better, store incremental values, one entry per item, instead
+of rewriting one large value.
+
 ### Back up and restore {#single-node-local-backup}
 
 To back up, stop **local** with SIGTERM, wait for it to exit, and copy the
@@ -196,6 +221,7 @@ explains.
 | Nodes | One. | One. | One active process, with optional hot standbys. |
 | Step attempt drain on stop | 2 seconds. | 30 seconds. | 30 seconds. |
 | Lease duration | 30 seconds. | 30 seconds. | 10 seconds. |
+| Default **snapshot.logDeletionDelay** | 1 hour. | 1 hour. | 24 hours. |
 | Default partition count | 32 | 32 | 1024 |
 | Default listen address | 127.0.0.1:8801 | 127.0.0.1:8801 | 0.0.0.0:8801 |
 | Configuration | Flags and an optional overrides file. | Flags and an optional overrides file. | One YAML file. |

server-zd/crates/dexzd-server/src/config/engine_config.rs

@@ -154,7 +154,12 @@ pub struct SnapshotConfig {
     pub interval: Duration,
     /// Defaults to 3; number of completed snapshot manifests retained for recovery.
     pub retained_count: u32,
-    /// Defaults to 24 h; obsolete log objects wait this long in the deletion queue after snapshot coverage.
+    /// Defaults to 24 h for `serve` and 1 h for `dev` and `local`; obsolete log objects and
+    /// superseded snapshots wait this long in the deletion queue after snapshot coverage.
+    ///
+    /// A longer delay keeps more fallback to older snapshots and lets a standby or a long-stopped
+    /// node catch up from the log, and must exceed the longest snapshot upload. A shorter delay
+    /// frees object storage sooner. Must be greater than zero.
     #[serde(with = "humantime_serde")]
     pub log_deletion_delay: Duration,
 }

server-zd/crates/dexzd-server/src/config/source_config.rs

@@ -25,6 +25,8 @@ const SINGLE_NODE_DEFAULT_LISTEN: SocketAddr =
 const SINGLE_NODE_LEASE_DURATION: Duration = Duration::from_secs(30);
 /// Cuts SearchFlows fan-out for one node; default applier lanes shrink only above 32 CPUs.
 pub(crate) const SINGLE_NODE_PARTITION_COUNT: u32 = 32;
+/// No standby catches up from the log, and superseded objects share the RocksDB disk.
+const SINGLE_NODE_LOG_DELETION_DELAY: Duration = Duration::from_secs(60 * 60);
 /// Keeps the drain and teardown inside the 10-second stop budget of dexcli dev.
 const DEV_ATTEMPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
 
@@ -211,6 +213,10 @@ impl Config {
         let has_listen_override = has_yaml_key(override_document.as_ref(), &["api", "listen"]);
         let has_partition_count_override =
             has_yaml_key(override_document.as_ref(), &["cluster", "partitionCount"]);
+        let has_log_deletion_delay_override = has_yaml_key(
+            override_document.as_ref(),
+            &["snapshot", "logDeletionDelay"],
+        );
         let has_attempt_drain_timeout_override = has_yaml_key(
             override_document.as_ref(),
             &["lifecycle", "attemptDrainTimeout"],
@@ -263,6 +269,9 @@ impl Config {
         if !has_lease_duration_override {
             configuration.lease.duration = SINGLE_NODE_LEASE_DURATION;
         }
+        if !has_log_deletion_delay_override {
+            configuration.snapshot.log_deletion_delay = SINGLE_NODE_LOG_DELETION_DELAY;
+        }
         if !has_partition_count_override {
             configuration.cluster.partition_count = SINGLE_NODE_PARTITION_COUNT;
             if let Some(applier_lanes) = configuration.commit.applier_lanes

server-zd/crates/dexzd-server/src/config/tests.rs

@@ -633,6 +633,52 @@ fn single_node_partition_count_defaults_to_32_unless_the_overrides_set_it() {
     assert_eq!(serve.cluster.partition_count, 1024);
 }
 
+#[test]
+fn single_node_log_deletion_delay_defaults_to_one_hour_unless_the_overrides_set_it() {
+    let temporary = tempfile::tempdir().expect("test directory");
+    let other_snapshot_key_path = temporary.path().join("snapshot-interval.yaml");
+    fs::write(&other_snapshot_key_path, "snapshot:\n  interval: 5m\n").expect("write test config");
+    let explicit_delay_path = temporary.path().join("log-deletion-delay.yaml");
+    fs::write(&explicit_delay_path, "snapshot:\n  logDeletionDelay: 24h\n")
+        .expect("write test config");
+    for load in [Config::load_dev, Config::load_local] {
+        let without_overrides = load(&single_node_options(temporary.path().join("data"), None))
+            .expect("configuration without YAML");
+        assert_eq!(
+            without_overrides.snapshot.log_deletion_delay,
+            Duration::from_secs(60 * 60)
+        );
+        let with_other_snapshot_key = load(&single_node_options(
+            temporary.path().join("data"),
+            Some(other_snapshot_key_path.clone()),
+        ))
+        .expect("configuration with another snapshot key");
+        assert_eq!(
+            with_other_snapshot_key.snapshot.log_deletion_delay,
+            Duration::from_secs(60 * 60)
+        );
+        assert_eq!(
+            with_other_snapshot_key.snapshot.interval,
+            Duration::from_secs(5 * 60)
+        );
+        let with_explicit_delay = load(&single_node_options(
+            temporary.path().join("data"),
+            Some(explicit_delay_path.clone()),
+        ))
+        .expect("configuration with an explicit log deletion delay");
+        assert_eq!(
+            with_explicit_delay.snapshot.log_deletion_delay,
+            Duration::from_secs(24 * 60 * 60)
+        );
+    }
+
+    let serve = Config::parse_yaml("cluster:\n  nodeId: node-a\n").expect("serve YAML");
+    assert_eq!(
+        serve.snapshot.log_deletion_delay,
+        Duration::from_secs(24 * 60 * 60)
+    );
+}
+
 #[test]
 fn single_node_applier_lanes_above_the_default_partition_count_need_an_explicit_count() {
     let temporary = tempfile::tempdir().expect("test directory");