From 692ab8f5ac47603fd176f879162a1fc87421b2a8 Mon Sep 17 00:00:00 2001 From: Quanzheng Long Date: Fri, 6 Nov 2026 19:42:27 -0800 Subject: [PATCH 1/2] server-zd: default snapshot.logDeletionDelay to 1 hour in dev and local Single-node modes have no standby and keep the embedded object store on the same disk as RocksDB. A 24-hour delay kept superseded log batches and snapshots for a day, so disk use grew to about 2-3 times a day's writes. serve keeps 24 hours, and an explicit value still wins in every mode. --- docs/reference/server/configuration.mdx | 22 +++++++++ .../dexzd-server/src/config/engine_config.rs | 7 ++- .../dexzd-server/src/config/source_config.rs | 9 ++++ .../crates/dexzd-server/src/config/tests.rs | 46 +++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/docs/reference/server/configuration.mdx b/docs/reference/server/configuration.mdx index 1f4e54d477..e651ffdb5d 100644 --- a/docs/reference/server/configuration.mdx +++ b/docs/reference/server/configuration.mdx @@ -175,6 +175,27 @@ authenticators, it logs a warning at startup. Configure authentication before you expose it. Set **metrics.listen** in the overrides to enable [metrics](/reference/server/metrics). +### Disk use {#single-node-local-disk} + +**dev** and **local** default **snapshot.logDeletionDelay** to 1 hour. +**serve** defaults it to 24 hours. An explicit value in the overrides always +wins, and it must be greater than zero. After a snapshot covers a log object or +replaces an older snapshot, that object waits this long before it is deleted. +A longer delay keeps older snapshots and their log for fallback and lets a +stopped node catch up from the log, at the cost of disk space. The delay must +also exceed the longest snapshot upload. Changed in 0.1.21: before, **dev** +and **local** also defaulted to 24 hours. + +The embedded object store shares the disk with the local store. Plan for about +two to three times the data written during **snapshot.logDeletionDelay**, plus +the size of **store**. + +A Flow that rewrites a large value on every Step needs more disk. Its history +records wait for a newer [checkpoint](#history-checkpoints) before they can be +deleted. For such Flows, lower **history.checkpointCommandThreshold** from its +default of 1000. Better, store incremental values, one entry per item, instead +of rewriting one large value. + ### Back up and restore {#single-node-local-backup} To back up, stop **local** with SIGTERM, wait for it to exit, and copy the @@ -196,6 +217,7 @@ explains. | Nodes | One. | One. | One active process, with optional hot standbys. | | Step attempt drain on stop | 2 seconds. | 30 seconds. | 30 seconds. | | Lease duration | 30 seconds. | 30 seconds. | 10 seconds. | +| Default **snapshot.logDeletionDelay** | 1 hour. | 1 hour. | 24 hours. | | Default partition count | 32 | 32 | 1024 | | Default listen address | 127.0.0.1:8801 | 127.0.0.1:8801 | 0.0.0.0:8801 | | Configuration | Flags and an optional overrides file. | Flags and an optional overrides file. | One YAML file. | diff --git a/server-zd/crates/dexzd-server/src/config/engine_config.rs b/server-zd/crates/dexzd-server/src/config/engine_config.rs index 9f39f4807d..4cf2081172 100644 --- a/server-zd/crates/dexzd-server/src/config/engine_config.rs +++ b/server-zd/crates/dexzd-server/src/config/engine_config.rs @@ -154,7 +154,12 @@ pub struct SnapshotConfig { pub interval: Duration, /// Defaults to 3; number of completed snapshot manifests retained for recovery. pub retained_count: u32, - /// Defaults to 24 h; obsolete log objects wait this long in the deletion queue after snapshot coverage. + /// Defaults to 24 h for `serve` and 1 h for `dev` and `local`; obsolete log objects and + /// superseded snapshots wait this long in the deletion queue after snapshot coverage. + /// + /// A longer delay keeps more fallback to older snapshots and lets a standby or a long-stopped + /// node catch up from the log, and must exceed the longest snapshot upload. A shorter delay + /// frees object storage sooner. Must be greater than zero. #[serde(with = "humantime_serde")] pub log_deletion_delay: Duration, } diff --git a/server-zd/crates/dexzd-server/src/config/source_config.rs b/server-zd/crates/dexzd-server/src/config/source_config.rs index 8adb1d1c90..daf1c37aa8 100644 --- a/server-zd/crates/dexzd-server/src/config/source_config.rs +++ b/server-zd/crates/dexzd-server/src/config/source_config.rs @@ -25,6 +25,8 @@ const SINGLE_NODE_DEFAULT_LISTEN: SocketAddr = const SINGLE_NODE_LEASE_DURATION: Duration = Duration::from_secs(30); /// Cuts SearchFlows fan-out for one node; default applier lanes shrink only above 32 CPUs. pub(crate) const SINGLE_NODE_PARTITION_COUNT: u32 = 32; +/// No standby catches up from the log, and superseded objects share the RocksDB disk. +const SINGLE_NODE_LOG_DELETION_DELAY: Duration = Duration::from_secs(60 * 60); /// Keeps the drain and teardown inside the 10-second stop budget of dexcli dev. const DEV_ATTEMPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); @@ -211,6 +213,10 @@ impl Config { let has_listen_override = has_yaml_key(override_document.as_ref(), &["api", "listen"]); let has_partition_count_override = has_yaml_key(override_document.as_ref(), &["cluster", "partitionCount"]); + let has_log_deletion_delay_override = has_yaml_key( + override_document.as_ref(), + &["snapshot", "logDeletionDelay"], + ); let has_attempt_drain_timeout_override = has_yaml_key( override_document.as_ref(), &["lifecycle", "attemptDrainTimeout"], @@ -263,6 +269,9 @@ impl Config { if !has_lease_duration_override { configuration.lease.duration = SINGLE_NODE_LEASE_DURATION; } + if !has_log_deletion_delay_override { + configuration.snapshot.log_deletion_delay = SINGLE_NODE_LOG_DELETION_DELAY; + } if !has_partition_count_override { configuration.cluster.partition_count = SINGLE_NODE_PARTITION_COUNT; if let Some(applier_lanes) = configuration.commit.applier_lanes diff --git a/server-zd/crates/dexzd-server/src/config/tests.rs b/server-zd/crates/dexzd-server/src/config/tests.rs index 9c6921556e..bb92d40855 100644 --- a/server-zd/crates/dexzd-server/src/config/tests.rs +++ b/server-zd/crates/dexzd-server/src/config/tests.rs @@ -633,6 +633,52 @@ fn single_node_partition_count_defaults_to_32_unless_the_overrides_set_it() { assert_eq!(serve.cluster.partition_count, 1024); } +#[test] +fn single_node_log_deletion_delay_defaults_to_one_hour_unless_the_overrides_set_it() { + let temporary = tempfile::tempdir().expect("test directory"); + let other_snapshot_key_path = temporary.path().join("snapshot-interval.yaml"); + fs::write(&other_snapshot_key_path, "snapshot:\n interval: 5m\n").expect("write test config"); + let explicit_delay_path = temporary.path().join("log-deletion-delay.yaml"); + fs::write(&explicit_delay_path, "snapshot:\n logDeletionDelay: 24h\n") + .expect("write test config"); + for load in [Config::load_dev, Config::load_local] { + let without_overrides = load(&single_node_options(temporary.path().join("data"), None)) + .expect("configuration without YAML"); + assert_eq!( + without_overrides.snapshot.log_deletion_delay, + Duration::from_secs(60 * 60) + ); + let with_other_snapshot_key = load(&single_node_options( + temporary.path().join("data"), + Some(other_snapshot_key_path.clone()), + )) + .expect("configuration with another snapshot key"); + assert_eq!( + with_other_snapshot_key.snapshot.log_deletion_delay, + Duration::from_secs(60 * 60) + ); + assert_eq!( + with_other_snapshot_key.snapshot.interval, + Duration::from_secs(5 * 60) + ); + let with_explicit_delay = load(&single_node_options( + temporary.path().join("data"), + Some(explicit_delay_path.clone()), + )) + .expect("configuration with an explicit log deletion delay"); + assert_eq!( + with_explicit_delay.snapshot.log_deletion_delay, + Duration::from_secs(24 * 60 * 60) + ); + } + + let serve = Config::parse_yaml("cluster:\n nodeId: node-a\n").expect("serve YAML"); + assert_eq!( + serve.snapshot.log_deletion_delay, + Duration::from_secs(24 * 60 * 60) + ); +} + #[test] fn single_node_applier_lanes_above_the_default_partition_count_need_an_explicit_count() { let temporary = tempfile::tempdir().expect("test directory"); From d4e7bc0d927a96f04908c0acdb1475ea82dcc5d7 Mon Sep 17 00:00:00 2001 From: Quanzheng Long Date: Fri, 6 Nov 2026 19:48:28 -0800 Subject: [PATCH 2/2] docs: state when log and snapshot deletion starts and what single-node disk use includes Log objects wait for the covering snapshot to reach the delay, counted from its creation time, and displaced snapshots wait for the newer one. The disk estimate now names the retained snapshots in the embedded object store. --- docs/reference/server/configuration.mdx | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/reference/server/configuration.mdx b/docs/reference/server/configuration.mdx index e651ffdb5d..ed691ccd9e 100644 --- a/docs/reference/server/configuration.mdx +++ b/docs/reference/server/configuration.mdx @@ -179,16 +179,20 @@ you expose it. Set **metrics.listen** in the overrides to enable **dev** and **local** default **snapshot.logDeletionDelay** to 1 hour. **serve** defaults it to 24 hours. An explicit value in the overrides always -wins, and it must be greater than zero. After a snapshot covers a log object or -replaces an older snapshot, that object waits this long before it is deleted. -A longer delay keeps older snapshots and their log for fallback and lets a -stopped node catch up from the log, at the cost of disk space. The delay must -also exceed the longest snapshot upload. Changed in 0.1.21: before, **dev** -and **local** also defaulted to 24 hours. - -The embedded object store shares the disk with the local store. Plan for about -two to three times the data written during **snapshot.logDeletionDelay**, plus -the size of **store**. +wins, and it must be greater than zero. A log object is deleted once a snapshot +that covers it is this old. The age counts from the snapshot's creation time, +before its upload finishes. A snapshot beyond **snapshot.retainedCount** is +deleted once the newer snapshot that displaced it is this old. A longer delay +keeps older snapshots and their log for fallback and lets a stopped node catch +up from the log, at the cost of disk space. The delay must also exceed the +longest snapshot upload. Changed in 0.1.21: before, **dev** and **local** also +defaulted to 24 hours. + +The embedded object store shares the disk with the local store. Plan for the +size of **store**, plus the object store: the snapshots that +**snapshot.retainedCount** keeps, and the log and displaced snapshots written +during **snapshot.logDeletionDelay**. Together, the object store can reach two +to three times the data written during the delay. A Flow that rewrites a large value on every Step needs more disk. Its history records wait for a newer [checkpoint](#history-checkpoints) before they can be