diff --git a/docs/reference/server/configuration.mdx b/docs/reference/server/configuration.mdx index 1f4e54d477..ed691ccd9e 100644 --- a/docs/reference/server/configuration.mdx +++ b/docs/reference/server/configuration.mdx @@ -175,6 +175,31 @@ authenticators, it logs a warning at startup. Configure authentication before you expose it. Set **metrics.listen** in the overrides to enable [metrics](/reference/server/metrics). +### Disk use {#single-node-local-disk} + +**dev** and **local** default **snapshot.logDeletionDelay** to 1 hour. +**serve** defaults it to 24 hours. An explicit value in the overrides always +wins, and it must be greater than zero. A log object is deleted once a snapshot +that covers it is this old. The age counts from the snapshot's creation time, +before its upload finishes. A snapshot beyond **snapshot.retainedCount** is +deleted once the newer snapshot that displaced it is this old. A longer delay +keeps older snapshots and their log for fallback and lets a stopped node catch +up from the log, at the cost of disk space. The delay must also exceed the +longest snapshot upload. Changed in 0.1.21: before, **dev** and **local** also +defaulted to 24 hours. + +The embedded object store shares the disk with the local store. Plan for the +size of **store**, plus the object store: the snapshots that +**snapshot.retainedCount** keeps, and the log and displaced snapshots written +during **snapshot.logDeletionDelay**. Together, the object store can reach two +to three times the data written during the delay. + +A Flow that rewrites a large value on every Step needs more disk. Its history +records wait for a newer [checkpoint](#history-checkpoints) before they can be +deleted. For such Flows, lower **history.checkpointCommandThreshold** from its +default of 1000. Better, store incremental values, one entry per item, instead +of rewriting one large value. + ### Back up and restore {#single-node-local-backup} To back up, stop **local** with SIGTERM, wait for it to exit, and copy the @@ -196,6 +221,7 @@ explains. | Nodes | One. | One. | One active process, with optional hot standbys. | | Step attempt drain on stop | 2 seconds. | 30 seconds. | 30 seconds. | | Lease duration | 30 seconds. | 30 seconds. | 10 seconds. | +| Default **snapshot.logDeletionDelay** | 1 hour. | 1 hour. | 24 hours. | | Default partition count | 32 | 32 | 1024 | | Default listen address | 127.0.0.1:8801 | 127.0.0.1:8801 | 0.0.0.0:8801 | | Configuration | Flags and an optional overrides file. | Flags and an optional overrides file. | One YAML file. | diff --git a/server-zd/crates/dexzd-server/src/config/engine_config.rs b/server-zd/crates/dexzd-server/src/config/engine_config.rs index 9f39f4807d..4cf2081172 100644 --- a/server-zd/crates/dexzd-server/src/config/engine_config.rs +++ b/server-zd/crates/dexzd-server/src/config/engine_config.rs @@ -154,7 +154,12 @@ pub struct SnapshotConfig { pub interval: Duration, /// Defaults to 3; number of completed snapshot manifests retained for recovery. pub retained_count: u32, - /// Defaults to 24 h; obsolete log objects wait this long in the deletion queue after snapshot coverage. + /// Defaults to 24 h for `serve` and 1 h for `dev` and `local`; obsolete log objects and + /// superseded snapshots wait this long in the deletion queue after snapshot coverage. + /// + /// A longer delay keeps more fallback to older snapshots and lets a standby or a long-stopped + /// node catch up from the log, and must exceed the longest snapshot upload. A shorter delay + /// frees object storage sooner. Must be greater than zero. #[serde(with = "humantime_serde")] pub log_deletion_delay: Duration, } diff --git a/server-zd/crates/dexzd-server/src/config/source_config.rs b/server-zd/crates/dexzd-server/src/config/source_config.rs index 8adb1d1c90..daf1c37aa8 100644 --- a/server-zd/crates/dexzd-server/src/config/source_config.rs +++ b/server-zd/crates/dexzd-server/src/config/source_config.rs @@ -25,6 +25,8 @@ const SINGLE_NODE_DEFAULT_LISTEN: SocketAddr = const SINGLE_NODE_LEASE_DURATION: Duration = Duration::from_secs(30); /// Cuts SearchFlows fan-out for one node; default applier lanes shrink only above 32 CPUs. pub(crate) const SINGLE_NODE_PARTITION_COUNT: u32 = 32; +/// No standby catches up from the log, and superseded objects share the RocksDB disk. +const SINGLE_NODE_LOG_DELETION_DELAY: Duration = Duration::from_secs(60 * 60); /// Keeps the drain and teardown inside the 10-second stop budget of dexcli dev. const DEV_ATTEMPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); @@ -211,6 +213,10 @@ impl Config { let has_listen_override = has_yaml_key(override_document.as_ref(), &["api", "listen"]); let has_partition_count_override = has_yaml_key(override_document.as_ref(), &["cluster", "partitionCount"]); + let has_log_deletion_delay_override = has_yaml_key( + override_document.as_ref(), + &["snapshot", "logDeletionDelay"], + ); let has_attempt_drain_timeout_override = has_yaml_key( override_document.as_ref(), &["lifecycle", "attemptDrainTimeout"], @@ -263,6 +269,9 @@ impl Config { if !has_lease_duration_override { configuration.lease.duration = SINGLE_NODE_LEASE_DURATION; } + if !has_log_deletion_delay_override { + configuration.snapshot.log_deletion_delay = SINGLE_NODE_LOG_DELETION_DELAY; + } if !has_partition_count_override { configuration.cluster.partition_count = SINGLE_NODE_PARTITION_COUNT; if let Some(applier_lanes) = configuration.commit.applier_lanes diff --git a/server-zd/crates/dexzd-server/src/config/tests.rs b/server-zd/crates/dexzd-server/src/config/tests.rs index 9c6921556e..bb92d40855 100644 --- a/server-zd/crates/dexzd-server/src/config/tests.rs +++ b/server-zd/crates/dexzd-server/src/config/tests.rs @@ -633,6 +633,52 @@ fn single_node_partition_count_defaults_to_32_unless_the_overrides_set_it() { assert_eq!(serve.cluster.partition_count, 1024); } +#[test] +fn single_node_log_deletion_delay_defaults_to_one_hour_unless_the_overrides_set_it() { + let temporary = tempfile::tempdir().expect("test directory"); + let other_snapshot_key_path = temporary.path().join("snapshot-interval.yaml"); + fs::write(&other_snapshot_key_path, "snapshot:\n interval: 5m\n").expect("write test config"); + let explicit_delay_path = temporary.path().join("log-deletion-delay.yaml"); + fs::write(&explicit_delay_path, "snapshot:\n logDeletionDelay: 24h\n") + .expect("write test config"); + for load in [Config::load_dev, Config::load_local] { + let without_overrides = load(&single_node_options(temporary.path().join("data"), None)) + .expect("configuration without YAML"); + assert_eq!( + without_overrides.snapshot.log_deletion_delay, + Duration::from_secs(60 * 60) + ); + let with_other_snapshot_key = load(&single_node_options( + temporary.path().join("data"), + Some(other_snapshot_key_path.clone()), + )) + .expect("configuration with another snapshot key"); + assert_eq!( + with_other_snapshot_key.snapshot.log_deletion_delay, + Duration::from_secs(60 * 60) + ); + assert_eq!( + with_other_snapshot_key.snapshot.interval, + Duration::from_secs(5 * 60) + ); + let with_explicit_delay = load(&single_node_options( + temporary.path().join("data"), + Some(explicit_delay_path.clone()), + )) + .expect("configuration with an explicit log deletion delay"); + assert_eq!( + with_explicit_delay.snapshot.log_deletion_delay, + Duration::from_secs(24 * 60 * 60) + ); + } + + let serve = Config::parse_yaml("cluster:\n nodeId: node-a\n").expect("serve YAML"); + assert_eq!( + serve.snapshot.log_deletion_delay, + Duration::from_secs(24 * 60 * 60) + ); +} + #[test] fn single_node_applier_lanes_above_the_default_partition_count_need_an_explicit_count() { let temporary = tempfile::tempdir().expect("test directory");