feat(persistence): automatic AOF rewrite + un-gated multi-shard BGREWRITEAOF (#433)

CHANGELOG.md

@@ -6,7 +6,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- **The AOF now compacts itself (#433): Redis-parity automatic rewrite.**
+  `--auto-aof-rewrite-percentage` (default 100, `0` disables) and
+  `--auto-aof-rewrite-min-size` (default `64mb`, size strings accepted)
+  trigger a background rewrite once the AOF has grown the given percentage
+  over its size after the last rewrite. Before this, the AOF grew with write
+  volume rather than dataset size — observed 4.8 GB on disk for a 2.43 GB
+  dataset, ~1 GB/day — until the diskfull guard paused writes. A monitor
+  thread samples the on-disk size once a second and dispatches the same
+  entry point as `BGREWRITEAOF`; a failed dispatch backs off 60 s instead of
+  hot-retrying. Both knobs appear in `CONFIG GET`.
+- **Multi-shard `BGREWRITEAOF` is un-gated.** The per-shard fan-out rewrite
+  (cooperative snapshot + synchronized manifest commit) is now the default —
+  the historical gate dated from a pre-C4 design that lost ~38% of keys, and
+  the current path holds exact INCR recovery across a rewrite straddling a
+  live write stream plus SIGKILL (crash matrix, 5/5 repeat runs).
+  `--experimental-per-shard-rewrite` is deprecated (warns, no-op).
+
 ### Fixed
+- **`INFO persistence` reports real AOF state (#432).** `aof_enabled` and
+  `aof_rewrite_in_progress` were hardcoded `0` even with `--appendonly yes`
+  (the default) and a rewrite running; they now reflect reality, and new
+  `aof_base_size` / `aof_current_size` fields expose the growth the
+  auto-rewrite trigger acts on — an operator can finally see the
+  AOF-vs-dataset ratio the diskfull incident hid.
+- **The per-shard BGREWRITEAOF crash matrix no longer reports phantom data
+  loss on nearly-full hosts.** The harness lacked `--disk-free-min-pct 0`
+  and parsed `MOONERR diskfull` INCR rejections as silently-dropped writes
+  (the host root volume hovers at ~4% free, making it intermittent). The
+  suite now disables the guard, panics on any non-numeric INCR reply, and is
+  green 5/5 consecutive runs.
 - **Replicas now apply streamed `SWAPDB` (#386), and the record reaches the
   wire exactly once per client call.** Two stacked defects: (1) the replica's
   apply path had no SWAPDB intercept — generic dispatch hard-errors ("must be

docs/runbooks/multi-shard-aof-rewrite.md

@@ -52,15 +52,25 @@ linearly with shard count.
 
 ### BGREWRITEAOF in per-shard mode
 
-`BGREWRITEAOF` is **not yet supported** for PerShard layouts. Issuing it on a
-PerShard instance returns the following error immediately:
-
-```
-ERR BGREWRITEAOF is not yet supported under per-shard AOF layout; per-shard rewrite ships in step 6 of the per-shard AOF migration
-```
-
-Per-shard BGREWRITEAOF (each shard compacts its own log independently, with
-all N acks awaited before returning confirmation) is tracked for v0.2.
+`BGREWRITEAOF` is **fully supported** on PerShard layouts (#433, un-gated
+2026-08): the per-shard fan-out folds every shard cooperatively (C4 snapshot
+via the shard event loop), advances each shard's manifest entry, and the last
+writer performs a single synchronized seq commit + old-generation prune.
+Exactness (no dropped, no double-applied writes across a rewrite that
+straddles a live write stream + SIGKILL) is pinned by
+`tests/crash_matrix_per_shard_bgrewriteaof.rs`.
+`--experimental-per-shard-rewrite`, the old opt-in, is deprecated and a
+no-op.
+
+### Automatic rewrite (#433)
+
+The AOF compacts itself, Redis-style: when its on-disk size exceeds
+`auto-aof-rewrite-min-size` (default `64mb`) AND has grown
+`auto-aof-rewrite-percentage`% (default `100`) over its size after the last
+rewrite, a background rewrite is dispatched automatically. Set the
+percentage to `0` to disable automatic rewrites (manual `BGREWRITEAOF`
+always works). Watch `INFO persistence` → `aof_base_size` /
+`aof_current_size` to see the trigger's inputs.
 
 ---
 

src/command/config.rs

@@ -41,6 +41,17 @@ pub fn config_get(
         ),
         (b"appendonly", runtime_config.appendonly.clone()),
         (b"appendfsync", runtime_config.appendfsync.clone()),
+        (
+            b"auto-aof-rewrite-percentage",
+            server_config.auto_aof_rewrite_percentage.to_string(),
+        ),
+        (
+            // Redis reports this in bytes; normalize the "64mb"-style input.
+            b"auto-aof-rewrite-min-size",
+            crate::config::ServerConfig::parse_size(&server_config.auto_aof_rewrite_min_size)
+                .unwrap_or(64 * 1024 * 1024)
+                .to_string(),
+        ),
         (b"databases", server_config.databases.to_string()),
         (b"bind", server_config.bind.clone()),
         (b"port", server_config.port.to_string()),

src/command/connection.rs

@@ -278,13 +278,26 @@ pub fn info(db: &Database, _args: &[Frame]) -> Frame {
     sections.push_str("\r\n");
 
     sections.push_str("# Persistence\r\n");
+    // #432: aof_enabled / aof_rewrite_in_progress / sizes are real state, not
+    // hardcoded zeros. Sizes come from the auto-rewrite monitor's statics
+    // (#433); refresh_current_size keeps `aof_current_size` honest when INFO
+    // is read between monitor ticks (one directory walk — INFO is cold path).
+    let aof_enabled = crate::persistence::aof::auto_rewrite::AOF_ENABLED
+        .load(std::sync::atomic::Ordering::Relaxed);
+    let aof_current_size = if aof_enabled {
+        crate::persistence::aof::auto_rewrite::refresh_current_size()
+    } else {
+        0
+    };
     sections.push_str(&format!(
         "loading:0\r\n\
          rdb_bgsave_in_progress:{}\r\n\
          rdb_last_save_time:{}\r\n\
          rdb_last_bgsave_status:{}\r\n\
-         aof_enabled:0\r\n\
-         aof_rewrite_in_progress:0\r\n\
+         aof_enabled:{}\r\n\
+         aof_rewrite_in_progress:{}\r\n\
+         aof_base_size:{}\r\n\
+         aof_current_size:{}\r\n\
          aof_backpressure_dropped:{}\r\n\
          spill_batches_flushed:{}\r\n\
          spill_completions_dropped:{}\r\n\
@@ -303,6 +316,14 @@ pub fn info(db: &Database, _args: &[Frame]) -> Frame {
         } else {
             "err"
         },
+        u8::from(aof_enabled),
+        u8::from(
+            crate::command::persistence::AOF_REWRITE_IN_PROGRESS
+                .load(std::sync::atomic::Ordering::SeqCst)
+        ),
+        crate::persistence::aof::auto_rewrite::AOF_BASE_SIZE
+            .load(std::sync::atomic::Ordering::Relaxed),
+        aof_current_size,
         crate::persistence::aof::AOF_BACKPRESSURE_DROPPED
             .load(std::sync::atomic::Ordering::Relaxed),
         crate::storage::tiered::spill_thread::spill_batches_flushed_total(),

src/config.rs

@@ -117,26 +117,32 @@ pub struct ServerConfig {
     #[arg(long, default_value_t = false)]
     pub unsafe_multishard_aof: bool,
 
-    /// [EXPERIMENTAL] Enable per-shard BGREWRITEAOF (compaction) for the
-    /// `--shards >= 2 + --appendonly yes` PerShard layout.
-    ///
-    /// Default `false`: BGREWRITEAOF stays gated in PerShard mode (the
-    /// shipped, crash-safe "append-only, no in-place compaction" behavior).
-    /// When `true`, BGREWRITEAOF fans the rewrite out to every per-shard
-    /// writer (synchronized seq bump + single manifest commit). This path is
-    /// validated by `tests/crash_matrix_per_shard_bgrewriteaof.rs` and is
-    /// opt-in until the both-runtime crash matrix is green by default.
-    ///
-    /// The flag only takes effect alongside `per_shard_aof_active`; it is a
-    /// no-op for `--shards 1` (TopLevel rewrite already works) and for
-    /// `--appendonly no`.
+    /// [DEPRECATED — no-op] Per-shard BGREWRITEAOF is the DEFAULT since #433
+    /// (the fan-out compaction path is validated by
+    /// `tests/crash_matrix_per_shard_bgrewriteaof.rs` and the auto-rewrite
+    /// suite). Passing this flag only emits a deprecation warning. Remove it
+    /// from launch commands; it will be deleted in a future release.
     #[arg(long, default_value_t = false)]
     pub experimental_per_shard_rewrite: bool,
 
     /// AOF fsync policy (always/everysec/no)
     #[arg(long, default_value = "everysec")]
     pub appendfsync: String,
 
+    /// Automatic AOF rewrite trigger: rewrite when the AOF has grown by this
+    /// percentage over its size after the last rewrite (Redis parity:
+    /// `auto-aof-rewrite-percentage`). `0` disables automatic rewrites;
+    /// manual `BGREWRITEAOF` still works. Default 100 (= rewrite at 2× the
+    /// post-rewrite size), same as Redis.
+    #[arg(long = "auto-aof-rewrite-percentage", default_value_t = 100)]
+    pub auto_aof_rewrite_percentage: u64,
+
+    /// Automatic AOF rewrite floor: never auto-rewrite while the total AOF
+    /// size is below this (Redis parity: `auto-aof-rewrite-min-size`).
+    /// Accepts size strings ("64mb", "1gb") or raw bytes. Default "64mb".
+    #[arg(long = "auto-aof-rewrite-min-size", default_value = "64mb")]
+    pub auto_aof_rewrite_min_size: String,
+
     /// Max time (ms) a write may block awaiting the `appendfsync=always`
     /// fsync ack before the write is failed instead of parking the
     /// connection forever. Design-for-failure bound: a stalled disk must

src/main.rs

@@ -968,40 +968,18 @@ fn main() -> anyhow::Result<()> {
     // Compute bind address for SO_REUSEPORT per-shard listeners (Linux io_uring path).
     let bind_addr = format!("{}:{}", config.bind, config.port);
 
-    // FIX-W1-4: gate BGREWRITEAOF whenever per-shard AOF is active
-    // (num_shards >= 2 + appendonly=yes). The original gate was too narrow:
-    // it required disk_offload to be enabled, missing the plain AOF case.
-    // Per-shard rewrite is not yet implemented (AofPoolSendError::
-    // RewriteUnsupportedInPerShard); the pool already refuses the message,
-    // but this early gate provides a stable, documented error to operators
-    // BEFORE the channel send so no in-progress flag is flipped.
-    // Verified 2026-05-26: multi-shard BGREWRITEAOF loses ~38% of keys on
-    // restart. Gate lifted only when multi-part AOF replay ships (v2.0+).
-    // See docs/runbooks/multi-shard-aof-rewrite.md.
-    // [F6] When `--experimental-per-shard-rewrite` is set, leave the gate OPEN
-    // so BGREWRITEAOF routes to the per-shard fan-out coordinator
-    // (try_send_rewrite_per_shard): synchronized seq bump + single manifest
-    // commit across all per-shard writers, validated by
-    // tests/crash_matrix_per_shard_bgrewriteaof.rs. Default (flag off) keeps
-    // the gate closed — the shipped, crash-safe "no in-place compaction"
-    // behavior that avoided the historical ~38%-key-loss-on-restart.
-    if config.per_shard_aof_active(num_shards) {
-        if config.experimental_per_shard_rewrite {
-            tracing::warn!(
-                shards = num_shards,
-                appendonly = %config.appendonly,
-                "BGREWRITEAOF per-shard rewrite ENABLED (--experimental-per-shard-rewrite). \
-                 Per-shard fan-out compaction is active; this path is experimental."
-            );
-        } else {
-            moon::command::persistence::MULTI_SHARD_AOF_REWRITE_UNSAFE
-                .store(true, std::sync::atomic::Ordering::Relaxed);
-            tracing::warn!(
-                shards = num_shards,
-                appendonly = %config.appendonly,
-                "BGREWRITEAOF gated: per-shard AOF layout active (see docs/runbooks/multi-shard-aof-rewrite.md). Use --shards 1, or --experimental-per-shard-rewrite to enable per-shard compaction."
-            );
-        }
+    // #433: per-shard BGREWRITEAOF is the DEFAULT — the gate that refused it
+    // (historical ~38%-key-loss era, pre-C4 cooperative snapshot) is retired.
+    // The fan-out path is validated by tests/crash_matrix_per_shard_bgrewriteaof.rs
+    // (exact INCR recovery across a straddling rewrite + SIGKILL) and
+    // tests/aof_auto_rewrite.rs. `MULTI_SHARD_AOF_REWRITE_UNSAFE` is never set
+    // at boot anymore; the refusal branch it guards stays as dead-man code for
+    // tests and any future re-gate.
+    if config.experimental_per_shard_rewrite {
+        tracing::warn!(
+            "--experimental-per-shard-rewrite is deprecated and now a no-op: \
+             per-shard BGREWRITEAOF is the default (#433)."
+        );
     }
 
     // Create watch channel for snapshot triggers (auto-save and BGSAVE)
@@ -1764,6 +1742,31 @@ fn main() -> anyhow::Result<()> {
         moon::shard::shared_databases::replay_mq_wal(&mut slice_inits, dir_path);
     }
 
+    // #433: AOF auto-rewrite monitor + INFO size statics. Init AFTER recovery
+    // (the just-replayed generation is the growth baseline), spawn regardless
+    // of percentage so `INFO persistence` sizes stay fresh; percentage 0
+    // disables only the trigger.
+    if let Some(ref pool) = aof_pool {
+        moon::persistence::aof::auto_rewrite::init(
+            std::path::Path::new(&config.dir),
+            &config.appendfilename,
+        );
+        let min_size =
+            ServerConfig::parse_size(&config.auto_aof_rewrite_min_size).unwrap_or_else(|| {
+                tracing::warn!(
+                    "unparseable --auto-aof-rewrite-min-size {:?}; using 64mb",
+                    config.auto_aof_rewrite_min_size
+                );
+                64 * 1024 * 1024
+            });
+        moon::persistence::aof::auto_rewrite::spawn_monitor(
+            pool.clone(),
+            shard_databases.clone(),
+            config.auto_aof_rewrite_percentage,
+            min_size,
+        );
+    }
+
     // All shards recovered — mark server as ready for /readyz.
     moon::admin::metrics_setup::set_server_ready();
     // Register global ShardDatabases for MEMORY DOCTOR + Prometheus per-kind gauges.