ct stress

.gitignore

@@ -294,3 +294,4 @@ src/go/rpk/cmd/rpk/rpk
 
 # antithesis
 /.antithesis
+tests/antithesis/ct_stress/workload/workload

src/v/cluster/rm_stm.cc

@@ -2124,17 +2124,17 @@ ss::future<raft::stm_snapshot> rm_stm::do_take_local_snapshot(
     }
 
     stm_snapshot.abort_indexes = final_abort_indexes.copy();
-    kafka::offset start_kafka_offset = from_log_offset(start_offset);
+    // kafka::offset start_kafka_offset = from_log_offset(start_offset);
     stm_snapshot.highest_producer_id = _highest_producer_id;
     // producers state (includes idempotent and transactional producers)
     for (const auto& [_, state] : _producers) {
         auto snap = state->snapshot();
         // Discard finished requests below the local snapshot start offset,
         // they are no longer relevant.
-        std::erase_if(
-          snap.finished_requests, [start_kafka_offset](const auto& req) {
-              return req.last_offset < start_kafka_offset;
-          });
+        // std::erase_if(
+        //   snap.finished_requests, [start_kafka_offset](const auto& req) {
+        //       return req.last_offset < start_kafka_offset;
+        //   });
         if (
           !snap.finished_requests.empty()
           || state->has_transaction_in_progress()) {

tests/antithesis/ct_stress/docker-compose.yaml.j2

@@ -0,0 +1,109 @@
+name: {{ test_slug }}
+
+services:
+  minio:
+    image: {{ minio_image }}
+    container_name: minio
+    hostname: minio
+    platform: linux/amd64
+    init: true
+    restart: "no"
+    command: server /data
+    environment:
+      MINIO_REGION_NAME: {{ s3_region }}
+      MINIO_ROOT_USER: {{ s3_access_key }}
+      MINIO_ROOT_PASSWORD: {{ s3_secret_key }}
+      MINIO_BROWSER: "off"
+    expose:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 5s
+      retries: 60
+
+  minio-init:
+    image: {{ minio_image }}
+    container_name: minio-init
+    hostname: minio-init
+    platform: linux/amd64
+    init: true
+    restart: "no"
+    entrypoint: ["/bin/bash", "/scripts/minio-init.sh"]
+    environment:
+      MINIO_URL: http://minio:9000
+      BUCKET: {{ bucket }}
+      ACCESS_KEY: {{ s3_access_key }}
+      SECRET_KEY: {{ s3_secret_key }}
+    volumes:
+      - ./scripts:/scripts:ro
+    depends_on:
+      minio:
+        condition: service_healthy
+{% for i in range(nodes) %}
+
+  redpanda-{{ i }}:
+    image: {{ redpanda_image }}
+    container_name: redpanda-{{ i }}
+    hostname: redpanda-{{ i }}
+    platform: linux/amd64
+    init: true
+    # No restart policy: docker-compose restarting a killed broker would
+    # nullify Antithesis node-termination faults. Antithesis restarts the
+    # container itself after the fault window. See
+    # https://antithesis.com/docs/concepts/fault_injection/fault_types/#node-termination
+    restart: "no"
+    entrypoint: ["/scripts/redpanda-entrypoint.sh"]
+    command:
+      - redpanda
+      - start
+      # Pin node_id to the container index so redpanda-N is node N. Auto-assigned
+      # ids follow registration order, not container names, which makes log
+      # correlation error-prone.
+      - --node-id={{ i }}
+      - --smp={{ smp }}
+      - --memory={{ memory }}
+      - --reserve-memory=0M
+      - --overprovisioned
+      - --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      - --advertise-kafka-addr=PLAINTEXT://redpanda-{{ i }}:9092
+      - --rpc-addr=0.0.0.0:33145
+      - --advertise-rpc-addr=redpanda-{{ i }}:33145
+{% if i > 0 %}
+      - --seeds=redpanda-0:33145
+{% endif %}
+    volumes:
+      - ./scripts:/scripts:ro
+    ulimits:
+      nofile:
+        soft: 131072
+        hard: 131072
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9644/v1/status/ready"]
+      interval: 5s
+      timeout: 5s
+      retries: 120
+    depends_on:
+      minio-init:
+        condition: service_completed_successfully
+{% endfor %}
+
+  # franz-go workload. Its entrypoint (workload setup) waits for the cluster,
+  # emits setup_complete, then idles so Antithesis runs the baked-in test
+  # commands (/opt/antithesis/test/v1/main/) in this container.
+  workload:
+    image: {{ workload_image }}
+    container_name: workload
+    hostname: workload
+    platform: linux/amd64
+    init: true
+    restart: "no"
+    environment:
+      KAFKA_BROKERS: "{% for i in range(nodes) %}{% if not loop.first %},{% endif %}redpanda-{{ i }}:9092{% endfor %}"
+      ADMIN_HOSTS: "{% for i in range(nodes) %}{% if not loop.first %},{% endif %}redpanda-{{ i }}:9644{% endfor %}"
+      EXPECTED_BROKERS: "{{ nodes }}"
+    depends_on:
+{% for i in range(nodes) %}
+      redpanda-{{ i }}:
+        condition: service_healthy
+{% endfor %}

tests/antithesis/ct_stress/scripts/bootstrap.yaml.j2

@@ -0,0 +1,39 @@
+# Initial cluster configuration applied on first cluster bootstrap (staged to
+# /etc/redpanda/.bootstrap.yaml on every node by redpanda-entrypoint.sh).
+cloud_storage_enabled: true
+cloud_storage_access_key: {{ s3_access_key }}
+cloud_storage_secret_key: {{ s3_secret_key }}
+cloud_storage_region: {{ s3_region }}
+cloud_storage_api_endpoint: minio
+cloud_storage_api_endpoint_port: 9000
+cloud_storage_disable_tls: true
+cloud_storage_url_style: path
+cloud_storage_bucket: {{ bucket }}
+
+auto_create_topics_enabled: false
+
+default_redpanda_storage_mode_tiered_impl: tiered_v2
+
+# One metastore/domain partition (kafka_internal/ct_l1_domain) to have
+# more concentrated activity. Since they don't interact between each other,
+# more partitions don't add more coverage.
+cloud_topics_num_metastore_partitions: 1
+
+# Shrink cloud-topics object sizes so a light workload still produces many
+# L0/L1 objects and dense indexes. This drives reconciliation, leveling and
+# the offset-mapping paths across many object boundaries in a short run,
+# where the checker's ordering/contiguity assertions do their work.
+cloud_topics_produce_batching_size_threshold: 65536   # 64 KiB (default 4 MiB)
+cloud_topics_reconciliation_max_object_size: 1048576  # 1 MiB (default 80 MiB)
+# NB: config key is cloud_topics_indexing_interval (not ..._l1_...).
+cloud_topics_indexing_interval: 16384                 # 16 KiB (default 4 MiB)
+
+# Compact cloud topics more often than the 30s default so the compacted
+# topic (ctc) goes through many compaction rounds in a short run.
+cloud_topics_compaction_interval_ms: 10000
+
+# Smallest metastore LSM memtable and SST read chunk the config allows
+# (both floor at 1 MiB), to get as much SST churn / chunked object-storage
+# reads as the low metastore load permits.
+cloud_topics_metastore_write_buffer_size: 1048576     # 1 MiB (min; default 16 MiB)
+cloud_topics_metastore_sst_chunk_size: 1048576        # 1 MiB (min; default 24 MiB)

tests/antithesis/ct_stress/scripts/minio-init.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Copyright 2026 Redpanda Data, Inc.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.md
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0
+# ==================================================================
+#
+# One-shot init: creates the object storage bucket Redpanda is
+# configured with. Services wait on it via
+# depends_on: service_completed_successfully.
+
+set -euo pipefail
+
+: "${MINIO_URL:?}"
+: "${BUCKET:?}"
+: "${ACCESS_KEY:?}"
+: "${SECRET_KEY:?}"
+
+mc alias set local "$MINIO_URL" "$ACCESS_KEY" "$SECRET_KEY"
+mc mb --ignore-existing "local/$BUCKET"
+
+echo "bucket $BUCKET ready"

tests/antithesis/ct_stress/scripts/redpanda-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright 2026 Redpanda Data, Inc.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.md
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0
+# ==================================================================
+
+set -euo pipefail
+
+# Copy the bootstrap config off its read-only bind mount onto the container's
+# writable layer, then start Redpanda. Under Antithesis a bind mount is backed
+# by overlay-over-squashfs, which mishandles O_DIRECT (close() returns EINVAL);
+# Redpanda opens config files with O_DIRECT and would crash reading it there.
+# The writable layer handles O_DIRECT, so a plain (buffered) copy sidesteps it.
+# See seastar issue 3518.
+cp /scripts/bootstrap.yaml /etc/redpanda/.bootstrap.yaml
+
+exec rpk "$@"

tests/antithesis/ct_stress/test.yml

@@ -0,0 +1,56 @@
+# Antithesis test manifest, consumed by tools/antithesis/template_test_package.py.
+#
+# vars provides the default values for the Jinja2 templates (*.j2) in this
+# directory; override individual values at packaging time with --var KEY=VALUE.
+#
+# images declares companion images built from this test's own sources. Each
+# is built, tagged with a hash of its build context, exposed to the templates
+# as the named variable, and (for real runs) pushed and registered with
+# Antithesis so libvoidstar is injected into it.
+name: cloud topics stress testing
+
+images:
+  workload_image:
+    context: workload
+
+# Antithesis fault scoping, applied at launch (--submit). List entries name
+# container roles: `redpanda` expands to every broker (redpanda-0..nodes-1),
+# other entries are literal container names. Every entry is sent explicitly
+# (empty lists as ""), so this manifest -- not the webhook defaults -- is the
+# source of truth for the fault posture.
+faults:
+  # Global toggles (pinned here so the fault posture is self-documenting).
+  # clock_skew: on by the webhook default anyway.
+  # cpu_mod: simulates different hardware by varying CPU/instruction speed,
+  #   reordering concurrent execution to surface races. Runs in addition to the
+  #   thread pausing induced by the Antithesis instrumentation in our
+  #   --config=antithesis build. Not recorded in the fault log.
+  clock_skew: true
+  cpu_mod: true
+  node_hang: [redpanda, workload]
+  node_throttle: [redpanda, workload]
+  # Explicitly empty: no containers included, so node termination is off. Enable
+  # it by adding brokers -- never minio (single instance, no S3 durability).
+  node_termination: []
+  # Containers excluded from network faults. Explicitly empty: nothing excluded,
+  # so every container -- minio too -- sees network faults, exercising
+  # object-storage disruption.
+  exclude_from_faults: []
+
+vars:
+  # Any image with the stock rpk entrypoint works here. Override with an
+  # instrumented build for real Antithesis runs (--redpanda-image).
+  redpanda_image: docker.redpanda.com/redpandadata/redpanda-nightly:latest
+  minio_image: minio/minio:RELEASE.2025-07-23T15-54-02Z
+  nodes: 5
+  # 2 shards (threads) per broker. Memory must fit Antithesis' ~10G total:
+  # 5 brokers x 1.2G = 6G leaves room for MinIO, the workload, the OS page
+  # cache and hypervisor overhead.
+  # This is below Redpanda's 1 GiB/shard floor (unavoidable: 5x2 shards at
+  # 1 GiB would need the whole 10G), so expect a cosmetic memory warning.
+  smp: 2
+  memory: 1200M
+  bucket: ct-stress
+  s3_access_key: panda-user
+  s3_secret_key: panda-secret
+  s3_region: panda-region

tests/antithesis/ct_stress/workload/Dockerfile

@@ -0,0 +1,50 @@
+# syntax=docker/dockerfile:1
+#
+# ==================================================================
+# Copyright 2026 Redpanda Data, Inc.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.md
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0
+# ==================================================================
+#
+# ct_stress Antithesis workload image. Builds the franz-go workload with
+# Antithesis assertion cataloging, then ships it on a glibc base so the SDK
+# can dlopen /usr/lib/libvoidstar.so (injected by Antithesis) at runtime.
+# CGO must stay enabled for that path; without libvoidstar the SDK falls
+# back to local behavior, so the same image also runs locally.
+
+FROM golang:1.26-bookworm AS build
+WORKDIR /src
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+# Generate the assertion catalog (antithesis_catalog.go) so every assertion
+# is pre-registered with Antithesis, then build with cgo for libvoidstar.
+RUN go run github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-instrumentor@v0.7.2 \
+    -assert_only -skip_test_files . \
+    && CGO_ENABLED=1 go build -o /out/workload .
+
+FROM debian:bookworm-slim
+ENV NO_COLOR=1
+
+# The binary lives inside the test template as a helper_ file (Antithesis
+# ignores helper_-prefixed entries) and each command is a RELATIVE symlink to
+# it. Relative, in-directory links stay valid when the template is copied out
+# of the container (e.g. by `snouty validate` or Antithesis discovery); an
+# absolute symlink to a path outside the template would dangle. The binary
+# reports its own commands, so the list lives only in the Go source.
+COPY --from=build /out/workload /opt/antithesis/test/v1/main/helper_workload
+RUN cd /opt/antithesis/test/v1/main \
+    && for c in $(./helper_workload list-commands); do \
+         ln -s helper_workload "$c"; \
+       done
+
+# The container idles here (setup waits for health, emits setup_complete,
+# then blocks) so Antithesis can execute the test commands inside it.
+ENTRYPOINT ["/opt/antithesis/test/v1/main/helper_workload", "setup"]