Add txset construction mode to apply load

docs/apply-load-benchmark-sac.cfg

@@ -7,10 +7,13 @@
 APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="sac"
 
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The Soroban multiplier applies to transaction
+# envelopes after SAC batching.
 
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,

docs/apply-load-benchmark-token.cfg

@@ -7,10 +7,12 @@
 APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="custom_token"
 
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. Token transfers need two accounts per candidate.
 
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
@@ -61,4 +63,4 @@ NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self"
 
 [QUORUM_SET]
 THRESHOLD_PERCENT=100
-VALIDATORS=["$self"]
+VALIDATORS=["$self"]

docs/apply-load-for-meta.cfg

@@ -9,10 +9,13 @@
 # Select the apply-load mode.
 APPLY_LOAD_MODE="ledger-limits"
 
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate generation uses Herder's queue size multipliers (2x ledger capacity
+# by default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The classic multiplier scales the candidate
+# count; the Soroban multiplier scales the candidate resource budget.
 
 # Custom meta path - if not set it will be written to a temp directory and
 # cleaned up after running the benchmark

docs/apply-load-ledger-limits.cfg

@@ -7,10 +7,13 @@
 # Select the apply-load mode.
 APPLY_LOAD_MODE="ledger-limits"
 
-# Which timing path to use: "apply" times only transaction application, while
-# "txset-validation-and-apply" also simulates a non-leader receiving and
-# validating a tx set before applying it. Tx-set creation is not measured.
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every benchmark reports tx-set construction, validation, ledger close and
+# receive-to-close timings.
+
+# Candidate generation uses Herder's queue size multipliers (2x ledger capacity
+# by default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The classic multiplier scales the candidate
+# count; the Soroban multiplier scales the candidate resource budget.
 
 # Medida metrics (histograms in particular) in apply path cause severe and
 # non-deterministic performance degradation. While this has to be addressed

docs/apply-load-max-sac-tps.cfg

@@ -1,15 +1,20 @@
 # This is the Stellar Core configuration example for using the load generation 
 # (apply-load) tool for testing the theoretical max SAC (Stellar asset contract)
-# transfer TPS via binary search (measured based on apply time only).
+# transfer TPS via binary search driven by application timing.
 
 # The core with this configuration should run using `./stellar-core apply-load`
 
 # Select the apply-load mode.
 APPLY_LOAD_MODE="max-sac-tps"
 
-# Which timing path to use. The max-sac-tps search targets apply-only close
-# time, so it only supports "apply".
-APPLY_LOAD_TIMING_PHASES = "apply"
+# Every search sample reports tx-set construction, validation, ledger close
+# and receive-to-close timings. Only application timing drives the search;
+# APPLY_LOAD_TIME_WRITES controls whether that timing includes writes.
+
+# Candidate counts use Herder's queue size multipliers (2x ledger capacity by
+# default), purposefully overfilling the candidate list to simulate a busy
+# production validator's queue. The Soroban multiplier applies to transaction
+# envelopes after SAC batching.
 
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
@@ -30,7 +35,7 @@ APPLY_LOAD_MAX_SAC_TPS_MIN_TPS = 1000
 # Upper bound of the TPS in binary search
 APPLY_LOAD_MAX_SAC_TPS_MAX_TPS = 15000
 
-# Number of seconds to apply the ledger for
+# Target application time per ledger, in milliseconds
 APPLY_LOAD_TARGET_CLOSE_TIME_MS = 1000
 
 # The only relevant network configuration parameter - number of transaction
@@ -59,9 +64,9 @@ APPLY_LOAD_BL_LAST_BATCH_LEDGERS = 0
 ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true
 # Diagnostic events should generally be disabled, but can be enabled for debug
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = false
-# Set up a plenty of genesis accounts - benchmark will fail if the number is 
-# not sufficient. This should be at least 2x of the maximum TPL, but can be set
-# higher than that.
+# Reserve one account per Soroban candidate at the rounded upper search bound,
+# plus the classic candidates and two setup accounts. SAC batching reduces the
+# envelope count before the Soroban queue multiplier is applied.
 GENESIS_TEST_ACCOUNT_COUNT = 100000
 
 # Minimal core config boilerplate

docs/metrics.md

@@ -62,6 +62,7 @@ crypto.verify.miss                        | meter     | number of signature cach
 crypto.verify.total                       | meter     | sum of both hits and misses
 crypto.verify.tx-valid-hit                | meter     | signature cache hits that occurred while validating transactions (outside of background signature validation)
 crypto.verify.tx-valid-total              | meter     | sum of both hits and misses during transaction validation (outside of background signature validation)
+herder.txset.build                        | timer     | time spent building the tx set proposed at nomination from the transaction queues
 herder.txset.validate                     | timer     | time spent turning a received tx set into an applicable tx set and validating it on a validity-cache miss
 herder.pending[-soroban]-txs.age0         | counter   | number of gen0 pending transactions
 herder.pending[-soroban]-txs.age1         | counter   | number of gen1 pending transactions

docs/software/commands.md

@@ -19,45 +19,35 @@ Common options can be placed at any place in the command line.
 ## Command line options
 Command options can only by placed after command.
 
-* **apply-load**: Benchmarks Soroban transaction application time using
-    synthetic transactions. By default the benchmark is isolated to mostly just
-    executing the transactions and thus it omits a lot of the supporting
-    mechanisms (such as overlay, SCP, mempool etc). It may also measure tx-set
-    validation and consensus processing via `APPLY_LOAD_TIMING_PHASES` (see
-    below). This command will generate enough
-    transactions to fill up a synthetic transaction queue (it's just a list of
-    transactions with the same limits as the real queue), and then create a
-    transaction set off of that to apply. This can also be used to record the
-    synthetic ledger close metadata emitted during the benchmark, and then use
-    it for benchmarking the meta consumers.
-  * This can only be used when `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true`
-  * The mode is selected in the config file using `APPLY_LOAD_MODE`:
-    - `APPLY_LOAD_MODE="ledger-limits"`: the default mode that measures the
-      ledger close time for applying transactions.
-    - `APPLY_LOAD_MODE="max-sac-tps"`: determines maximum TPS for the load
-      consisting only of fast SAC transfer.
-    - `APPLY_LOAD_MODE="benchmark"`: benchmarks a fixed-size ledger of model
-      transactions. Use `APPLY_LOAD_MODEL_TX` to select the model transaction.
-  * `APPLY_LOAD_TIMING_PHASES` selects one of two timing paths:
-    - `"apply"`: the default apply-only benchmark. Its close helper still calls
-      `checkValid`, but that happens before the recorded ledger-close timer and
-      leaves the caches warm, as consensus validation would on a live node.
-    - `"txset-validation-and-apply"`: simulates a non-leader receiving the tx
-      set over the wire, validating it through local consensus (with the node
-      as its own single-validator quorum), and then applying it. It reports
-      validation, ledger close, and end-to-end time in addition to the
-      apply-only output. It does not simulate network transport, peer fetching,
-      or multi-node timing. Leader-side tx-set creation and signing happen
-      before the measured span. The signature verification cache is cleared
-      before validation, then retained so apply sees the warm cache produced by
-      validation.
-    `"txset-validation-and-apply"` is not supported with
-    `APPLY_LOAD_MODE="max-sac-tps"`; that search targets apply-only close time.
-  * Load generation is configured in the Core config file. The relevant settings
-    all begin with `APPLY_LOAD_`. See full example configurations with
-    per-setting documentation in the `docs` directory
-    (all the `apply-load-*.cfg` files demonstrate different modes and use 
-    cases).
+* **apply-load**: Benchmarks tx-set construction, validation and application
+    using synthetic transactions. Every workload reports these phases plus
+    receive-to-close time. It can also record synthetic ledger close metadata
+    for benchmarking meta consumers.
+  * This can only be used when `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true`.
+  * `APPLY_LOAD_MODE` selects the workload:
+    - `"ledger-limits"`: generates load against configured resource limits for
+      a fixed number of ledgers.
+    - `"benchmark"`: benchmarks a fixed-size ledger of model transactions
+      selected by `APPLY_LOAD_MODEL_TX`.
+    - `"max-sac-tps"`: uses noisy binary search to find the SAC throughput that
+      meets `APPLY_LOAD_TARGET_CLOSE_TIME_MS`. Every sample reports all phases,
+      but only application timing drives the search. `APPLY_LOAD_TIME_WRITES`
+      selects ledger-close timing (including writes) or transaction-apply
+      timing (excluding writes). Phase summaries cover all sampled search loads.
+  * Construction uses an overfilled synthetic candidate list to simulate a busy
+    validator's queue. All modes use Herder's queue size multipliers (2× ledger
+    capacity by default). They scale model and classic candidate counts; in
+    `ledger-limits` mode the Soroban multiplier scales the candidate resource
+    budget. SAC batching is applied before scaling envelope counts.
+  * Each benchmark builds the set, reconstructs it from wire bytes, then
+    validates and applies it through local consensus with a single-validator
+    quorum. The signature cache is cleared before validation and retained for
+    application. Receive-to-close timing starts at wire decoding and excludes
+    construction. Candidate generation, serialization, signing and setup are
+    outside the reported phases. Network transport, peer fetching, transaction
+    queue submission and multi-node timing are not simulated.
+  * See `docs/apply-load-*.cfg` for example configurations and per-setting
+    documentation.
 
 * **calculate-asset-supply**: Calculates total supply of an asset from the live and hot archive bucket lists IF the total supply fits in a 64 bit signed integer. Also validates against totalCoins for the native asset. Uses `--code <CODE>` and `--issuer <ISSUER>` to specify the asset. Uses the native asset if neither `--code` nor `--issuer` is given.
 * **catchup <DESTINATION-LEDGER/LEDGER-COUNT>**: Perform catchup from history

src/herder/HerderImpl.cpp

@@ -47,6 +47,7 @@
 
 #include "util/GlobalChecks.h"
 #include <algorithm>
+#include <chrono>
 #include <ctime>
 #include <fmt/format.h>
 
@@ -82,6 +83,7 @@ HerderImpl::SCPMetrics::SCPMetrics(Application& app)
           {"scp", "envelope", "invalidsig"}, "envelope"))
     , mTriggerPrepareStartFallback(app.getMetrics().NewMeter(
           {"scp", "trigger", "prepare-start-fallback"}, "trigger"))
+    , mTxSetBuild(app.getMetrics().NewTimer({"herder", "txset", "build"}))
 {
 }
 
@@ -1664,6 +1666,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     // Since we are not currently applying, it is safe to use read-only LCL, as
     // it's guaranteed to be up-to-date
     auto lcl = mLedgerManager.getLastClosedLedgerHeader();
+    auto const txSetBuildStart = std::chrono::steady_clock::now();
     PerPhaseTransactionList txPhases;
     txPhases.emplace_back(mTransactionQueue.getTransactions(lcl.header));
 
@@ -1752,6 +1755,12 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     mTransactionQueue.ban(
         invalidTxPhases[static_cast<size_t>(TxSetPhase::CLASSIC)]);
 
+    // Stop before addTxSet below: its side effects can include SCP callbacks
+    // and even externalizing a ledger, which are not tx set building.
+    mSCPMetrics.mTxSetBuild.Update(
+        std::chrono::duration_cast<std::chrono::nanoseconds>(
+            std::chrono::steady_clock::now() - txSetBuildStart));
+
     auto txSetHash = proposedSet->getContentsHash();
 
     // Inform the item fetcher so queries from other peers about his txSet

src/herder/HerderImpl.h

@@ -362,6 +362,9 @@ class HerderImpl : public Herder
         // network-close-time anchor to the local prepare-start anchor.
         medida::Meter& mTriggerPrepareStartFallback;
 
+        // Time spent building the tx set proposed at nomination.
+        medida::Timer& mTxSetBuild;
+
         SCPMetrics(Application& app);
     };
 

src/main/Config.cpp

@@ -464,23 +464,6 @@ parseApplyLoadModelTx(ConfigItem const& item)
         "invalid 'APPLY_LOAD_MODEL_TX', expected one of: sac, custom_token, "
         "soroswap");
 }
-
-ApplyLoadTimingPhases
-parseApplyLoadTimingPhases(ConfigItem const& item)
-{
-    auto phases = readString(item);
-    if (phases == "apply")
-    {
-        return ApplyLoadTimingPhases::APPLY_ONLY;
-    }
-    if (phases == "txset-validation-and-apply")
-    {
-        return ApplyLoadTimingPhases::TX_SET_VALIDATION_AND_APPLY;
-    }
-    throw std::invalid_argument(
-        "invalid 'APPLY_LOAD_TIMING_PHASES', expected one of: apply, "
-        "txset-validation-and-apply");
-}
 #endif
 
 template <typename T>
@@ -1883,11 +1866,6 @@ Config::processConfig(std::shared_ptr<cpptoml::table> t)
                  }},
                 {"APPLY_LOAD_TIME_WRITES",
                  [&]() { APPLY_LOAD_TIME_WRITES = readBool(item); }},
-                {"APPLY_LOAD_TIMING_PHASES",
-                 [&]() {
-                     APPLY_LOAD_TIMING_PHASES =
-                         parseApplyLoadTimingPhases(item);
-                 }},
 #endif // BUILD_TESTS
                 {"GENESIS_TEST_ACCOUNT_COUNT",
                  [&]() {

src/main/Config.h

@@ -83,13 +83,6 @@ enum class ApplyLoadModelTx
     CUSTOM_TOKEN,
     SOROSWAP
 };
-
-// Which apply-load timing path to use.
-enum class ApplyLoadTimingPhases
-{
-    APPLY_ONLY,
-    TX_SET_VALIDATION_AND_APPLY
-};
 #endif
 
 class Config : public std::enable_shared_from_this<Config>
@@ -432,10 +425,6 @@ class Config : public std::enable_shared_from_this<Config>
     // If set to true, database writes will count towards TPS calculation.
     // Otherwise, BucketList writes will not be counted.
     bool APPLY_LOAD_TIME_WRITES = true;
-
-    // Which apply-load timing path to use.
-    ApplyLoadTimingPhases APPLY_LOAD_TIMING_PHASES =
-        ApplyLoadTimingPhases::APPLY_ONLY;
 #endif // BUILD_TESTS
 
     // Waits for merges to complete before applying transactions during catchup