diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index be3cceeb25..983b18f081 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -242,6 +242,66 @@ pub enum ChaosScenarioArg { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, + /// Two executors cut off from each other while agents on one invoke agents + /// on the other. The control: the link carries no traffic. + S2, + /// One executor unable to resolve the shard manager's name, while its + /// connection to it stays up. The second control: that connection is built + /// once with an infinite idle TTL, so DNS is never consulted again. + S4, + /// The same DNS failure, held across a shard-manager restart so the + /// executor has to rebuild the connection and resolve a name that no longer + /// resolves. S4's companion. + Mf2, + /// Executor pod kill while agents are having their state reverted. + S7, + /// Executor pod kill while agents are being deleted. + S6, + /// Executor pod kill while a component rollback is in flight. + S9, + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. + S16, + /// The same cut, held for longer than the key-value retry budget. + S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster for the + /// length of a writer failover. + S14, + /// Executors cut off from the Redis cache in front of the key-value layer, + /// for longer than a caller is willing to wait. + S18, + /// The same Redis cache slowed rather than removed. + S17, + /// The key-value PostgreSQL cluster slowed rather than removed. S17's + /// mirror on the other half of the split key-value layer. + S15, + /// S15 driving `ephemeral` alone: the control for the elimination series. + S15A, + /// S15A plus `durable`. + S15B, + /// S15B plus `promise`. + S15C, + /// The indexed-oplog PostgreSQL cluster slowed rather than removed. S14's + /// mirror, and the one delay with no control stream: every agent commits + /// its oplog to that cluster. + S23, + /// The first composed fault: a worker-executor killed while the key-value + /// PostgreSQL cluster is unreachable. + Mf1, + /// MF1 with the overlap pushed past the shard-manager's assign timeout. + Mf1b, + /// Worker-service CPU saturation while agents drive cross-pod RPC. S2's + /// populations with the fault moved off the link and onto the relay. + S21, + /// One executor's wall clock moved half a minute behind the cluster, where + /// the quota lease is the only contract two clocks are compared across. + S19, } /// Density subcommand action. diff --git a/golem-test-framework/src/benchmark/results.rs b/golem-test-framework/src/benchmark/results.rs index 15785319ae..9ccd9685a7 100644 --- a/golem-test-framework/src/benchmark/results.rs +++ b/golem-test-framework/src/benchmark/results.rs @@ -525,6 +525,17 @@ pub struct RunMetadata { /// Container image tag of the deployed `worker-service`. #[serde(skip_serializing_if = "Option::is_none", default)] pub worker_service_image_tag: Option, + /// Image digest of the deployed `worker-executor`, when the manifest pins + /// one. A tag can be moved; the digest is what actually identifies the + /// build a run tested, which is the whole point of recording it. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_executor_image_digest: Option, + /// Image digest of the deployed `registry-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub registry_service_image_digest: Option, + /// Image digest of the deployed `worker-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_service_image_digest: Option, /// Aurora ACU capacity for the main (`golem_dev`) cluster at run start. #[serde(skip_serializing_if = "Option::is_none", default)] pub aurora_acu_main: Option, @@ -575,6 +586,9 @@ impl RunMetadata { worker_executor_image_tag: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_TAG"), registry_service_image_tag: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_TAG"), worker_service_image_tag: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_TAG"), + worker_executor_image_digest: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_DIGEST"), + registry_service_image_digest: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_DIGEST"), + worker_service_image_digest: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_DIGEST"), aurora_acu_main: env_f64("GOLEM_BENCH_AURORA_ACU_MAIN"), aurora_acu_indexed: env_f64("GOLEM_BENCH_AURORA_ACU_INDEXED"), aurora_acu_keyvalue: env_f64("GOLEM_BENCH_AURORA_ACU_KEYVALUE"), diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 00ed926cfc..16f37bf78a 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -175,7 +175,11 @@ scenarios: # executors that stay connected keep getting told the truth, and the # disagreement between the two groups is the whole experiment. targetCount: 1 - durationSecs: 120 + # Must equal phases.faultSecs; see the note on S16. The joiner partition + # this scenario applies mid-fault is already sized from faultSecs, so a + # shorter value here also made the two partitions lapse at different + # times. + durationSecs: 180 phases: baselineSecs: 300 @@ -318,6 +322,151 @@ scenarios: signalTimeoutSecs: 1800 + # S10 — executor pod kill while scheduled actions are pending (GOL-378). + # + # The only scenario whose workload the driver is not holding a connection to. + # Every target registers an action ten seconds ahead, over and over, so a few + # hundred actions are always accepted and not yet run. Killing the executor + # that owns their shards asks whether the platform still runs each of them, + # exactly once, after the shards move. + # + # Like S8 the driver names the pod: it picks the executor owning the largest + # share of targets and reports its address in `baseline-ready.json`. Unlike + # S8, the targets on the *other* executor keep registering too — they are the + # control group, and the report splits the two. + - code: S10 + name: executor-crash-during-scheduled-fire + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8. + mode: one + durationSecs: 60 + + phases: + # Eighteen leads' worth of registrations, so the baseline measures a + # cluster that has been firing scheduled actions steadily rather than one + # that has just started. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything claims the dead executor's due actions. + faultSecs: 120 + # Registrations continue throughout. Long enough that the actions + # registered during the fault also fall due, fire and are accounted for + # inside the run rather than after it. + recoverySecs: 300 + + scheduled: + # 100 targets, each with its own emitter. Also the resolution of the + # report: a lost action localises to one target out of a hundred. + targets: 100 + # 50 registrations a second overall, which is half what the mixed workload + # offers. The scheduled path is the whole experiment here rather than one + # stream of four, so the rate is set by what keeps the pending population + # large rather than by a share of a budget. + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant. It also has to + # comfortably exceed the workflow's inject-and-verify path — signal poll + # (5s) plus `kubectl apply` plus waiting for `AllInjected` — or everything + # registered before the kill would already have fired by the time the pod + # died. + leadSecs: 10 + # What recovering a pending action may cost, and the number the fire-delay + # percentiles are reported against. Derived rather than picked: the + # executor's scheduler holds a 30s lease and refreshes every 2s, and a + # shard reassignment has to complete before the surviving executor claims + # anything at all. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + leaseBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # extra consequence here: the retry goes out under the original key, and + # that key is what the scheduled action carries back to the target. So a + # retried registration that registered twice shows up as one token firing + # twice, not as an arithmetic argument about totals. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S11 — executor pod kill while agents are suspended on promises (GOL-377). + # + # The only scenario whose agents are asleep when the fault lands. Each waiter + # creates a promise, parks inside an invocation awaiting it, and is resolved + # from outside a few seconds later. Killing the executor that owns them asks + # whether a completion the platform accepted still reaches the agent it was + # meant to wake, exactly once, after the shards move. + # + # Like S10 the driver names the pod and keeps driving the waiters on the other + # executor as a control group. Unlike every other scenario, its agents live in + # the promise component rather than the counters one. + - code: S11 + name: executor-crash-during-promise-completion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8 and S10. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the kill + # lands on a population that has been cycling steadily rather than one + # still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can resume a waiter the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that waiters disturbed by the + # kill get several whole rounds afterwards, so a waiter that recovered + # late is distinguishable from one that never did. + recoverySecs: 300 + + promise: + # 200 waiters, each parked on at most one promise, so 200 agents are + # suspended at any instant. Also the resolution of the report: a lost + # wakeup localises to one waiter out of two hundred. + waiters: 200 + # 5s parked before the completion goes out, so ~40 completions a second + # across the pool. It has to comfortably exceed the workflow's + # inject-and-verify path — signal poll (5s) plus `kubectl apply` plus + # waiting for `AllInjected` — or every promise armed before the kill would + # already have been completed by the time the pod died. + dwellMillis: 5000 + # What resuming a suspended waiter may cost, and the number the wakeup + # delay percentiles are reported against. Derived rather than picked: a + # shard reassignment has to complete before anything owns the waiter, and + # then the worker has to be recovered and its oplog replayed to the point + # it was parked at. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + wakeupBudgetSecs: 60 + + retryPolicy: + # Identical to the others. One extra consequence here, spelled out in + # integration-tests/src/chaos/waiters.rs: a completion retry can repair a + # wakeup that was lost, but only for a completion that had *not* already + # been accepted — so the question this scenario asks, about completions the + # platform confirmed, is untouched by it. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + # S13 — rolling executor restarts under load (GOL-367). # # One executor killed every 60 seconds for five minutes while the mixed @@ -384,3 +533,2587 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S3 — executor / worker-service network partition (GOL-370). + # + # The only fault in the suite the platform never learns about. S1 cuts the + # shard-manager off from an executor and the cluster reacts: health checks are + # missed and the shards move. S3 cuts the link worker-service uses, leaves the + # shard-manager link alone, and so the executor keeps its shards and stays in + # the routing table. worker-service is told, correctly, that this pod owns + # those agents, and keeps trying to reach a pod it cannot reach. There is no + # route around it, because as far as the platform is concerned nothing is + # wrong. + # + # Like S8, S10 and S11 the driver names the pod rather than letting Chaos Mesh + # pick one, and keeps driving the agents on the other executor as the control + # group. That control group is where the interesting finding would be: a + # partition it is not part of should cost it nothing. + - code: S3 + name: executor-worker-service-partition + enabled: true + + fault: + kind: network-partition + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8, S10 and S11. + # + # Only the executor side is pinned. The other side of the cut is every + # worker-service replica, selected by label — a partition from two of the + # three would leave the agents reachable through the third and the run + # would measure nothing. + mode: one + durationSecs: 180 + + phases: + # Long enough for cold starts and route warm-up to settle, so the cut + # lands on a population that has been cycling steadily. + baselineSecs: 300 + # Has to comfortably exceed the caller's 120s attempt timeout, or no + # operation would ever reach the timeout inside the window and the + # pending-then-timeout behaviour the scenario exists to show would fall + # outside it. 180s leaves room for the first attempt to time out, the + # one permitted retry to go out, and that retry to still be waiting when + # the link comes back. + faultSecs: 180 + # Every isolated agent has to get several whole operations after the heal, + # so one that recovered slowly is distinguishable from one that never did. + recoverySecs: 420 + + isolation: + # 200 agents, split by shard ownership across the two executors, so ~100 + # end up isolated and ~100 are the control group. Also the resolution of + # the report: a lost agent localises to one out of two hundred. + agents: 200 + # One operation per agent per second, so ~200 ops/s offered while + # everything is reachable. Higher than the other scenarios' 100/s because + # half of it stops the moment the fault lands: the number that has to be + # large is the *control* group's, since a collateral-damage finding is a + # percentage change in its throughput and a small sample cannot show one. + intervalMillis: 1000 + # The most of its baseline the isolated group may keep for the partition + # to count as observed. Not zero: an operation submitted late in the fault + # window can still be waiting when the link returns, and then confirms. + # At this cadence that is a handful against a baseline of hundreds, so + # anything near 25% means the cut did not take hold. + isolatedCeilingPercent: 25 + # The least of its baseline the control group must keep. Below this, + # serving an unreachable executor cost the agents that had nothing to do + # with it — the sharpest finding this scenario can produce. + # + # 75% rather than 95%: worker-service invalidates one process-wide routing + # table entry on every failed call, so some shared cost is expected by + # construction. `invalidation_min_delay` bounds it to twice a second, + # which should keep it far above this line. + controlFloorPercent: 75 + # What being served again may cost an isolated agent once the link is + # back. Recorded, not asserted: nothing has to be recovered here — no + # shard moves and no worker restarts — so this is really a measure of how + # long worker-service's own retry loop takes to notice, and how long that + # may be is a judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S3: the retry is what turns a single 120s stall + # into a second attempt under the same key. If the link heals in between, + # that retry succeeds against work the first attempt may already have + # started — which is exactly the case the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S2 — executor-to-executor partition, cross-pod RPC control (GOL-368). + # + # The suite's control, and the only scenario whose fault is expected to change + # nothing at all. + # + # Executors hold no connection to each other. `WorkerExecutorClient` appears + # nowhere in golem-worker-executor; when an agent invokes an agent its own + # executor does not own, `DirectWorkerInvocationRpc::invoke_and_await` hands + # the call to `worker_proxy`, which is a client of worker-service. So executor + # A reaches executor B's agents through a third party, and cutting the A-to-B + # link cuts a link carrying no traffic. This run is what turns that reading of + # the code into a measurement on a real cluster, and what fails if someone + # later adds a direct path. + # + # ── Why a control needs more guards than a normal scenario ─────────────────── + # + # A scenario that expects damage fails safe: the fault misses, the damage is + # absent, and the report says the fault was not observed. This one fails the + # other way. A run whose pairs were accidentally co-located, or whose workload + # never started, produces exactly the clean report a good run produces. Hence + # `crossPodFloorPercent` below, which stops the run before the fault window + # rather than letting it report numbers it did not earn. + # + # One weakness is inherent and stated in the result rather than hidden: nothing + # in the cluster is supposed to stop, so there is no in-run evidence the + # partition took hold. Chaos Mesh reporting AllInjected is all there is. + - code: S2 + name: executor-executor-partition + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Both sides selected by label, and `all` on both. On the two-replica + # topology a chaos run uses that is every executor there is, so each drops + # traffic to the other and there is no pod for the driver to name — unlike + # S3, where naming one side is the whole basis of the comparison. + # + # The populations here are divided by where each agent *pair* landed, not + # by which executor the fault was aimed at, so pinning would buy nothing. + mode: all + durationSecs: 180 + + phases: + # Same as S3: long enough for cold starts and route warm-up to settle, so + # the cut lands on a population that has been cycling steadily. A control + # is only as good as the baseline it is compared against. + baselineSecs: 300 + # Comfortably past the caller's 120s attempt timeout. If executor-to- + # executor traffic did exist, an RPC caught by the cut would stall for a + # full attempt and its retry, and both have to fit inside the window or + # the collapse this scenario hunts would land in the recovery phase. + faultSecs: 180 + # Long enough that a population which held up during the fault and then + # fell over afterwards is distinguishable from one that never moved. + recoverySecs: 420 + + ownership: + # Shards must not move here: nothing in this fault touches the + # shard-manager. Sampled so that a run where they did move says so, since + # a reassignment would mean the two populations are no longer the ones + # that were paired. + settleSecs: 120 + + relay: + # The least share of callers whose callee must be on the other executor. + # + # Neither the driver nor the platform chooses this — both halves of a pair + # are placed by hashing their agent ids, so on two executors it lands near + # 50% but is not guaranteed. 25% is a floor, not an expectation: below it + # the partition has almost nothing to cut and the run aborts with + # FaultTargetUnverified rather than reporting a clean result. + crossPodFloorPercent: 25 + # The assertion. Deliberately higher than any other floor in the suite: + # every other scenario sets one low enough to survive the disruption it + # injects, and here there is nothing to survive. A real drop means an + # executor was talking to an executor. + # + # 90 rather than 100 because throughput is measured against the group's + # own baseline and ordinary cluster jitter moves that a few percent. + crossPodFloorThroughputPercent: 90 + # The same floor for pairs that landed on one executor. They never leave + # the pod, so the partition cannot reach them even in principle — and if + # both populations drop the cause is not the link under test. Kept as a + # separate number so the report can tell those two apart. + coLocatedFloorThroughputPercent: 90 + # The guard against this whole scenario being vacuous, and the one the + # first run showed was needed. + # + # Throughput cannot tell the two populations apart: the driver sets the + # cadence, so both run at the rate they were asked to whether or not a + # call leaves the pod. The first run's cells were 9.51/s and 10.49/s + # before, during and after, which says nothing about where the callees + # were. Latency does say: run 33789281692 measured cross-pod at 151ms p50 + # against co-located at 101ms, a flat 50ms premium in every phase, and + # that premium is the relay hop through worker-service. + # + # So if the two populations cost the same, they are not two populations, + # and every comparison this scenario makes is between two samples of the + # same thing. + # + # 5ms rather than something near the measured 50: this is a floor against + # *zero*, not a model of what the hop should cost, and a cluster with + # faster links should not fail for being fast. + crossPodPremiumFloorMs: 5 + + workload: + # S12's mixed workload plus the stream that makes this scenario mean + # anything. `rpcAgents` is the load-bearing one: every other stream is one + # client call landing on one agent, so no other traffic in the suite ever + # crosses an executor boundary of its own accord. + # + # Each RPC caller implies a second agent, its callee, so 200 here creates + # 400 agents. The caller runs `Counter.increment_through_rpc`, which calls + # `Counter.increment` on `{caller}-inner` — the driver reproduces that + # derivation to decide, before the run, which executor owns each half. + # + # 200 for the same reason S3 drives 200 agents: the verdict is a + # percentage change in each population's throughput, roughly half of + # these land in each population, and a small pool cannot show one. + # + # The durable stream is halved against S1's 200 to keep the total agent + # count in the same range, since each RPC caller brings a callee with it. + durableAgents: 100 + ephemeralAgents: 50 + scheduledAgents: 50 + promiseAgents: 50 + # Zero. Quota traffic crosses the shard-manager link, which this fault + # does not touch, so it would add load without adding signal. + quotaAgents: 0 + rpcAgents: 200 + ratePerSec: 100 + + retryPolicy: + # Identical to every other scenario. Not load-bearing here in the usual + # way — a run where nothing fails has nothing to retry — but a retry is + # exactly what would appear if the architecture claim were false, so + # leaving it on is what makes that visible. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S21 — worker-service CPU saturation under cross-pod RPC (GOL-382). + # + # S2's populations with S2's fault taken away. S2 cut the link between the two + # executors and expected nothing to happen, because an agent-to-agent call the + # local executor does not own is relayed through worker-service. This one + # leaves the link alone and starves that relay of CPU instead. + # + # ── What is actually being measured ────────────────────────────────────────── + # + # Both populations cross worker-service. A cross-pod call crosses it twice + # (driver → worker-service → executor A → worker-service → executor B) and a + # co-located call crosses it once, so the *gap* between them is one + # worker-service hop and nothing else. S2 measured that gap at 38-50ms p50 + # across two runs while looking for evidence its pairing was real. Here it is + # the instrument: a fault that reaches worker-service widens it, and a fault + # that slowed the two populations equally would leave it flat whatever it hit. + # + # This matters because throughput cannot do the job. The driver sets the + # cadence, so both populations run at the rate they were asked to until the + # platform is too slow to keep up at all. S2's cells were 9.51/s and 10.49/s + # before, during and after a partition. Waiting for those to move would mean + # waiting for worker-service to have already fallen over. + # + # ── How the stress is sized ────────────────────────────────────────────────── + # + # A word on the field name first, because this suite uses "worker" for three + # unrelated things. `stressors.cpu.workers` in the manifest is Chaos Mesh's own + # field for the number of **stress-ng CPU burner threads** it starts inside the + # target container. It is not a worker-executor, not a worker-service replica, + # and not the running-workers set several scenarios below talk about. + # + # worker-service on golem-dev requests 200m of CPU and sets no limit, so those + # threads are bounded by `workers × load` rather than by a cgroup quota, and + # the workflow scales the deployment to 3 replicas on 4-vCPU management nodes, + # one replica per node. The ceiling is a probe deadline rather than a core + # count: the liveness probe hits /healthcheck every 15s with a 1s timeout and a + # failure threshold of 3, so roughly 45 seconds of a starved event loop is a + # kill, and the readiness probe pulls a pod out of the Service endpoints in + # about 30. Either turns this scenario into an unplanned worker-service outage + # and answers a different question. A worker-service restart across this window + # is a finding, not noise — see the runbook. + - code: S21 + name: worker-service-cpu-saturation + enabled: true + + fault: + kind: cpu-stress + target: worker-service + # Every replica, by label. Loading one of three would leave two thirds of + # the relay traffic untouched, and the premium is a p50 across all of it. + mode: all + durationSecs: 180 + + phases: + # Same shape as S2, and for the same reason: the premium is compared + # against an undisturbed baseline, so that baseline has to be past cold + # starts and route warm-up. + baselineSecs: 300 + faultSecs: 180 + # Long enough to tell a population that degraded and came back from one + # that degraded and stayed down. That distinction is the run's correctness + # claim, since the fault itself is bounded in time. + recoverySecs: 420 + + ownership: + # Nothing here touches the shard-manager, so shards must not move. If they + # do, the pairing describes agents that have since relocated and the two + # populations are no longer the ones that were measured. + settleSecs: 120 + + relay: + # Identical to S2: the split is decided by hashing agent ids, not by + # anything this scenario arranges, so it needs the same gate against a run + # whose pairs all landed together. + crossPodFloorPercent: 25 + # Read only by the after-heal check under this expectation. Cross-pod is + # *supposed* to fall inside the window, so a during-fault drop is recorded + # as the degradation the run set out to measure rather than raised as a + # finding. What is not allowed is failing to come back: the fault is + # bounded in time, so its effect has to be bounded too. + crossPodFloorThroughputPercent: 70 + # Carried for the shared report shape. Under relay-degraded the co-located + # population is judged against cross-pod rather than against this floor — + # a co-located call crosses worker-service once and a cross-pod call + # crosses it twice, so the shorter path losing more is the finding, at + # whatever absolute level both of them sit. + coLocatedFloorThroughputPercent: 70 + # The same vacuity guard S2 needs, and needed first: if the two + # populations cost the same on the undisturbed baseline they are not two + # populations, and everything below compares a thing to itself. + crossPodPremiumFloorMs: 5 + expectation: relay-degraded + # The only size this scenario judges. Not how much the fault hurt — that + # is what the run is for and the numbers are reported — but whether the + # hurt stopped when the fault did. + # + # Two floors used to sit here asking the fault to degrade by a minimum. + # Both went: 150% on the premium was a guess against a measured 120%, and + # the run that cleared it turned out to have caught a bimodal baseline on + # the right side of its swing. Meanwhile both runs at this rate left + # latency above baseline a full recovery window after the heal and still + # climbing, which is the thing worth failing on. + # + # 110 rather than tighter because the recovery window follows the fault + # immediately and its first minute is legitimately still draining. + recoveryFloorPercent: 110 + + workload: + # S2's populations unchanged, and deliberately so: the placement split is + # decided by hashing agent ids, so keeping the counts keeps the two + # populations the same two populations S2 measured. Only the rate below + # differs. + durableAgents: 100 + ephemeralAgents: 50 + scheduledAgents: 50 + promiseAgents: 50 + # Zero, as in S2. Quota traffic crosses the shard-manager link rather than + # worker-service, so it would add load without adding signal. + quotaAgents: 0 + rpcAgents: 200 + # Eight times the rest of the suite, and knowingly past the 25% cap + # documented on `rate_per_sec`. That cap is a margin for faults that remove + # capacity — S13 kills executors, so a rate comfortable at two has to + # survive one — and this fault removes none. What it does need is a + # worker-service that is actually spending CPU, because a fault that takes + # CPU away from a service that wants none measures scheduling delay rather + # than saturation. + # + # The first run measured what it wants: 0.05 cores per pod at 91 ops/s, + # on 4-vCPU nodes carrying one replica each. Two stressors take 1.8 cores + # and leave 2.2, so worker-service has to want 2.2 before losing them + # registers, which is 44x this rate. Eight times gets it to 0.4 cores and + # leaves the executors at 2.8 of their 8, measured flat at 0.35 through + # the whole first run. Closing the rest of the gap belongs to the + # stressor, not to the workload. + # + # The agent counts do not rise with it. Each agent is delivering 0.2 ops/s + # against a serial ceiling near 8/s, so the rate has roughly 40x of room + # before an agent is the constraint. + ratePerSec: 800 + + retryPolicy: + # Load-bearing here in a way it is not in S2. A starved relay is exactly + # what produces transport failures, and a retry that succeeds is the + # difference between a degraded platform and a lossy one. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S19 — one executor's wall clock moved half a minute behind (GOL-383). + # + # A clock that is uniformly wrong is undetectable from inside the pod it runs + # on: every comparison the pod makes between two of its own readings is + # correct, because the error cancels. Only a comparison between its clock and + # another machine's can go wrong, and on this platform there is exactly one of + # those — the quota lease. The shard-manager mints `expiresAt` on its clock and + # the executor decides against its own clock when to renew and whether the + # lease is already dead. + # + # The scheduler is deliberately *not* the target, though the ticket names it. + # An executor claims due actions `WHERE shardId = ANY(...) AND availableAtMs <= + # now`, and shard sets are disjoint, so two executors can never contend for one + # action however far apart their clocks are. There is no lease to lose and no + # double fire to produce: the owner fires late, by exactly the skew. It is + # still driven here, because the *recovery edge* is where that lateness becomes + # visible — see `skew.rs`. + # + # ── Why a skew run needs more guards than a normal scenario ────────────────── + # + # This is the S2 problem in a sharper form. A run whose injection silently + # failed produces exactly the report a run whose injection landed and did no + # harm produces, and neither has anything missing from it. Two guards close + # that: `skew.toleranceMs` fails a run whose measured offset does not match the + # one asked for, and the driver refuses to inject at all if every quota agent + # landed on one executor — the shard-manager only reclaims a stale lease when + # some *other* pod touches the same resource, so a one-sided population makes + # the fault inert by construction. + - code: S19 + name: executor-clock-skew + # Parked, and it must stay parked until TimeChaos is understood here. The + # skew is applied by ptrace, and every attempt so far left the executor + # *stopped* rather than running on a moved clock, so the scenario has never + # measured the thing it exists to measure. The first attempt also left a + # chaos resource whose finalizer would not clear, which held Flux suspended + # for three hours. + # + # Disabled rather than deleted because the switch is exactly what it is for: + # a full suite pass skips this, and naming `S19` explicitly still runs it, + # which is what working on it needs. See docs/runbooks/cloud-chaos-s19.md in + # golem-cloud. + enabled: false + + fault: + kind: clock-skew + target: worker-executor + # One executor, and the workflow narrows the selector to the pod the + # driver named. `all` would move every clock by the same amount, which is + # a cluster that agrees with itself and disagrees only with the driver — + # no lease disagreement anywhere, and a run that measures nothing. + mode: one + # Must equal phases.faultSecs, for S16's reason: Chaos Mesh recovers a + # TimeChaos on its own `duration` while the workflow holds the phase for + # faultSecs, so a shorter value here leaves the tail of the fault phase + # measuring a cluster whose clock was already correct. + durationSecs: 420 + + phases: + # Long enough for cold starts and route warm-up to settle, and for several + # lease renewals to have gone round undisturbed, so the baseline is a + # steady state rather than a warm-up. + baselineSecs: 300 + # Sized to the renewal cycle rather than to a caller's timeout. On + # golem-dev the lease runs 60s and the executor renews when it believes + # fewer than 20s remain, so a pod 30s behind renews about 10s after its + # lease has actually expired, once every ~70s. 420s gives half a dozen of + # those exposure windows, which is enough for a p50 to mean something. + faultSecs: 420 + # Two things have to finish in here: the quota stream returning to its own + # baseline once the clock is corrected, which costs at most one renewal + # cycle, and the scheduled backlog draining under a corrected clock, which + # is where the skew's cost to scheduled work actually shows. + recoverySecs: 300 + + workload: + # The population the fault can reach. Split by shard ownership across the + # two executors, so ~60 sit on the skewed pod and ~60 are the control — + # and the control half is not decoration: without a second pod renewing + # against the same resource, nothing ever reclaims the stale lease. + quotaAgents: 120 + # Along for the exactly-once oracle. A duplicate execution is the headline + # guarantee the ticket asks about, and the durable stream is the only one + # that can be read back exactly. + durableAgents: 60 + # Zero on purpose, same as S16: only the `scheduled` block's registrations + # carry a token into the target's fire log, and setting both is refused at + # load time. + scheduledAgents: 0 + # Nothing about a promise or an ephemeral agent crosses a clock boundary, + # so both would add operations to the report without adding evidence. + ephemeralAgents: 0 + promiseAgents: 0 + rpcAgents: 0 + # Split evenly between the two active streams, so ~50 quota reservations a + # second. Far below the `chaos-quota` rate limit, deliberately: a refusal + # here has to mean a lease the executor could not renew, not a bucket the + # workload drained itself. + ratePerSec: 100 + + scheduled: + # 60 targets, classified against whichever executor the quota split picked + # rather than choosing it themselves. Ownership is per agent id and these + # are a different agent type, so the executor holding most quota agents + # need not hold most targets — which is fine, and why the placement is + # read separately. + targets: 60 + # 20 registrations a second overall. Lower than S10's 50 because the fire + # log here is read after a 420s fault and a 300s recovery, and every fire + # is an entry the target has to hand back in one invocation. + intervalMillis: 3000 + # Comfortably longer than the skew, so an action registered during the + # fault is still pending when the clock is corrected. That is the whole + # scheduled measurement: the backlog fires with a *corrected* stamp and + # shows a real delay of about the skew, where during the fault the same + # fires read as perfectly punctual because the pod stamps its own late + # fire with its own late clock. + leadSecs: 30 + # What a late fire may cost, and the number the delay percentiles are + # reported against. Sized to the skew rather than to a reassignment: no + # shard moves here, so the only thing an action can be late by is the + # offset plus the scheduler's own poll interval. + leaseBudgetSecs: 60 + + skew: + # Half a minute, backwards. Not a round number: on golem-dev the executor + # renews with 20s of headroom, so any skew smaller than that keeps every + # renewal inside the valid window and the fault is inert. 30s overshoots + # it by 10, so the skewed pod renews 10s after its lease has expired by + # the granting clock, every cycle, for the whole window. + # + # Forwards does nothing at all: a clock that runs fast makes the executor + # renew early, which costs an extra RPC and produces no disagreement. + # Refused at load time. + injectedOffsetMs: -30000 + # What the measured offset may be out by before the run is called one that + # injected nothing. Generous against the number it guards — the probe + # carries its own invocation round trip, which is milliseconds — and far + # enough below 30000 that a clock which never moved cannot pass. + toleranceMs: 5000 + # How far above its own baseline the quota stream's p50 may still sit once + # the clock is corrected. 150 rather than the 110 the relay scenario uses, + # because this p50 is a plain invocation of tens of milliseconds and a + # tenth of that is inside ordinary run-to-run spread. + # + # Reported, not failed: losing a lease under skew is a legitimate response + # and how long getting it back may take is a judgement. + recoveryFloorPercent: 150 + # Readings per group per round, and there are four rounds — one on the + # undisturbed baseline and three through the fault. A median rather than a + # single reading, so one probe that caught a slow invocation cannot decide + # the run. + probesPerRound: 3 + + retryPolicy: + # Identical to the others. The consequence specific to this scenario is + # that a reservation parked behind a lost lease looks like a slow call + # rather than a failed one, so the retry rarely fires — which is itself + # worth knowing, and why the latency cells are the measurement rather than + # the acceptance counts. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S7 — executor pod kill during agent state revert (GOL-371). + # + # The only scenario that disturbs work the platform is trying to *undo*. Each + # agent builds its counter up with a run of increments and then asks for some + # of them back, over and over, while an executor is killed underneath. + # + # It is also the only read-back in the suite with no band of doubt in it. The + # last increment of a round returns the counter's value, so the driver knows + # exactly what the agent was worth before the revert and exactly how many + # invocations it asked to take back. Afterwards there are two legal values and + # nothing between them, which is why S7's findings fail the run outright. + # + # S4 — the shard manager's name stops resolving on one worker-executor + # (GOL-373). + # + # The second control of the set, next to S2, and the one whose expected result + # is hardest to distinguish from a run that measured nothing. + # + # The executor holds one connection to the shard manager, built by + # `GrpcClient::new` with an idle TTL of `Duration::MAX` + # (golem-service-base/src/grpc/client.rs). DNS is consulted when that + # connection is established and never again while it lives, so SERVFAIL + # against a live channel should cost nothing at all. Establishing that is the + # point: "a transient DNS outage costs us nothing" is a claim worth holding + # evidence for rather than reading off the source. MF2 (GOL-537) is the + # companion that takes the connection away first and forces a re-resolution + # while the name is poisoned. + # + # ── What the fault can reach ───────────────────────────────────────────────── + # + # Two things, and neither is what the ticket first assumed. `register` once at + # startup, and quota lease renewal every ten seconds. The executor never calls + # `get_routing_table` — only worker-service does — and ownership and health + # checks are both pushed shard-manager to executor, so the executor's own + # resolver cannot affect either. "Without ownership loss" is right, but because + # a DNS problem here cannot get near ownership, not because ownership survives + # one. + # + # ── What proves the fault landed ───────────────────────────────────────────── + # + # Nothing in this file, and nothing the driver can see. The whole scenario + # expects flat numbers, so an unapplied fault and a correctly applied one + # produce the same result — the shape that cost a window on S2 and again on + # S19. The workflow proves it separately, before the baseline: it puts a + # busybox pod under the same DNSChaos with the same patterns and requires the + # poisoned name to fail and an external name to still resolve. See + # docs/runbooks/cloud-chaos-s4.md in golem-cloud. + # + # ── Infrastructure this needs, which no other scenario does ────────────────── + # + # A Chaos Mesh DNS server, network policy in both directions around it, and a + # target whose root filesystem is writable — chaos-daemon injects by writing a + # backup and a temp file into the container's /etc, so it cannot inject into a + # read-only root at all. The first two are Flux-managed and must be on + # golem-cloud's main before a run; the third the workflow patches on and off + # around the run. All three are in the runbook. + - code: S4 + name: executor-shard-manager-dns-failure + enabled: true + + fault: + kind: dns-failure + target: worker-executor + # One executor, and the workflow narrows the selector to the pod the driver + # named. The other executor is not a spare: the headline number is this + # one's quota latency as a percentage of the other's over the same window, + # so `all` would leave the run with nothing to divide by. + mode: one + # Must equal phases.faultSecs, for S16's reason: Chaos Mesh recovers a + # DNSChaos on its own `duration` while the workflow holds the phase for + # faultSecs, so a shorter value here leaves the tail of the fault phase + # measuring an executor whose resolver was already back. + durationSecs: 300 + + phases: + # Long enough for cold starts and route warm-up to settle, and for several + # lease renewals to have gone round undisturbed, so the baseline the + # recovery reading is taken against is a steady state. + baselineSecs: 300 + # Thirty renewal cycles. Sized to the renewal loop rather than to a + # caller's timeout, because the renewal is the only executor traffic the + # poisoned name is on the path of, and one that never re-resolves has to + # be given many chances not to before "it never re-resolves" is worth + # saying. + faultSecs: 300 + # Shorter than the fault window, unlike most scenarios here. There is no + # backlog to drain and no reassignment to settle — if the fault reached + # nothing there is nothing to come back from, and if it reached the quota + # stream a rebuilt connection is one renewal cycle away. + recoverySecs: 240 + + ownership: + # Shards must not move here: nothing in this fault touches the shard + # manager, and the executor's resolver is on the far side of every path + # that could. Sampled so that a run where they did move says so, since a + # reassignment would mean the two groups are no longer the ones the + # comparison was aimed with. + settleSecs: 120 + + workload: + # The population the fault can reach, split by shard ownership across the + # two executors so roughly 60 sit on the target and 60 are the control. + # The control half is the measurement, not a comfort — see `resolution`. + quotaAgents: 120 + # Along for the exactly-once oracle. A duplicate execution is the headline + # guarantee, and the durable stream is the only one that can be read back + # exactly. + durableAgents: 60 + # Zero on purpose. Nothing about a scheduled action, a promise, an + # ephemeral agent or a cross-pod call crosses the executor's own resolver, + # so each would add operations to the report without adding evidence. + scheduledAgents: 0 + ephemeralAgents: 0 + promiseAgents: 0 + rpcAgents: 0 + # Split evenly between the two active streams, so ~50 quota reservations a + # second. Far below the `chaos-quota` rate limit, deliberately: a refusal + # here has to mean a lease the executor could not renew, not a bucket the + # workload drained itself. + ratePerSec: 100 + + resolution: + # Mirrored from kubernetes/chaos/golem-dev/dnschaos-s4.yaml rather than read + # from it, because the driver never touches Kubernetes and an archived + # result should say what was made unresolvable without the manifest beside + # it. Nothing checks the two agree; the workflow's capability preflight is + # where that is established. + poisonedName: shard-manager.golem-release.svc.cluster.local + # How far above the *control executor's* p99, in the same window, this + # executor's quota p99 may sit before the run records `quota-degraded`. + # + # Against the other executor rather than against its own baseline, because + # a cost here would be concurrent with the fault: both halves run the same + # workload against the same shard manager at the same instant, so the + # difference between them is the fault and not the hour. + # + # A p99 rather than a p50, and MF2 run 2 is why. What this fault does when + # it bites is park a minority of reservations for tens of seconds, which + # barely moves a median: that run read 122% at p50 while the same cells + # carried 124,064% at p99. Comparing medians for a tail phenomenon + # measures the wrong thing. + # + # 300 rather than something tighter because a p99 between two populations + # of sixty agents is not a stable number — S4 run 1 read 121% here on a + # cluster where the fault provably reached nothing, and its baseline + # windows sat around 129%. 300 clears that spread with room and is still + # three orders of magnitude below what a real park produces. + degradationCeilingPercent: 300 + # How far above its own baseline this executor's post-fault quota p50 may + # sit before the run records `quota-did-not-recover`. 150 for S19's reason: + # a tenth of a tens-of-milliseconds p50 is inside ordinary run-to-run + # spread. + # + # Both are recorded rather than failed. A quota stream that ran slower on + # the pod that could not resolve the shard manager is the most interesting + # result S4 could produce, and failing the run on it would bury it under a + # red cross instead of putting it in front of someone. + recoveryFloorPercent: 150 + # Which way the ceiling is read. `survives` means a target group that fell + # behind its control is the finding — something re-resolved a name it was + # not supposed to need. MF2 sets the opposite and is checked against this + # one at load time. + expectation: survives + + retryPolicy: + # Identical to the others. The consequence specific to this scenario is + # that there should be nothing for it to do: the fault is not expected to + # produce a transport error at all, so a retry that fires here is itself + # worth reading. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # MF2 — the same DNS failure, held across a shard-manager restart (GOL-537). + # + # S4 with the thing that makes it null taken away. S4 established that an + # executor rides out a DNS outage because its shard-manager channel is built + # once with an infinite idle TTL and never rebuilt. MF2 restarts the shard + # manager inside the DNS window, so the channel has to be rebuilt and the + # rebuild is the first moment resolution matters. + # + # ── Why the control group is sharper here than anywhere else ───────────────── + # + # The restart is a *shared* shock. Both executors lose their connection to it, + # not just the one under DNS chaos. The only thing that differs between them is + # whether they can resolve `shard-manager...` to get the connection back. So + # the gap between the two groups is the DNS failure with everything else held + # constant — same workload, same shard manager, same instant. + # + # That is also why `expectation: degrades` is load-bearing rather than + # cosmetic. Under it, two executors that came out alike is the finding: it + # means the restart did not force a re-resolution and the run measured S4 + # again under an MF code. + # + # ── What the restart is expected NOT to do ────────────────────────────────── + # + # Move shards. The shard manager rebuilds its assignment from Aurora on + # startup and discovers executors through the Kubernetes API rather than by + # waiting to be registered — `shard-manager-pod-reader` in the golem-release + # RBAC is that grant. Health checks run shard-manager to executor, so the + # target's broken resolver cannot affect them either. A run where shards moved + # is reporting something other than what it injected, and the ownership + # samples on both sides of the restart are there to say so. + # + # ── Infrastructure ────────────────────────────────────────────────────────── + # + # Identical to S4's, and just as absent by default: an operator opens the DNS + # chaos window before the run. See docs/runbooks/cloud-chaos-mf2.md and + # docs/runbooks/chaos-dns-capability.md in golem-cloud. + - code: MF2 + name: executor-dns-failure-across-shard-manager-restart + enabled: true + + fault: + kind: dns-failure + target: worker-executor + mode: one + # Must equal phases.faultSecs, for S16's reason. + durationSecs: 420 + # Named rather than globbed: a composed scenario has two manifests sharing + # its code, so leaving the choice to alphabetical order would decide which + # fault is which. + manifest: dnschaos-mf2.yaml + + composed: + kind: pod-kill + target: shard-manager + manifest: podchaos-mf2.yaml + # A third of the way in, leaving ~280s of overlap. The number that matters + # is what it has to clear, and that is the quota lease, not a timeout. + # + # On golem-dev the lease runs 60s and the executor renews when it believes + # fewer than 20s remain. After the restart drops the connection, the target + # executor's next renewal attempt has to resolve the name and cannot, so + # the lease runs out at most 60s later and its reservations park. 280s is + # four of those cycles: enough that the parking is a steady state rather + # than a single blip, and enough for the p50 to mean something. + afterFraction: 0.33 + # The floor is one full lease plus headroom. Below 90s the overlap may not + # have outlasted a single lease lifetime, and then the run cannot + # distinguish "could not re-resolve" from "had not needed to yet" — which + # is exactly the confusion this scenario exists to remove. + minOverlapSecs: 90 + + phases: + # S4's, so the pair is readable against each other. + baselineSecs: 300 + # Longer than S4's 300. The window has to contain the restart, the + # executor noticing its connection is gone, several failed renewals, the + # lease lapsing, and then long enough after that for the parked + # reservations to be a measurement rather than an edge. + faultSecs: 420 + # Longer than S4's too. There is real work to come back from here: the + # target has to re-resolve, rebuild the channel, re-acquire a lease and + # drain whatever parked behind it. S4's 240 would cut that short and + # report a recovery that had not finished. + recoverySecs: 420 + + ownership: + # Sampled either side of the restart as well as at the end. The shard + # manager going away is the one part of this composition that could move + # shards, and a run where it did is no longer comparing the two + # populations it was aimed with. + settleSecs: 120 + + workload: + # S4's exactly. The pair is only readable if the population is the same. + quotaAgents: 120 + durableAgents: 60 + scheduledAgents: 0 + ephemeralAgents: 0 + promiseAgents: 0 + rpcAgents: 0 + ratePerSec: 100 + + resolution: + poisonedName: shard-manager.golem-release.svc.cluster.local + # The same ceiling as S4, read the other way. Under `degrades` a reading + # at or under 300% is the finding: both executors lost the connection and + # only one could resolve the name to rebuild it, so they should not match. + # + # Kept identical to S4's on purpose. A different number would make the two + # runs incomparable, and comparability is the entire reason MF2 exists as + # a separate scenario rather than as a variant of S4. + # + # Run 2 measured 124,064% here, so the margin either side of this number is + # enormous — which is what a well-chosen threshold for a fault this blunt + # should look like. + degradationCeilingPercent: 300 + # Higher than S4's 150. The target executor here really does have to + # re-acquire a lease after the name comes back, so its post-fault p50 + # legitimately sits above baseline for a cycle or two. 200 is generous + # against a p50 of tens of milliseconds and still catches a stream that + # never came back at all. + recoveryFloorPercent: 200 + expectation: degrades + + retryPolicy: + # Identical to S4's. Unlike S4 the retry is expected to fire here: a + # reservation parked behind a lease the executor cannot renew will time + # out, and one same-key retry after 5s is the bounded response. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # Like S8, S10, S11 and S3 the driver names the pod and keeps driving the + # agents on the other executor as a control group. + - code: S7 + name: executor-crash-during-revert + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts to settle, so the kill lands on a population + # that has been cycling rounds steadily rather than one still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can revert an agent the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that every agent runs several + # whole rounds after the kill, so a round disturbed by it is followed by + # ones that were not. + recoverySecs: 300 + + revert: + # 200 agents, split by shard ownership across the two executors. Also the + # resolution of the report: a torn revert localises to one agent out of + # two hundred. + agents: 200 + # Four increments then a revert of two, so a completed round is worth +2. + # + # `revertInvocations` must not exceed `incrementsPerRound`: a revert that + # reached further back would land in the region an earlier revert already + # deleted, and the platform refuses that outright — every round would fail + # and the run would measure nothing. The driver checks this before + # starting rather than discovering it per round. + incrementsPerRound: 4 + revertInvocations: 2 + # Between rounds. With five operations per round this keeps roughly one + # agent in six mid-round at any instant, which is the population a kill + # has something to land in. It also has to leave the round long enough + # that the workflow's inject-and-verify path — signal poll (5s) plus + # `kubectl apply` plus waiting for `AllInjected` — does not fall entirely + # between two rounds. + intervalMillis: 500 + # What recovering a reverted agent may cost. Recorded, not asserted: the + # floor is a shard reassignment plus the worker recovery that replays the + # oplog up to the revert, and how much more than that is acceptable is a + # judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The revert is never retried**, and + # that exception is load-bearing rather than an oversight: a revert has no + # idempotency key and is not idempotent — asking twice for "the last two + # invocations" takes back four. A retried revert that had actually landed + # would look exactly like the platform tearing a truncation, which is the + # finding this scenario exists to make. See `chaos/reverts.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S6 — executor pod kill during agent deletion (GOL-372). + # + # S7 asks the platform to forget some of an agent's work. S6 asks it to forget + # the agent, and kills the executor while it is doing so. + # + # The same two-value oracle, one step further: invoking a deleted id creates a + # *new* agent, so when a slot is used again it must report either 1 (the + # deletion took) or its old value plus one (it did not). Neither is a defect + # alone — a lost response leaves the question open — but "the platform + # confirmed the deletion and the agent is still worth what it was" is the + # resurrection this scenario is named for, and it fails the run. + # + # Worth knowing before reading a result: the happy path is already defended. + # `Worker::start_deleting` stops the background status flush and checkpointer + # first, specifically so neither can resurrect the cached status after the + # durable removal. The question is whether that defence survives the pod dying + # between the mark and the removal. + - code: S6 + name: executor-crash-during-deletion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + baselineSecs: 180 + faultSecs: 120 + # Rounds continue throughout, so every slot is reused several times after + # the kill and a round it disturbed is followed by ones it did not. + recoverySecs: 300 + + delete: + # 200 slots, split by shard ownership. Also the resolution of the report: + # a resurrection localises to one slot out of two hundred. + agents: 200 + # Three increments before each delete, so an agent is worth 3 when it + # goes. More than one is load-bearing rather than arbitrary: with a single + # increment a fresh agent and a resurrected one both read 1 and the oracle + # cannot tell them apart. The driver refuses to start below 2. + incrementsPerRound: 3 + # Between rounds. A round is four operations, so this keeps the delete a + # meaningful share of each slot's cycle and gives the kill something to + # land in. + intervalMillis: 500 + # What recovering a slot may cost once its executor is back. Recorded, not + # asserted. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The delete is never retried**, for a + # reason specific to this scenario: `delete_worker_internal` starts with a + # metadata lookup and returns `worker_not_found` when there is nothing + # there, so deleting twice does not succeed twice. A delete whose response + # was lost but which landed would come back on retry as a refusal, and the + # run would record "refused, and gone anyway" — one of the violations this + # scenario exists to detect. See `chaos/deletions.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S9 — executor pod kill during a component rollback (GOL-369). + # + # S5 moves agents forward onto a new build and kills an executor while that is + # happening. S9 moves them forward, waits for that to land, and then moves them + # back — killing an executor during the return leg. + # + # The return leg is the one that matters operationally. A rollback is what you + # reach for when the new build is already going wrong, so a rollback happening + # under a dying executor is the situation you would actually be in. + # + # The rollback is a redeploy: `agent-counters` is uploaded again as a new + # revision and every agent is asked to move to it. `Counter::component_version` + # is compiled into each build, so an agent that has genuinely returned reports + # 1 from the code that is running rather than from metadata about what the + # platform believes. + - code: S9 + name: executor-crash-during-rollback + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # `one` plus the namespace and opt-in label selectors in podchaos-s9.yaml + # bound the blast radius. Unlike S8 the driver does not name the pod: + # agents are mid-rollback across both executors, so either one interrupts + # returns in flight and which one carries no information. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the roll + # forward is applied to a population that is running rather than arriving. + baselineSecs: 300 + # Has to cover the kill, the reschedule, and the rollback finishing on + # both the surviving executor and the replacement. + faultSecs: 120 + # Returns continue after the kill and agents have to converge before the + # census decides which build they are on. + recoverySecs: 300 + + workload: + # Durable agents are the population being rolled back, so the pool is + # large: a stuck agent localises to one out of 200. + durableAgents: 200 + # Not rolled back — they exist to show whether acceptance degraded during + # the kill, separately from the rollback. + ephemeralAgents: 50 + # Durable state the rollback does not touch, so they are the control: + # their read-back should be unaffected. + scheduledAgents: 50 + promiseAgents: 0 + # No quota stream, for the same reason as S5: a lease that cannot be + # renewed would add a second failure mode to a scenario that has one. + quotaAgents: 0 + ratePerSec: 100 + + rollback: + # How long to let the roll-forward land before measuring it. Too short and + # the census reads a population still in transit, and the gate below + # refuses a rollback that would have been fine. + settleSecs: 90 + # The share of agents that must actually report the new build before the + # rollback is attempted at all. Below it the driver stops: rolling agents + # back to a build they never left would pass every check and prove + # nothing, which is the worst artifact this suite can produce. + rolledForwardFloorPercent: 90 + # Retries for the rollback *control plane*, counted apart from the + # workload's own. Different question: a request refused because its + # agent's executor just died says nothing about correctness, but an agent + # nobody successfully asked to come back explains a stale agent later + # without excusing one. + controlRetries: 2 + controlRetryDelaySecs: 5 + # How far into the rollback to ask for the kill. As in S5 this is when the + # driver *asks*; applying the PodChaos and confirming it takes a few + # seconds more, and the result records both so how far in it landed is + # readable rather than assumed. + killDelaySecs: 2 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S16 — key-value PostgreSQL outage recovery (GOL-379). + # + # The first fault in the suite aimed at something the platform depends on + # rather than at the platform itself. The executors keep running, keep their + # shards and keep answering the shard-manager. What they lose is the Aurora + # cluster underneath them, which on golem-dev carries promises, the + # running-workers set, user key-value data and the scheduler's own schema. + # + # The oplog is on a different cluster and the worker-status hot cache is in + # Redis, and neither is touched. That is the interesting part: the platform + # keeps the ability to record what it did and loses the ability to know what + # it is doing. + # + # There is no control group of executors, because all three share the cluster. + # The control is the baseline window, and the first thing the report has to + # establish is that throughput collapsed at all — a partition that silently + # failed to take hold would otherwise produce a clean-looking account of an + # undisturbed cluster. + - code: S16 + name: keyvalue-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, unlike every other partition in this suite. A storage + # outage that only some executors could see would be a routing fault + # wearing a database's clothes, and S3 already covers that shape. + mode: all + # Must equal phases.faultSecs. Chaos Mesh recovers a NetworkChaos on its + # own `duration`, while the workflow holds the phase for faultSecs and + # only then deletes the object. Setting this lower lifts the partition + # early and leaves the tail of the fault phase measuring a healed + # cluster, which is how the first S16 run came to report 16.87% of + # baseline throughput during a total outage. + durationSecs: 60 + + phases: + # Long enough for cold starts, component loading and route warm-up to + # settle, and for the scheduler to be firing steadily, so the cut lands on + # a cluster in steady state. + baselineSecs: 300 + # Sized to the outage this scenario models rather than to the caller's + # timeout: the worst case an AWS storage failover imposes, which is an + # Aurora reader promotion of about a minute and an ElastiCache Multi-AZ + # promotion of about thirty seconds. The key-value retry budget is built + # to cover exactly that, so an outage of this length is one the platform + # is expected to absorb without losing an executor. + # + # Deliberately shorter than S3's 180s, and not for the reason S3 gives. + # S3 needs the caller's 120s attempt timeout to fire inside the window + # because pending-then-timeout is the behaviour it exists to show. Here + # the opposite is the claim: invocations issued during the cut should + # still be waiting when the link returns and then complete. Throughput is + # attributed by completion, so the fault window still reads as a collapse + # and the recovery window as the catch-up, which is what says the + # partition took hold. + # + # Do not raise this past the retry budget without meaning to. The budget + # is 15 attempts, ~93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so roughly 168s. Beyond that the + # recovery-index write in `status_flusher` exhausts and panics by design, + # and the scenario stops measuring absorption and starts measuring the + # crash it is supposed to have removed. + faultSecs: 60 + # Every stream has to get several whole operations after the heal, so a + # stream that recovered slowly is distinguishable from one that never + # did. Scheduled actions registered during the outage also have to fall + # due and fire inside the run rather than after it. + recoverySecs: 420 + + workload: + # The population read back agent by agent, so a lost increment localises + # to one agent out of 200. + durableAgents: 200 + # No durable state of their own, but they still need the running-workers + # set to start, so they say whether acceptance degraded separately from + # whether state survived. + ephemeralAgents: 50 + # Zero on purpose. The scheduled stream is driven by the `scheduled` block + # below instead, because only its registrations carry a token into the + # target's fire log and scheduler lag cannot be measured without one. + # Setting both is refused at load time. + scheduledAgents: 0 + # Promises live in the cluster being taken away, so this stream is + # directly on the fault path rather than incidental to it. + promiseAgents: 50 + # No quota stream: a lease renewal failing would add a shard-manager + # failure mode to a scenario that is about storage. + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + # 100 targets, each with its own emitter, same as S10. Also the resolution + # of the report: a lost action localises to one target out of a hundred. + targets: 100 + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant, so the outage + # begins with a large population of work the platform has promised to do + # and cannot currently read. + leadSecs: 10 + # What recovering a pending action may cost. Larger than S10's 60s and + # deliberately so: there is no shard reassignment here, but the scheduler + # cannot claim or acknowledge anything for the whole 180s outage, so the + # floor is the outage itself. 240s covers the outage plus a lease TTL and + # some catch-up without being so generous that a regression hides inside + # it. + # + # Recorded, not asserted, like every other budget in the suite. + leaseBudgetSecs: 240 + + storage: + # The Aurora writer endpoint the workflow names in the NetworkChaos + # manifest's `externalTargets`. Recorded here so the archived result says + # which storage the run was about; the driver never resolves or contacts + # it. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # The least of the fault window every stream must answer nothing at all + # for, as a share of that window, for the outage to count as observed. + # + # This was a ceiling on during-fault throughput as a share of baseline. + # That number is still reported, but it cannot be the verdict: it is a + # rate averaged over the whole window, while an absorbed outage does all + # its serving in the few seconds at the window's edges. The same handful + # of edge confirmations reads as ~8% of a 180s window and ~26% of a 60s + # one, so the threshold tracked the window length rather than the + # platform, and shortening this window to 60s duly tripped it on a + # partition that had plainly landed (run 33099228775: quiet 58.0s of + # 62.3s on every stream, yet 25.79% against a 15% ceiling). + # + # Quiet time is measured against the window's own edges, so it means the + # same thing whatever the window is. 50% is far from both outcomes it has + # to tell apart: a landed outage is quiet for over 90% of the window, an + # undisturbed cluster for a few percent. + expect: + # Everything is behind this cut, so every stream that was serving is + # expected to stop. See `OutageExpectation` for why the rule is named + # per scenario rather than shared: S18 cuts a store only part of the + # workload touches, and this rule reads that arrangement backwards. + kind: whole-workload + quietFloorPercent: 50 + # What serving again may cost once the cluster is reachable. Recorded, not + # asserted: this is really a measure of how long sqlx's pool takes to + # notice, plus whatever recovery scan the executors run, and how long that + # may be is a judgement. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S16: the retry is what turns a single stalled + # attempt into a second one under the same key. If the database comes back + # in between, that retry lands against work the first attempt may already + # have half-done, which is exactly what the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S22 — the long-outage companion to S16 (GOL-499). Same cut, same driver, + # held past the point where the platform can absorb it. + # + # S16 asks whether golem rides out a storage failover. This asks what happens + # when the storage does not come back in time. The key-value retry budget is + # 15 attempts, roughly 93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so about 168s. Past that the recovery-index + # write in `status_flusher` panics on purpose, on the grounds that an executor + # which cannot maintain its own running-workers index should be replaced + # rather than left running with agents it can no longer account for. + # + # So executor restarts are the *expected* outcome here, not a defect, and + # nothing in this entry treats them as one. What the run has to establish is + # that the exit is the intended one and that nothing is lost across it: the + # scheduled-fire account and the exactly-once account are the same assertions + # S16 makes, and they are the ones that fail the build. + - code: S22 + name: keyvalue-postgres-outage-past-budget + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16. A budget exhausted on only some of them would + # leave the survivors serving and make the replay cost unreadable. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as S16: long enough for cold starts, component loading and route + # warm-up to settle, and for the scheduler to be firing steadily. + baselineSecs: 300 + # Comfortably past the ~168s budget rather than just over it. A window + # that only just exceeds the budget would make the run's outcome depend + # on which second an operation happened to start in, so a rerun could + # come back with restarts or without them for no reason worth reporting. + # At 300s every tracked transition in the window exhausts. + faultSecs: 300 + # Longer than S16's 420s, and for a reason S16 does not have: the + # executors are replaced here, so recovery is not just a pool noticing + # its database came back. It is pods rescheduling, rejoining the routing + # table, and every agent that was on them replaying from oplog or + # snapshot. Scheduled actions registered during the outage also have to + # fall due and fire inside the run rather than after it. + recoverySecs: 600 + + workload: + # Identical to S16 on purpose. The two runs are meant to be read side by + # side, and a different population would make the difference between them + # a question about the workload rather than about the outage length. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Wider than S16's 240s. The lease has to outlast the outage plus the + # replay behind it, or actions would be judged late for the recovery this + # scenario exists to measure. + leaseBudgetSecs: 480 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same rule and floor as S16, and it should be met even more comfortably: + # an outage this long leaves the streams silent for nearly the whole + # window. + expect: + kind: whole-workload + quietFloorPercent: 50 + # Recorded, not asserted, as everywhere else — but read differently here. + # In S16 this measures a connection pool noticing. Here it measures a + # cluster rebuilding itself, and it is the number that says whether the + # deliberate crash is an acceptable trade. + recoveryBudgetSecs: 300 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # S14 — the oplog half of the storage matrix (GOL-376). Same driver as S16 and + # S22, same cut, aimed at the other Aurora cluster. + # + # S16 takes away the cluster that says what the platform is doing: promises, + # the running-workers set, user key-value data and the scheduler's schema. S14 + # takes away the one that says what it did. The oplog is the only thing on the + # indexed cluster, so under this cut promises still resolve, the scheduler + # still claims and acknowledges on time, and the running-workers set is still + # writable — and nothing durable can be committed. The two scenarios are + # near-mirror images and are meant to be read side by side. + # + # The retry budget is not the interesting quantity here the way it is in S16 + # and S22. golem-dev gives indexed storage 200 attempts with a 10s cap, which + # is over half an hour of stalling, so this window cannot exhaust it. What is + # worth watching instead is the error classification: `retry_storage_op` + # panics immediately on any indexed-storage error it was not handed as + # transient, and transient means `PoolTimedOut` or `Io` and nothing else. An + # executor lost during this run is a misclassification finding, not an + # exhaustion one. + - code: S14 + name: indexed-oplog-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16 and for the same reason: a storage outage only + # some executors could see would be a routing fault wearing a database's + # clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 60 + + phases: + # Same as S16, so the two are comparable: long enough for cold starts, + # component loading and route warm-up to settle, and for the scheduler to + # be firing steadily. + baselineSecs: 300 + # The same worst-case Aurora writer promotion S16 is sized to, about a + # minute. GOL-376 was written with 30s, before that number had been + # worked out; 60s is the failover this is supposed to model and matching + # S16 is what makes the two results readable against each other. + faultSecs: 60 + # As S16: every stream needs several whole operations after the heal, and + # scheduled actions registered during the outage have to fall due and fire + # inside the run. + recoverySecs: 420 + + workload: + # Identical to S16, deliberately. The comparison between the two runs is + # the point, and a different population would turn the difference into a + # question about the workload. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as S16: the scheduled stream comes from the + # block below, because only its registrations carry a token into the + # target's fire log. Setting both is refused at load time. + scheduledAgents: 0 + # Promises live on the cluster this cut leaves alone, so unlike in S16 + # this stream is not directly on the fault path. It still degrades, + # because completing a promise wakes an agent that then cannot commit, + # and that gap between "the promise was written" and "the agent got + # anywhere" is worth having on the record. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Same as S16's, and it should be met with room to spare. The scheduler is + # outside this cut, so claims and acknowledgements keep their timing and + # the delay reduces to how long the fired invocation could not commit. + # That makes S14 the cleanest read of scheduler lag in the suite: in S16 + # and S22 the scheduler is inside the outage and the two costs cannot be + # separated. + leaseBudgetSecs: 240 + + storage: + # The indexed writer endpoint, not the key-value one. This is the only + # field that differs materially from S16. + endpoint: golem-postgres-dev-indexed.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same rule, floor and reasoning as S16. + expect: + kind: whole-workload + quietFloorPercent: 50 + # Recorded, not asserted. Expected to be small: nothing is replaced here, + # so this is a connection pool noticing its database is back plus whatever + # the stalled commits still have to do. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S18 — Redis outage (GOL-384) + # + # The fourth scenario on the storage driver and the first that does not cut a + # database. golem-dev runs the key-value layer as `NamespaceRouted`: the + # `Worker`, `AgentStatus` and `AgentStatusCheckpoint` namespaces go to a Redis + # cache and everything else goes to Postgres. S16 cut the Postgres half and + # left this one standing; S18 cuts exactly what S16 left. + # + # It is sized differently from the other three, and the reason is in the + # config rather than in the ambition. Those three are measured against a + # platform retry budget — S16 and S14 stay inside one, S22 runs past one. This + # client has no budget to run past: golem-dev sets the cache's + # RETRIES__MAX_ATTEMPTS to 0, which the code turns into a fred reconnect + # policy with unlimited attempts, and fred's defaults then give the client no + # command timeout and an unbounded command buffer. A cache write during the + # cut does not fail. It waits. + # + # So the only clock that can run out is the caller's, and the window is set + # against that instead: past the 120s attempt timeout, so operations time out + # and retry under the same idempotency key while the original write is still + # queued inside fred. Two landings of one operation is what this run is for. + # --------------------------------------------------------------------------- + - code: S18 + name: redis-cache-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in the other three: a cache outage only some + # executors could see would be a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 180 + + phases: + # Same as the other three, so all four are comparable. + baselineSecs: 300 + # Past the workload's 120s ATTEMPT_TIMEOUT plus its 5s retry delay, so an + # operation that starts early in the window times out, retries, and has + # that retry still in flight when the heal arrives. At 60s nothing would + # time out and the run would only show slower operations completing, which + # the config already tells us without spending a run. + faultSecs: 180 + # As the others. The drain here is larger than S14's — three minutes of + # coalesced status writes and queued commands come back at once — so this + # needs to cover a burst rather than a trickle. + recoverySecs: 420 + + workload: + # Identical to S16 and S14, deliberately, so the difference between the + # four results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + # Expected to be one of the two streams that actually stops. Completing a + # promise suspends and wakes an agent, and a suspend is a lifecycle + # boundary, which is one of the few things that still crosses this cache + # synchronously. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Raised from the 240 the other three use, because the window itself is + # 180s. A fire at the start of the outage is delayed by roughly the whole + # window before it can make progress, and then joins a drain queue behind + # three minutes of backlog. 300 leaves headroom for that tail without + # making a delay the platform should be ashamed of look acceptable. + leaseBudgetSecs: 300 + + storage: + # The Redis primary endpoint, not either Aurora writer. Both Postgres + # clusters stay reachable for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + # The one scenario in this family that does not use the whole-workload + # rule, because this cut is partial by construction and that rule reads a + # partial cut backwards. Run 33130077355 proved it: `durable` held 100% + # of its baseline throughout, exactly as NamespaceRoutedKeyValueStorage + # says it should, and the shared rule reported the partition as one that + # never landed. + # + # `ephemeral` is the only stream named. Its agents are created and torn + # down per operation, so every one of them crosses a lifecycle boundary, + # which is one of the few things that still reaches this cache + # synchronously. It went silent for 99.997% of that first window. + # + # `promise` was expected to be the second and was not: the mixed + # workload's promise stream is get_promise+complete in one round trip + # against a durable agent and never suspends. The suspending variant is + # `promise-wait`, which S11 drives and this scenario does not enable. + # + # The serving floor is the half a shared rule cannot state. Durable, + # scheduled and promise are claimed to be off this fault path because of + # where the code routes their namespaces, and that claim is worth + # asserting: a stall there would mean the status blob is not off the + # commit path the way AgentStatusFlusher describes. All three held + # 99.94-100.06% on the first run, so 50 catches a real regression without + # tripping on noise. + expect: + kind: partial-workload + silenced: [ephemeral] + quietFloorPercent: 50 + servingFloorPercent: 50 + # Recorded, not asserted. Expected to be larger than S14's: nothing is + # replaced, but three minutes of coalesced status writes and buffered + # commands all arrive at once when the connection returns. + recoveryBudgetSecs: 180 + + retryPolicy: + # Identical to the other three, and load-bearing here rather than + # incidental. Retrying transport errors once under the same idempotency + # key is the whole mechanism by which a stalled operation and its retry + # can both land, which is the question the run exists to answer. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S17 — Redis latency degradation (GOL-375) + # + # The fifth scenario on the storage driver, and the first that does not break + # anything. S18 took the Redis cache away; this leaves it reachable and makes + # it slow. That is a different question rather than a milder version of the + # same one: S18 asked whether the platform survives losing its worker-status + # store, and this asks whether it degrades or breaks when that store gets + # slower. "It went quiet" would be a failure here, not the expected result. + # + # GOL-375 asks for three things S18 could not give it — invocation latency + # under load, RSS peak, and pending-work evidence — and for a determination of + # whether worker-status writes block invocation completion or are best effort. + # S18 answered half of that: best effort for a resident durable agent, because + # AgentStatusFlusher takes the blob off the commit path, and blocking for + # anything crossing a lifecycle boundary. This run gets the other half, which + # is what that costs when the store is merely slow rather than gone. + # --------------------------------------------------------------------------- + - code: S17 + name: redis-cache-latency + enabled: true + + fault: + # Recorded for the archive rather than acted on: the workflow selects the + # manifest by scenario code and reads the Chaos Mesh kind out of the + # manifest itself. It compares this string only against `none`, + # `network-partition` and `pod-kill`, none of which this is. + kind: network-delay + target: worker-executor + # Every executor, as in every other storage scenario. A cache that is slow + # for only some executors is a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as every other storage scenario, so they stay comparable. + baselineSecs: 300 + # Longer than any of the others, and not for the reason S18's window is + # long. Nothing times out under a 500ms delay — an ephemeral operation + # goes from about 212ms to under a second, nowhere near the 120s attempt + # timeout — so there is no caller clock to outrun. The length is for the + # two things GOL-375 asks for that need time to develop: a memory series + # long enough to show whether the executor's dirty-status set grows, and + # a steady state rather than a transient. At the workflow's 15s sampling + # interval this is about 20 memory samples inside the window. + faultSecs: 300 + # As the others. + recoverySecs: 420 + + workload: + # Identical to S14, S16, S18 and S22, so the difference between the five + # results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # The scheduler keeps its own schema on Postgres, which this fault does + # not touch, so its claims and acknowledgements should be unaffected. S18 + # confirmed that: during-fault p99 was 2094ms against a 2099ms baseline. + # 240 as the other three, and it should not be approached. + leaseBudgetSecs: 240 + + storage: + # The same Redis endpoint S18 cuts. Both Aurora clusters stay reachable + # and unslowed for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + + expect: + # Neither of the rules a cut uses. Under a delay nothing is supposed to + # fall silent, so silence is not available as evidence that the fault + # landed and time has to stand in for it. + kind: latency-degradation + + # `ephemeral` alone, for the reason S18 measured rather than predicted: + # its agents are created and torn down per operation, so every one + # crosses a lifecycle boundary, and that is one of the few things still + # reaching this cache synchronously. Under S18 it was the only stream + # that stopped. + slowed: [ephemeral] + + # Ephemeral's baseline median was 212ms in S18's run. The delay is 500ms + # applied to egress only (`direction: to`, which Chaos Mesh's webhook + # requires for netem against an external target), so one Redis round + # trip inside the operation puts it at about 712ms, or 3.4x. Two put it + # near 5.7x. + # + # 2.0 sits below the single-round-trip case with room to spare while + # still failing the run that matters: a netem rule that never applied + # leaves the stream at roughly 1.0x, and the report would otherwise be + # full of healthy numbers describing an experiment that never happened. + slowdownFloor: 2.0 + + # The other three, and the claim that makes the 6.59x above attributable + # to this cache rather than to a slower cluster. Their agents are + # resident and durable, so `AgentStatusFlusher` keeps the status blob off + # the commit path and nothing they do crosses Redis synchronously — and + # the first run bore that out exactly, at 61ms, 100ms and 91ms in all + # three windows. S15 makes the mirror claim with the two lists swapped. + steady: [durable, promise, scheduled] + + # Room for the spill of sharing executors and a runtime with a stream + # that is six times slower than usual, and nothing like the multiple a + # real dependency on this cache would produce. Measured at 1.00x. + steadyCeiling: 1.5 + + # A delay should cost time, not work. Throughput is not expected to move + # much: the per-stream in-flight budget is MAX_IN_FLIGHT/4 = 256, and at + # 33 operations per second and 712ms each the stream sits around 23 in + # flight, so nothing queues. A stream falling below half its baseline + # therefore means the added latency broke something rather than slowed + # it — a timeout fired, a pool drained — and that is worth failing on. + servingFloorPercent: 50 + + # Recorded, not asserted. Expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for and no backlog to + # drain, only the last few in-flight operations finishing at their slower + # pace. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to every other storage scenario. Not load-bearing here the + # way it is in S18, because nothing should time out and so nothing should + # retry — but a retry count above zero would itself be a finding, and + # leaving the policy the same is what makes that comparable. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15 — key-value PostgreSQL latency degradation (GOL-374) + # + # S17 with the two halves of the key-value layer swapped. That fault slows the + # Redis cache and this one slows the Aurora cluster behind it, and because + # `NamespaceRoutedKeyValueStorage` splits the namespaces between them, the two + # runs should move opposite sets of streams. Reading them side by side is the + # point: each is the other's control. + # + # ── Which streams actually reach this cluster ────────────────────────────── + # + # Worth spelling out, because "the workload is promise- and schedule-heavy" + # undersells it and the durable stream is the surprise. + # + # * durable — `AgentStatusFlusher::on_status_changed` updates the + # `RunningWorkers` recovery index *synchronously on the hot path*, and + # only when the tracking predicate flips. An idle agent invoked once goes + # Idle → Running → Idle, so the predicate flips twice and the invocation + # waits on two writes to this cluster. The status blob itself is off the + # commit path; the recovery index deliberately is not. + # * promise — `create` is one `set_if_not_exists` and `complete` is an + # `exists` followed by another, all in the `Promise` namespace, which + # routes here. + # * scheduled — the emitter's `schedule_poll_at` registers into the + # scheduler's own schema, which lives on this same cluster under + # `GOLEM__SCHEDULER_STORAGE__CONFIG__HOST`. + # * ephemeral — reaches none of it. `on_status_changed` returns immediately + # for an ephemeral agent, and its status goes to the Redis cache, so this + # is the one stream the delay should not touch. Under S17 it was the only + # one that did. + # + # ── What the first run measured, which was not what it predicted ────────── + # + # Run 33210769609. The prediction here was 17x on durable and promise and 6x + # on scheduled, from counting round trips, with a note that the connection + # pools would fit "neither by much". The measurement was 644x, 939x and + # 1640x: durable went from 61ms to 39s, promise from 92ms to 86s, scheduled + # from 62ms to 102s. A 500ms delay came out three orders of magnitude larger + # at the caller. + # + # So this is not a degradation scenario in practice. Counting round trips + # gives the cost of one operation running alone; it says nothing about what + # happens when every operation is doing the same thing to a pool of 64 + # connections, and queueing is the whole answer. Nothing was lost — every key + # exactly once, every schedule fired once, 300/300 agents consistent, every + # stream serving again within 4s — but the platform did not stay usable. + # + # Two predictions that were wrong in an informative direction: + # + # * No executor was lost. `PoolTimedOut` never reached the panic, so the + # S16 finding was not reproduced from latency alone. The queue built in + # front of the pool rather than inside its acquire timeout. + # * `ephemeral`, which reaches none of this cluster, ran 235x slower. 46 + # seconds is ninety 500ms round trips and the stream makes none, so it was + # queueing behind the three streams that do. Whatever gate that is, it is + # not the per-account concurrency semaphore — golem-dev sets + # `MAX_CONCURRENT_AGENTS_PER_EXECUTOR` to the unlimited sentinel, so that + # path is bypassed. See the runbook. + # --------------------------------------------------------------------------- + - code: S15 + name: keyvalue-postgres-latency + enabled: true + + fault: + # As S17. Recorded for the archive rather than acted on: the workflow + # picks the manifest by scenario code and reads the Chaos Mesh kind out of + # the manifest itself. + kind: network-delay + target: worker-executor + # Every executor. A store that is slow for only some of them is a routing + # fault wearing a database's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # S17's, unchanged, so the two delays differ only in what they aim at. + baselineSecs: 300 + # 300s for S17's reasons and one more. Nothing here should approach the + # 120s attempt timeout, so there is no caller clock to outrun; the length + # is for a memory series long enough to have a slope and for a steady + # state rather than a transient. The one more is the pool arithmetic + # above: a pool that is going to saturate does it by filling, and filling + # takes time that a 60s window does not give it. + faultSecs: 300 + recoverySecs: 420 + + workload: + # Identical to the other full-workload storage scenarios, deliberately, + # even though GOL-374 asks for "promise- and schedule-heavy" work and a + # reweighted population would + # press the pools harder. Two of the storage runs are a matched pair + # and comparability is what makes the pair worth anything; and the durable + # stream turns out to be the heaviest user of this cluster anyway, at two + # synchronous writes per invocation. Deliberately overloading the pool is + # a different experiment — S22 is to S16 what that would be to this — and + # it should be its own scenario rather than a thumb on this one's scale. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # As the others. Unlike S17 the scheduler is *inside* this delay, so its + # claims and acknowledgements each gain a round trip — but 240s of lease + # against a claim that costs milliseconds plus 500ms is not a number this + # run can approach, and a lease expiry here would be a real finding rather + # than a threshold that was set too tight. + leaseBudgetSecs: 240 + + storage: + # The key-value Aurora cluster, which is also where the scheduler keeps + # its schema. The same endpoint S16 and S22 cut. Neither Redis nor the + # indexed-oplog cluster is touched: slowing a second store would leave + # every finding unattributable. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + # As S17. Under a delay nothing is supposed to fall silent, so silence + # is not available as evidence that the fault landed and time has to + # stand in for it. + kind: latency-degradation + + # The three streams that reach this cluster synchronously, per the + # walk-through above. Measured at 644x, 939x and 1640x — see the note + # above for why the round-trip arithmetic underestimated that by two + # orders of magnitude. + slowed: [durable, promise, scheduled] + + # 2.0, as S17, and for the same reason rather than for the arithmetic + # above: the floor exists to fail a run where the netem rule never + # applied, which leaves every stream near 1.0x. Setting it near the + # predicted 6x would instead be a second, unannounced assertion that the + # prediction is right, and the prediction is the thing under test. + slowdownFloor: 2.0 + + # The control, and the mirror image of S17's list. An ephemeral agent + # never enters the recovery index and its status goes to Redis, so if + # this stream slows with the others, the delay reached something wider + # than this cluster and the run cannot say what it measured. + steady: [ephemeral] + + # As S17. Ephemeral is the noisiest stream in the suite at rest — its + # p99 runs to seven times its median — but the check is on medians, and + # the median has held at 196-212ms across every storage run so far. + # + # The first run broke this at 235x, and deliberately left as it is: the + # ceiling did its job. Raising it to accommodate the result would turn + # the one number that caught the coupling into a number that describes + # it. + steadyCeiling: 1.5 + + # A delay should cost time, not work. Held at 50 for comparability, but + # the first run showed this floor cannot be read on its own once the + # in-flight budget saturates: throughput is concurrency over latency, so + # a stream 900x slower cannot hold its rate however healthy it is. + # Promise came in at 11.9% of baseline while carrying a hundred times + # more work in flight than at rest. `outage.rs` now suppresses the + # finding when the stream's own slowdown accounts for the shortfall, + # which is the only case where the two floors were measuring one fact. + servingFloorPercent: 50 + + # Recorded, not asserted, and expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for. If an executor was + # lost to the pool timeout this will be the number that shows it, as the + # replacement has to reload its agents before it can serve. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to every other storage scenario. Nothing should time out and + # so nothing should retry; a retry count above zero is itself a finding. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15A / S15B / S15C — the S15 elimination series + # + # S15's first run left one thing unexplained. `ephemeral` slowed 235x under a + # delay aimed at a cluster it never touches, and the search for a shared gate + # inside the executor came up empty: worker-service was exonerated by the + # traces, agent creation ran in single-digit milliseconds, memory and + # filesystem pools had headroom, the per-account concurrency semaphore is + # bypassed on golem-dev, the tokio global queue was flat at zero with the busy + # ratio *falling*, and neither the oplog cluster nor Redis moved. Everything + # was parked, nothing was competing, and no mechanism in the source accounts + # for it. + # + # So the next step is elimination rather than more forensics: hold the fault + # and the ephemeral stream fixed, and add one stream at a time until ephemeral + # starts to suffer. Whichever addition does it names the interaction, and the + # code reading has somewhere to start. + # + # S15A ephemeral the control: nothing should move + # S15B ephemeral + durable durable is the heaviest user + # S15C ephemeral + durable + promise + # S15 ephemeral + durable + promise + scheduled already run: 235x + # + # ── Why ratePerSec differs between them, which is not a knob being fiddled ── + # + # `workload::start` divides `ratePerSec` evenly across the streams that have + # agents, so a constant here would give ephemeral 100/s alone and 33/s in a + # threesome, and the comparison would be between three different ephemeral + # workloads rather than between three fault environments. Each entry sets the + # rate that holds ephemeral near the ~33/s S15 ran it at. Near, not at: S15 + # and S15C give it 100/3 = 33.33/s while S15B and S15A give it 34/s, a 2% + # spread left uncorrected because S15A and S15B have already been run at 34 + # and changing the rate now would break comparison with their results. + # + # A second confound is not removable the same way. `MAX_IN_FLIGHT / + # active.len()` gives ephemeral a budget of 1024, 512 and 341 across S15A, + # S15B and S15C, and that ceiling is a constant in the driver rather than a + # suite setting. It bounds throughput, not latency, and latency is what this + # series reads, so it is recorded rather than corrected. It does *not* + # separate S15C from S15: `active` counts only streams with agents in the + # workload block, and S15 leaves `scheduledAgents` at zero because its + # scheduled stream comes from the separate emitter, so both run three streams + # at 341 and that pair is free of this confound entirely. + # + # ── Why the scheduled block is present with zero targets ─────────────────── + # + # `storage_fault::run` requires the block. Zero targets means no registrations + # and no fires, so the scheduled stream is not driven and `drives_stream` + # reports it as absent, which is what keeps it out of the expectations below. + # --------------------------------------------------------------------------- + - code: S15A + name: keyvalue-postgres-latency-ephemeral-only + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + # The same file S15 applies, not a copy of it. Four manifests that must + # stay identical by hand is four chances for the fault to diverge from the + # run it is being compared against. + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + # S15's exactly. Shorter windows would be cheaper and the effect is large + # enough to survive them, but the whole value of this series is that its + # ephemeral numbers are comparable with S15's, and a median taken over a + # different window length is a different measurement. + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 0 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # One active stream, so ephemeral gets all of it. + ratePerSec: 34 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + + # Empty, and the only entry in the suite that is. Nothing this workload + # drives reaches the delayed cluster, so there is no stream whose + # slowdown could evidence the fault. That makes this a control rather + # than an experiment, and it means the run cannot fail on + # `slowdown-not-observed`: it cannot tell a delay that landed and did + # not matter from one that never applied. + # + # The evidence has to come from outside the workload, and it does. The + # scheduler polls its own schema on this cluster whether or not anything + # is scheduled, so `db_success_seconds{svc="scheduler_storage"}` moves + # under the fault regardless of what the workload does. Check it before + # believing a clean result here. + slowed: [] + + # Required by the schema and inert here, because the floor is only ever + # applied to the streams `slowed` names and it names none. Left at the + # series' value rather than at something arbitrary, so a later edit that + # gives this run a slowed stream inherits a floor that means what it + # does everywhere else. + slowdownFloor: 2.0 + + # The whole question. If ephemeral holds its 196ms median with nothing + # else running, then the 235x in S15 came from the company it was + # keeping and not from the fault. + steady: [ephemeral] + steadyCeiling: 1.5 + + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15B + name: keyvalue-postgres-latency-plus-durable + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + # S15's durable population, unchanged, so the load this adds is the load + # S15 had rather than a scaled-down version of it. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # Two active streams, so 34/s each. + ratePerSec: 68 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Durable waits on two synchronous writes to this cluster per + # invocation, so it should move a long way. S15 measured 644x. + slowed: [durable] + slowdownFloor: 2.0 + # And this is the reading that matters. If ephemeral moves here, durable + # alone is enough to do it and the interaction is between those two. + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15C + name: keyvalue-postgres-latency-plus-promise + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + # Three active streams, so 33.3/s each — the rate S15 ran them at. + ratePerSec: 100 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Promise makes the most calls of any stream: three of its own, plus the + # two its durable agent pays on each of the two invocations an operation + # takes. S15 measured 939x. + slowed: [durable, promise] + slowdownFloor: 2.0 + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S23 — indexed-oplog PostgreSQL latency degradation (GOL-525) + # + # S14 with the cut replaced by a delay, and the last empty cell of the storage + # matrix: outage and latency are now both covered on the key-value cluster + # (S16/S22 and S15), on the Redis cache (S18 and S17), and here on the + # indexed-oplog cluster (S14 and this). + # + # The fault shape, the phase lengths, the workload and the thresholds are + # S15's, unchanged, so the two reports differ only in which store was slowed. + # + # ── Why this one has no control stream ───────────────────────────────────── + # + # S15 and S17 each keep one stream on the far side of the delay, and that + # stream holding its pace is what proves the fault reached only what it aimed + # at. Nothing here can play that part, because every agent commits its oplog + # to this cluster: + # + # * durable — `PrimaryOplogService` *is* the indexed storage, so every + # commit is an `append_many` against `index_storage`. + # * ephemeral — never opens the primary, but `EphemeralOplogState::commit` + # appends to `lower.first()`, and with `indexed_storage_layers: 2` that + # first lower layer is a `CompressedOplogArchiveService` on the same + # indexed storage. So it pays an indexed write too, one layer down. + # * promise — its keys are on the key-value cluster, which this leaves + # alone. What it pays is the oplog commit of the durable agent underneath + # it, twice per operation. + # * scheduled — the scheduler's schema is also on the key-value cluster, so + # claims and acknowledgements are outside this delay. What is inside it is + # the fired invocation's own commit. + # + # And the write is on the caller's path in all four, not merely somewhere in + # the lifecycle: `InvocationHooks::on_agent_invocation_started` commits with + # `CommitLevel::Always` *before* the invocation body runs, for every agent + # mode. + # + # So `steady` is empty and the run asserts nothing about routing. The evidence + # that the delay reached only this cluster has to come from metrics instead, + # and the runbook says which: `db_*{svc="oplog"}` and `svc="compressed_oplog"` + # are this cluster, while `svc="worker"`, `"key_value"`, `"promise"` and + # `"scheduler_storage"` are the one that should stay flat. That `svc` names + # the caller rather than the pool is the correction S14 shipped. + # + # ── The arithmetic, and the gate S15 does not have ───────────────────────── + # + # golem-dev sets `GOLEM__INDEXED_STORAGE__CONFIG__MAX_CONCURRENT_OPS` to 64 in + # front of a 96-connection pool, and `PostgresIndexedStorage::configured` + # turns that setting into a `Semaphore`. The gate is therefore tighter than + # the pool, and it is the gate that binds: under a 500ms one-way delay it + # clears about 128 indexed operations per second per pod where the pool alone + # would clear 192. No other store in the suite has such a gate, which makes + # this the one run that can measure what it costs. + # + # Nothing here predicts a multiple from that. S15 counted round trips, said + # 17x, and measured 644x, because counting round trips gives the cost of one + # operation running alone and queueing was the whole answer. The same applies + # with more force here, since this cluster carries the heaviest write traffic + # the platform produces. + # + # ── What could break, which is not what S15 could break ──────────────────── + # + # A delay produces no errors, so the indexed retry budget — 200 attempts with + # a 10s cap on golem-dev — should never be entered at all. The way in is + # queueing rather than failure: an acquire that waits past sqlx's default 30s + # yields `PoolTimedOut`, `RepoError::is_transient` classifies it transient, and + # `retry_storage_op` retries. That path re-serializes the whole commit batch + # per attempt (GOL-480), so a run that enters it turns a code reading into a + # measurement. A run that does not enter it says the gate held. + # + # This is also the first scenario to put the on-executor oplog sweep under a + # degraded indexed store: its scan and its archive reads both cross this + # cluster and take permits from the same semaphore. Recorded, not asserted. + # --------------------------------------------------------------------------- + - code: S23 + name: indexed-oplog-postgres-latency + enabled: true + + fault: + # As S15 and S17. Recorded for the archive rather than acted on: the + # workflow picks the manifest by scenario code and reads the Chaos Mesh + # kind out of the manifest itself. + kind: network-delay + target: worker-executor + # Every executor. A store that is slow for only some of them is a routing + # fault wearing a database's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # S15's, unchanged, so the pair is readable. Long enough for a memory + # series with a slope, for a steady state rather than a transient, and for + # a pool that is going to saturate to actually fill. + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + # Identical to the other six. Comparability is the whole reason the + # storage runs share a workload, and this one is half of a matched pair + # with S15. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # As the others. Unlike S15 the scheduler is *outside* this delay, so a + # claim costs what it always did and 240s is not a number this run can + # approach. A lease expiry here would be a real finding rather than a + # threshold set too tight — and it would mean the delay reached a cluster + # it does not name. + leaseBudgetSecs: 240 + + storage: + # The indexed Aurora cluster's writer endpoint, carrying the oplog and + # nothing else. The same endpoint S14 cuts. Neither the key-value cluster + # nor Redis is touched: slowing a second store would leave every finding + # unattributable. + endpoint: golem-postgres-dev-indexed.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + # As S15 and S17. Under a delay nothing is supposed to fall silent, so + # silence is not available as evidence that the fault landed and time + # has to stand in for it. + kind: latency-degradation + + # All four, per the walk-through above. This is the only entry in the + # suite that names every stream, and it is not a hedge: each one has a + # named synchronous write to this cluster on its caller's path. + slowed: [durable, ephemeral, promise, scheduled] + + # 2.0, as S15 and S17. The floor exists to fail a run where the netem + # rule never applied, which leaves every stream near 1.0x. Setting it + # near a predicted figure would instead assert that the prediction is + # right, and the prediction is the thing under test. + slowdownFloor: 2.0 + + # Deliberately empty, and the one field of this entry that is a loss + # rather than a copy. See the walk-through: there is no stream on the + # far side of this delay to hold up as evidence, so the routing check + # S15 and S17 get for free has to be done by hand against the `svc` + # series named above. + steady: [] + + # A delay should cost time, not work. Held at 50 for comparability with + # S15, and read with S15's lesson attached: throughput is concurrency + # over latency, so once a stream's in-flight budget saturates a stream + # slowed by three orders of magnitude *must* fall below any serving + # floor. `outage.rs` suppresses the finding when the stream's own + # slowdown accounts for the shortfall, which is the case this run is + # most likely to produce on every stream at once. + servingFloorPercent: 50 + + # Recorded, not asserted. Expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for and no backlog to + # drain, only the last few in-flight commits finishing at their slower + # pace. If an executor was lost to the pool timeout this is the number + # that shows it, as the replacement has to reload its agents first. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the other six. Nothing should time out and so nothing + # should retry; a retry count above zero is itself a finding. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # ── MF1 ────────────────────────────────────────────────────────────────── + # + # The first composed fault: an executor killed while the key-value cluster is + # unreachable (GOL-381). + # + # Neither half is new. S16 takes the key-value cluster away for the length of + # an AWS failover and the platform absorbs it; S8 and S10 kill an executor + # under load and the platform recovers its shards. What is new is that each of + # those two faults leaves the platform a way out that the other one takes + # away. + # + # Under S16 nobody is lost. The shards nobody can currently read are still + # owned by a process that will read them when the store comes back, so the + # recovery is a matter of waiting. Under S8 the store that records the + # handover is working: the shard-manager reassigns, the survivor reads the + # running-workers set, and it knows which agents to bring up. + # + # Held together, a survivor has to take over shards while the running-workers + # set, the promises and the scheduler's schema are all behind the cut. That is + # the question, and it has no answer anywhere in the single-fault matrix. + # + # ── Why the window is 120s and not S16's 60s ───────────────────────────── + # + # Two constraints from opposite directions, and this is the only number in + # the entry that has to sit between them. + # + # The ceiling is the key-value retry budget, roughly 168s (see S16). Past it + # the recovery-index write in `status_flusher` exhausts and panics by design, + # every executor is replaced, and the run stops being about a kill during an + # outage and starts being S22 with an extra step. 120s leaves 48s of margin + # against the worst case, which is an operation issued at the instant of + # injection. + # + # The floor is what has to fit inside the *overlap*, which is half the window: + # the shard-manager has to notice the executor is gone, revoke and reassign + # its shards, and the survivor has to try to bring those agents up against a + # store it cannot reach. S16's 60s window would leave 30s for all of that, and + # a run that heals before the survivor has finished trying has not asked the + # question. + # + # ── The one thing this entry cannot check for you ──────────────────────── + # + # `composed.minOverlapSecs` is judged after the fact, not enforced. Every + # other gate in this suite can refuse before the window is spent — S2's + # pairing gate, S9's forward leg, the fire-count proof below — because the + # driver still holds the run at that point. Here it does not: by the time the + # second fault is due the fault window belongs to the workflow. So a + # composition that missed is reported rather than prevented, and the report + # says so loudly enough that a reader cannot mistake it for a clean run. + - code: MF1 + name: executor-kill-during-keyvalue-outage + enabled: true + + fault: + # The enclosing fault, and the one whose window the phases follow. Every + # executor cut off from the key-value cluster, exactly as S16. + kind: network-partition + target: worker-executor + mode: all + # Named explicitly rather than left to the workflow's glob. A composed + # scenario has two manifests carrying its code, and picking between them + # by alphabetical order is not a decision anyone made. + manifest: networkchaos-mf1.yaml + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 120 + + composed: + # The second fault, injected inside the first one's window and left in + # force until it heals. Recorded here for the archive; the workflow + # applies the manifest and the driver learns about it from the signal. + kind: pod-kill + target: worker-executor + manifest: podchaos-mf1.yaml + # 0.6 of the window, which leaves 48s of overlap, and the number that + # matters is the one it is under. + # + # `GOLEM__WORKER_EXECUTORS__ASSIGN_SHARDS_TIMEOUT` is **60s** on golem-dev + # and golem-prod. It was raised from the 5s default in August 2026 after a + # chaos run found the same failure on the revoke leg: 5s cannot cover a + # handover whose handler recovers every agent it was just given, so the + # shard-manager dropped the shards from the routing table and put them + # back on a pod that had already let them go. + # + # MF1's claim is that the platform rides the outage out, and that claim + # only holds while the reassignment call outlives the outage. The first + # run was at 0.5 and produced 59.63s of overlap against that 60s ceiling — + # it passed with under four tenths of a second to spare, by luck rather + # than by design, and a slightly slower fault would have turned it into a + # different experiment without saying so. + # + # The overlap can only ever come out *shorter* than this fraction implies, + # never longer: the workflow measures the remainder from the wall clock, + # so time spent confirming the injection comes out of the overlap. That is + # why there is a floor below and no ceiling above it. + # + # MF1B is the deliberate version of the run this nearly became. + afterFraction: 0.6 + # Below this the two faults did meet, but with too little of the outage + # left for a shard reassignment to run into it, which is the whole + # experiment. Comfortably under the 48s the schedule delivers, so an + # injection that took its time being confirmed still counts. + minOverlapSecs: 30 + + phases: + # S16's, unchanged. Long enough for cold starts, component loading and + # route warm-up to settle, and for the scheduler to be firing steadily. + baselineSecs: 300 + # See the walk-through above for both bounds. + faultSecs: 120 + # Longer than S16's 420. There is one more thing to wait for here: the + # killed executor has to be rescheduled, register with the shard-manager, + # take its shards back and reload the agents on them, and only then can + # the streams be said to have recovered. Scheduled actions registered + # during the outage still have to fall due and fire inside the run. + recoverySecs: 600 + + workload: + # S16's population, unchanged, so the two results are readable side by + # side. That comparability is the point of not tuning it: MF1's numbers + # only mean something against a run that had the outage and not the kill. + durableAgents: 200 + ephemeralAgents: 50 + # Zero on purpose. The scheduled stream is driven by the `scheduled` block + # below, because only its registrations carry a token into the target's + # fire log. Setting both is refused at load time. + scheduledAgents: 0 + promiseAgents: 50 + # No quota stream. A lease renewal failing would add a shard-manager + # failure mode to a run that already has two faults to attribute between. + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + # 100 targets, as S16 — and here they do a second job. The driver aims the + # kill at the executor owning the largest share of them, which is what + # turns the fire account's killed/elsewhere axis from a degenerate column + # into the run's own control group. In the ten single-fault storage codes + # nothing is killed and every target reads as `elsewhere`. + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Larger than S16's 240. The floor there was the outage itself; here it is + # the outage plus a shard reassignment that cannot complete until the + # store comes back. Recorded, not asserted, like every other budget here. + leaseBudgetSecs: 300 + + storage: + # The key-value Aurora cluster's writer endpoint. The same one S16 cuts, + # which is what makes the two runs comparable. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + # Everything is behind this cut, as in S16. Quiet time is measured + # against the window's own edges, so the floor means the same thing at + # 120s as it does at 60s. + kind: whole-workload + quietFloorPercent: 50 + + # Doubled from S16's 120, and for a reason that is not slack: serving + # again here means an executor that did not exist when the store came back + # has to be scheduled, registered and reloaded first. Recorded, not + # asserted. + recoveryBudgetSecs: 240 + + retryPolicy: + # Identical to S16's, and load-bearing for the same reason: retrying only + # transport errors, once, under the same idempotency key is what lets a + # retry reveal double execution rather than hide it. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + # ── MF1B ───────────────────────────────────────────────────────────────── + # + # MF1 with the overlap pushed past the shard-manager's assign timeout + # (GOL-381). Same two faults, same manifests, same workload, same endpoint. + # One number differs, and it is the one the result turns on. + # + # `GOLEM__WORKER_EXECUTORS__ASSIGN_SHARDS_TIMEOUT` is 60s on golem-dev and + # golem-prod. MF1 keeps its overlap under that and the reassignment call + # outlives the outage; here the overlap is ~97s, so the call gives up while the + # storage is still gone. What the cluster does next is not written down + # anywhere. + # + # ── Why this is a scenario and not a knob on MF1 ───────────────────────── + # + # Same reason S22 is not a longer S16. The two runs are judged by opposite + # expectations: MF1 says nothing should be disturbed, MF1B says the handover + # should fail and then recover. Folding them into one entry would leave the + # report unable to say which outcome it was hoping for, and the pair only + # means anything read side by side. + # + # ── The arithmetic, because three ceilings meet here ───────────────────── + # + # * The overlap must exceed **60s**, or the assign call never times out and + # this is MF1 with extra steps. 130s * (1 - 0.25) = 97.5s, and the + # workflow can only make that shorter, never longer. + # * The whole window must stay inside the **~168s** key-value retry budget, + # or the executors panic and the run is S22. 130s leaves 38s of margin. + # * A *fully* failed assign takes five attempts at 60s plus ~15s of backoff, + # about **315s**, which no survivable outage can contain. So this run does + # not reach "the shard-manager gave up and put the shards back". It + # reaches the first timeout and the retry after it, and that is the + # honest limit of what this fault can show. + # + # That last bound is worth stating plainly rather than discovering: on the + # timeouts golem-dev runs today, the routing-table corruption the August 2026 + # investigation documented is **unreachable through a storage outage**, + # because the executor's own retry budget expires first. MF1B measures the + # near miss. + - code: MF1B + name: executor-kill-past-the-assign-timeout + enabled: true + + fault: + kind: network-partition + target: worker-executor + # MF1's manifest, named rather than copied. Two files that must stay + # identical by hand is how a comparison silently stops being one; the + # S15A/S15B/S15C entries share S15's manifest for the same reason. + manifest: networkchaos-mf1.yaml + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 130 + + composed: + kind: pod-kill + target: worker-executor + manifest: podchaos-mf1.yaml + # A quarter in, leaving ~97.5s of overlap: past the 60s assign timeout + # with room for the retry after it to run for half a minute before the + # heal. + afterFraction: 0.25 + # The floor is the assign timeout plus headroom, not an arbitrary + # fraction. Below 70s the overlap may not have outlasted a single + # `assign_shards` attempt, and then the run is MF1 wearing MF1B's code — + # which is exactly the confusion this pair exists to remove. + minOverlapSecs: 70 + + phases: + # MF1's, so the two are readable side by side. + baselineSecs: 300 + # See the three ceilings above. + faultSecs: 130 + # MF1's. The handover has one more thing to do here — a timed-out assign + # has to be retried after the storage returns — so the recovery window is + # if anything more load-bearing than it is there. + recoverySecs: 600 + + workload: + # MF1's population, unchanged. Only the timing differs between these two + # entries, and that is the entire point. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Larger than MF1's 300s. The floor there was the outage plus one shard + # reassignment; here it is the outage plus a reassignment that has to be + # attempted twice. Recorded, not asserted. + leaseBudgetSecs: 360 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: whole-workload + quietFloorPercent: 50 + + # MF1's 240s. The extra failed assign attempt is the shard-manager's time, + # not the streams', so this does not need to grow with the window. + recoveryBudgetSecs: 240 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index af889b5156..db27bcea11 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -592,6 +592,29 @@ async fn run_chaos( ChaosScenarioArg::S5 => chaos::ScenarioCode::S5, ChaosScenarioArg::S12 => chaos::ScenarioCode::S12, ChaosScenarioArg::S13 => chaos::ScenarioCode::S13, + ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, + ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, + ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, + ChaosScenarioArg::S2 => chaos::ScenarioCode::S2, + ChaosScenarioArg::S4 => chaos::ScenarioCode::S4, + ChaosScenarioArg::Mf2 => chaos::ScenarioCode::MF2, + ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, + ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, + ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, + ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, + ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, + ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, + ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, + ChaosScenarioArg::S17 => chaos::ScenarioCode::S17, + ChaosScenarioArg::S15 => chaos::ScenarioCode::S15, + ChaosScenarioArg::S15A => chaos::ScenarioCode::S15A, + ChaosScenarioArg::S15B => chaos::ScenarioCode::S15B, + ChaosScenarioArg::S15C => chaos::ScenarioCode::S15C, + ChaosScenarioArg::S23 => chaos::ScenarioCode::S23, + ChaosScenarioArg::Mf1 => chaos::ScenarioCode::MF1, + ChaosScenarioArg::Mf1b => chaos::ScenarioCode::MF1B, + ChaosScenarioArg::S21 => chaos::ScenarioCode::S21, + ChaosScenarioArg::S19 => chaos::ScenarioCode::S19, }; let config = suite .scenario(code, allow_disabled) @@ -623,6 +646,56 @@ async fn run_chaos( chaos::ScenarioCode::S13 => { chaos::scenarios::s13::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S10 => { + chaos::scenarios::s10::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S11 => { + chaos::scenarios::s11::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S3 => { + chaos::scenarios::s3::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S19 => { + chaos::scenarios::s19::run(&config, &manifest, &deps, &signals, &outputs).await + } + code @ (chaos::ScenarioCode::S4 | chaos::ScenarioCode::MF2) => { + chaos::scenarios::dns_fault::run( + code, &config, &manifest, &deps, &signals, &outputs, + ) + .await + } + code @ (chaos::ScenarioCode::S2 | chaos::ScenarioCode::S21) => { + chaos::scenarios::relay_fault::run( + code, &config, &manifest, &deps, &signals, &outputs, + ) + .await + } + chaos::ScenarioCode::S7 => { + chaos::scenarios::s7::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S6 => { + chaos::scenarios::s6::run(&config, &manifest, &deps, &signals, &outputs).await + } + chaos::ScenarioCode::S9 => { + chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await + } + code @ (chaos::ScenarioCode::S14 + | chaos::ScenarioCode::S15 + | chaos::ScenarioCode::S15A + | chaos::ScenarioCode::S15B + | chaos::ScenarioCode::S15C + | chaos::ScenarioCode::S16 + | chaos::ScenarioCode::S17 + | chaos::ScenarioCode::S18 + | chaos::ScenarioCode::S22 + | chaos::ScenarioCode::S23 + | chaos::ScenarioCode::MF1 + | chaos::ScenarioCode::MF1B) => { + chaos::scenarios::storage_fault::run( + code, &config, &manifest, &deps, &signals, &outputs, + ) + .await + } }; deps.kill_all().await; @@ -855,7 +928,7 @@ async fn run_density( // Emit the cell result as a single-result suite so the JSON shape // matches cloud-perf (BenchmarkSuiteResultCollection), and the - // buildspec can upload it directly to S3. + // buildspec can upload it directly to golem-bench-results. let mut suite_result = BenchmarkSuiteResult::new(&format!("density-{section}")); suite_result.add(result); if let Some(run_id) = cloud_bench_run_id() { diff --git a/integration-tests/src/benchmarks/density/agent.rs b/integration-tests/src/benchmarks/density/agent.rs index 96dba8f7bc..0b3fcc28fd 100644 --- a/integration-tests/src/benchmarks/density/agent.rs +++ b/integration-tests/src/benchmarks/density/agent.rs @@ -185,7 +185,8 @@ pub struct CellConfig { } impl CellConfig { - /// The cell's name, used in S3 paths and result identifiers. Encodes the + /// The cell's name, used in `golem-bench-results` keys and result identifiers. + /// Encodes the /// full axis set in human-readable terms (no cryptic single letters). pub fn cell_name(&self) -> String { let mut parts = vec![ diff --git a/integration-tests/src/benchmarks/density/prep.rs b/integration-tests/src/benchmarks/density/prep.rs index 209076716e..f5e208b473 100644 --- a/integration-tests/src/benchmarks/density/prep.rs +++ b/integration-tests/src/benchmarks/density/prep.rs @@ -20,7 +20,8 @@ //! //! The buildspec runs prep exactly once at suite start (against a freshly-wiped //! cluster) and writes the resulting [`PrepManifest`] to a file that is -//! uploaded to S3 and passed to every per-cell invocation via +//! uploaded to the `golem-bench-results` bucket and passed to every per-cell +//! invocation via //! `--prep-manifest`. The manifest carries the account token and all component //! IDs so per-cell invocations need no by-name lookup and no re-tokenization — //! this is also the resume mechanism: a resumed run reloads the same manifest. diff --git a/integration-tests/src/chaos/composed.rs b/integration-tests/src/chaos/composed.rs new file mode 100644 index 0000000000..41b388014f --- /dev/null +++ b/integration-tests/src/chaos/composed.rs @@ -0,0 +1,487 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Two faults held at once, and whether they really were (GOL-381). +//! +//! Every other scenario in the suite injects one fault and reads the platform's +//! answer to it. The `MF` codes inject a second one inside the first one's +//! window, and that changes what can go wrong with the *run* rather than with +//! the platform: the two faults can miss each other. A kill that lands after +//! the storage came back is a perfectly ordinary S16 followed by a perfectly +//! ordinary S8, and it produces a result that looks exactly like a composed run +//! whose composition worked. +//! +//! So this module reports the shape of the composition rather than any +//! behaviour of the platform. It answers three questions, in the order they +//! stop mattering: +//! +//! 1. Did the second fault land at all? +//! 2. Did it land inside the first one's window? +//! 3. Was it inside for long enough to mean anything? +//! +//! None of them is about golem. All three are about whether the rest of the +//! report is worth reading, which is the same job [`crate::chaos::relay`]'s +//! pairing gate does for S2 and the forward-leg gate does for S9. The +//! difference is that those two can refuse before the window is spent, and this +//! one cannot: the driver has already handed the fault window to the workflow +//! by the time the second fault is due. +//! +//! ## Why the overlap is measured from the kill to the heal +//! +//! The two faults are not the same shape. The enclosing fault is a *condition* +//! that lasts from injection to heal: for MF1, storage is unreachable for +//! exactly as long as the NetworkChaos rules are installed. The inner fault is +//! an *instant* whose consequence persists: `pod-kill` deletes the pod once, and +//! what lasts is the cluster being one executor short. +//! +//! Overlap is therefore how much of the enclosing window the cluster spent +//! short-handed — from the kill to the heal — and not the intersection of two +//! durations. A `duration` on the inner PodChaos governs how long Chaos Mesh +//! considers its experiment running and has nothing to do with how long the pod +//! is gone. +//! +//! ## What is deliberately not here +//! +//! Whether the killed executor came back, and how long it took. That is not +//! knowable from the fault signals — the workflow reports what it asked the +//! cluster to do, not what the cluster did about it — and it is already +//! answered better elsewhere: the ownership samples show the shards leaving the +//! killed executor and returning, with timestamps, and the read-back shows +//! whether anything was lost on the way. Re-deriving a weaker version of that +//! from two timestamps would give a reader a second number to reconcile with +//! the first. + +use crate::chaos::signal::FaultInjected; +use crate::chaos::split::round2; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +/// One of the two faults, as the workflow reported it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComposedLeg { + /// Identifier of the Chaos Mesh object, so a leftover can be traced back. + pub fault_id: String, + /// Fault kind as the workflow named it, e.g. `pod-kill`. + pub kind: String, + /// What it was aimed at: a deployment name for an unpinned fault, a pod + /// name for a pinned one. + pub target: String, + pub injected_at: DateTime, +} + +impl From<&FaultInjected> for ComposedLeg { + fn from(signal: &FaultInjected) -> Self { + Self { + fault_id: signal.fault_id.clone(), + kind: signal.kind.clone(), + target: signal.target.clone(), + injected_at: signal.injected_at, + } + } +} + +/// A way the composition failed to be a composition. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ComposedViolation { + /// No second fault was ever reported. The run injected one fault and is a + /// single-fault scenario wearing an `MF` code. + SecondaryNeverInjected, + /// The second fault landed before the first one was injected, or after it + /// healed. Two faults in sequence rather than one composed fault. + SecondaryOutsidePrimary, + /// The second fault landed inside the window but too near its end for the + /// combination to have been held for any length of time. + OverlapTooShort, +} + +impl ComposedViolation { + pub fn as_str(self) -> &'static str { + match self { + ComposedViolation::SecondaryNeverInjected => "secondary-never-injected", + ComposedViolation::SecondaryOutsidePrimary => "secondary-outside-primary", + ComposedViolation::OverlapTooShort => "overlap-too-short", + } + } +} + +impl std::fmt::Display for ComposedViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, with the evidence for it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComposedFinding { + pub violation: ComposedViolation, + pub detail: String, +} + +/// How the two faults lined up. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComposedFaultReport { + /// The fault whose window the run's phases follow. + pub primary: ComposedLeg, + /// The fault injected inside it. Absent when it never landed, which is + /// itself the first finding. + #[serde(skip_serializing_if = "Option::is_none")] + pub secondary: Option, + /// Seconds from the first injection to the second. Negative means the + /// second one landed first. + #[serde(skip_serializing_if = "Option::is_none")] + pub secondary_offset_secs: Option, + /// Seconds the cluster spent under both faults: from the second injection + /// to the first fault's heal. + /// + /// `None` when the run never saw a heal, which is an abort rather than a + /// short overlap and is why the two cases are not folded together. + #[serde(skip_serializing_if = "Option::is_none")] + pub overlap_secs: Option, + /// That overlap as a share of the enclosing window, which is the figure + /// that survives a change of window length. + #[serde(skip_serializing_if = "Option::is_none")] + pub overlap_percent: Option, + /// The floor the overlap is judged against, echoed so the report can be + /// read without the suite YAML. + pub overlap_floor_secs: u64, + pub findings: Vec, +} + +impl ComposedFaultReport { + /// Reduces the two fault signals into the account. + /// + /// `primary_recovered_at` is the enclosing fault's heal. It is separate + /// from the leg because the driver learns it later, and because a run that + /// aborted before the heal still has a primary worth reporting. + pub fn build( + primary: &FaultInjected, + primary_recovered_at: Option>, + secondary: Option<&FaultInjected>, + overlap_floor: Duration, + ) -> Self { + let floor_secs = overlap_floor.as_secs(); + let mut findings = Vec::new(); + + let Some(secondary) = secondary else { + findings.push(ComposedFinding { + violation: ComposedViolation::SecondaryNeverInjected, + detail: format!( + "the second fault never reported itself active, so {} ran on its own and \ + nothing in this result describes two faults held at once", + primary.kind + ), + }); + return Self { + primary: primary.into(), + secondary: None, + secondary_offset_secs: None, + overlap_secs: None, + overlap_percent: None, + overlap_floor_secs: floor_secs, + findings, + }; + }; + + let offset = secs_between(primary.injected_at, secondary.injected_at); + let overlap = primary_recovered_at.map(|at| secs_between(secondary.injected_at, at)); + let window = primary_recovered_at.map(|at| secs_between(primary.injected_at, at)); + + // Ordering first. An overlap computed from a second fault that landed + // outside the window is a number with no meaning, so the two findings + // are exclusive rather than cumulative. + if offset < 0.0 { + findings.push(ComposedFinding { + violation: ComposedViolation::SecondaryOutsidePrimary, + detail: format!( + "{} landed {}s before {} was injected, so the two faults ran in sequence", + secondary.kind, + round2(-offset), + primary.kind + ), + }); + } else if overlap.is_some_and(|overlap| overlap <= 0.0) { + findings.push(ComposedFinding { + violation: ComposedViolation::SecondaryOutsidePrimary, + detail: format!( + "{} landed {}s after {} was injected, which is at or past its heal, so the \ + two faults ran in sequence", + secondary.kind, + round2(offset), + primary.kind + ), + }); + } else if let Some(overlap) = overlap + && overlap < floor_secs as f64 + { + findings.push(ComposedFinding { + violation: ComposedViolation::OverlapTooShort, + detail: format!( + "the cluster was under both faults for {}s, short of the {floor_secs}s this \ + run is judged by, so whatever the combination does had almost no time to \ + happen in", + round2(overlap) + ), + }); + } + + Self { + primary: primary.into(), + secondary: Some(secondary.into()), + secondary_offset_secs: Some(round2(offset)), + // Only a positive overlap is reported. A second fault that landed + // after the heal yields a negative number, and a reader who saw it + // would have to work out that it is the same fact the finding above + // already states. The offset is always there and always signed, so + // nothing is lost by leaving this one out. + overlap_secs: overlap.filter(|overlap| *overlap > 0.0).map(round2), + overlap_percent: match (overlap, window) { + (Some(overlap), Some(window)) if window > 0.0 && overlap > 0.0 => { + Some(round2(100.0 * overlap / window)) + } + _ => None, + }, + overlap_floor_secs: floor_secs, + findings, + } + } + + pub fn has_findings(&self) -> bool { + !self.findings.is_empty() + } + + /// Lines an operator has to read. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| format!("{}: {}", f.violation, f.detail)) + .collect() + } + + /// Lines that make the run readable without being problems themselves. + /// + /// The composition's own shape belongs here even on a clean run: every + /// number below it in the report was measured on a cluster under two + /// faults, and a reader who does not know when the second one landed cannot + /// place any of them. + pub fn note_lines(&self) -> Vec { + let Some(secondary) = &self.secondary else { + return Vec::new(); + }; + + let mut lines = vec![format!( + "{} on {} landed {}s into the {} on {}", + secondary.kind, + secondary.target, + self.secondary_offset_secs.unwrap_or_default(), + self.primary.kind, + self.primary.target + )]; + + if let (Some(overlap), Some(percent)) = (self.overlap_secs, self.overlap_percent) { + lines.push(format!( + "both faults were in force for {overlap}s, {percent}% of the enclosing window" + )); + } + + lines + } +} + +/// Seconds from `from` to `to`, negative when `to` is earlier. +fn secs_between(from: DateTime, to: DateTime) -> f64 { + (to - from).num_milliseconds() as f64 / 1000.0 +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeDelta; + use test_r::test; + + const FLOOR: Duration = Duration::from_secs(20); + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-09-03T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn signal(kind: &str, at: DateTime) -> FaultInjected { + FaultInjected { + fault_id: format!("chaos-mf1-{kind}"), + kind: kind.to_string(), + target: "worker-executor".to_string(), + injected_at: at, + } + } + + fn violations(report: &ComposedFaultReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The shape the scenario exists to produce: the kill lands halfway through + /// the outage and the cluster is short-handed for the rest of it. + #[test] + fn a_kill_inside_the_window_is_clean_and_reports_its_overlap() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(30)); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(60)), + Some(&secondary), + FLOOR, + ); + + assert!(report.findings.is_empty(), "{:?}", report.findings); + assert_eq!(report.secondary_offset_secs, Some(30.0)); + assert_eq!(report.overlap_secs, Some(30.0)); + assert_eq!(report.overlap_percent, Some(50.0)); + } + + /// A negative overlap is not a measurement, and the ordering finding + /// already says what happened. Reporting one would leave a reader + /// reconciling two statements of the same fact, one of them nonsense. + #[test] + fn a_kill_outside_the_window_reports_no_overlap() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(150)); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(120)), + Some(&secondary), + FLOOR, + ); + + assert_eq!( + violations(&report), + vec![ComposedViolation::SecondaryOutsidePrimary] + ); + assert_eq!(report.overlap_secs, None); + assert_eq!(report.overlap_percent, None); + assert_eq!(report.secondary_offset_secs, Some(150.0)); + } + + /// The failure this whole module exists for. Without it the run reads as a + /// clean composed result, because every other account in the report is + /// perfectly happy to describe two faults that never met. + #[test] + fn a_kill_after_the_heal_is_two_faults_in_sequence() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(90)); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(60)), + Some(&secondary), + FLOOR, + ); + + assert_eq!( + violations(&report), + vec![ComposedViolation::SecondaryOutsidePrimary] + ); + } + + /// Nothing here can be judged against a floor, so the ordering finding is + /// raised alone rather than alongside a meaningless overlap. + #[test] + fn a_kill_before_the_outage_is_reported_once_not_twice() { + let primary = signal("network-partition", t0() + TimeDelta::seconds(30)); + let secondary = signal("pod-kill", t0()); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(90)), + Some(&secondary), + FLOOR, + ); + + assert_eq!( + violations(&report), + vec![ComposedViolation::SecondaryOutsidePrimary] + ); + assert_eq!(report.secondary_offset_secs, Some(-30.0)); + } + + /// Inside the window, but with seconds to spare rather than a window to + /// measure anything in. + #[test] + fn a_kill_at_the_very_end_of_the_window_is_too_short() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(55)); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(60)), + Some(&secondary), + FLOOR, + ); + + assert_eq!( + violations(&report), + vec![ComposedViolation::OverlapTooShort] + ); + assert_eq!(report.overlap_secs, Some(5.0)); + } + + /// A missing second fault is a finding about the run, and the report says + /// so rather than leaving a reader to notice the absent field. + #[test] + fn a_composition_that_never_happened_is_a_finding() { + let primary = signal("network-partition", t0()); + let report = + ComposedFaultReport::build(&primary, Some(t0() + TimeDelta::seconds(60)), None, FLOOR); + + assert_eq!( + violations(&report), + vec![ComposedViolation::SecondaryNeverInjected] + ); + assert!(report.secondary.is_none()); + assert!(report.note_lines().is_empty()); + } + + /// An abort before the heal leaves the overlap unknowable. Reporting it as + /// zero would raise `overlap-too-short` against a run that may well have + /// been composed correctly. + #[test] + fn an_unhealed_run_reports_no_overlap_rather_than_a_short_one() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(30)); + let report = ComposedFaultReport::build(&primary, None, Some(&secondary), FLOOR); + + assert!(report.findings.is_empty(), "{:?}", report.findings); + assert_eq!(report.overlap_secs, None); + assert_eq!(report.overlap_percent, None); + assert_eq!(report.secondary_offset_secs, Some(30.0)); + } + + /// The composition's shape is context on every run, including clean ones: + /// every figure below it was measured under two faults at once. + #[test] + fn a_clean_composition_still_explains_itself() { + let primary = signal("network-partition", t0()); + let secondary = signal("pod-kill", t0() + TimeDelta::seconds(30)); + let report = ComposedFaultReport::build( + &primary, + Some(t0() + TimeDelta::seconds(60)), + Some(&secondary), + FLOOR, + ); + + assert!(report.attention_lines().is_empty()); + assert_eq!(report.note_lines().len(), 2); + } +} diff --git a/integration-tests/src/chaos/deletions.rs b/integration-tests/src/chaos/deletions.rs new file mode 100644 index 0000000000..ab5a974cdc --- /dev/null +++ b/integration-tests/src/chaos/deletions.rs @@ -0,0 +1,384 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that are built up and then deleted outright (GOL-372). +//! +//! One emitter per agent slot, running rounds. A round is +//! `increments_per_round` increments followed by one `delete`. Then the slot is +//! used again: invoking a deleted agent id creates a **new** agent, so the next +//! round's first increment returns `1` if the deletion took and `V + 1` if the +//! old agent is still there. +//! +//! That is the same probe [`crate::chaos::reverts`] uses and it works for the +//! same reason: the round's last increment reports the value, so there are +//! exactly two legal answers afterwards and nothing between them. +//! +//! ### The delete must never be retried +//! +//! `delete_worker_internal` in the executor starts with `get_latest_metadata` +//! and returns `worker_not_found` when there is nothing there. So deleting an +//! agent twice does **not** return success twice: the second call is an error. +//! +//! A delete whose response was lost but which actually landed would therefore, +//! on retry, come back as a refusal — and the driver would record "the platform +//! refused, and the agent is gone anyway", which is one of the violations this +//! scenario exists to detect. The retry is switched off for the delete alone, +//! exactly as it is for a revert, and left on for the increments around it. +//! +//! ### What the kill is aimed at +//! +//! Deleting is four steps: interrupt the running worker, `start_deleting`, +//! remove it from the worker service, remove it from the active set. Only the +//! third is durable. `start_deleting` exists to stop a background status flush +//! from — in the executor's own words — "resurrecting the cached status" after +//! the removal, so the happy path is already defended against exactly the thing +//! this scenario is named for. The question is whether that defence survives the +//! pod dying between the mark and the removal. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{DeleteConfig, RetryPolicy}; +use chrono::{DateTime, Utc}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// The counter a freshly created agent carries. +/// +/// Named rather than written as `0` at the comparison, because it is the whole +/// definition of "the deletion took": an agent id that was deleted and then +/// invoked again is a *new* agent, and a new counter starts from nothing. +pub const COUNTER_OF_A_NEW_AGENT: u64 = 0; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteRound { + pub agent: String, + pub round: u32, + /// The value the round's last increment returned, so what the agent was + /// worth immediately before it was deleted. `None` when an increment did + /// not answer, which leaves the round unjudgeable rather than failed. + pub before_delete: Option, + /// What the delete call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// Whether the platform refused this delete by saying the agent was not + /// there, as opposed to refusing it for any other reason. + /// + /// Load-bearing rather than diagnostic. Deleting is not idempotent, and + /// worker-service retries a call whose executor became unreachable — so a + /// delete that *succeeded* on a pod that then died comes back to the caller + /// as not-found. Without this flag that reads as "the platform refused and + /// the agent is gone anyway", which is one of this scenario's violations. + #[serde(default)] + pub rejected_as_not_found: bool, + /// The counter the deletion left behind: `0` if the id came back as a new + /// agent, `before_delete` if the old one is still there. + /// + /// One meaning, whichever probe produced it, which is the point. A round is + /// normally probed by the *next* round's first increment, and an increment + /// reports the counter it just raised — so the value it leaves behind is + /// that minus one. The last round of a run has no increment after it and is + /// closed by a plain read instead, which reports the counter directly. + /// Storing the increment's own return would make those two probes disagree + /// by one, and only ever on the last round of each slot. That is exactly + /// the bug the first S6 run reported 125 times. + pub observed_after: Option, +} + +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// One line, and it earns a name because getting it wrong is invisible. The +/// account judges `observed_after` as the counter a delete left behind, and the +/// two probes that produce it disagree by one: an increment reports the counter +/// it just raised, a plain read reports the counter itself. The first S6 run +/// stored the increment's own return and reported 125 partial-state findings — +/// exactly the slots whose last round was closed by a read. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + +/// A running deletion workload. +pub struct DeleteHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl DeleteHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos delete workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agent slots a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Delete, index)) + .collect() +} + +/// Builds one agent up, deletes it, and checks it came back new. +/// +/// Run once before the baseline, against a throwaway id. It exists because the +/// first S11 run did not have its equivalent: a scenario whose premise is wrong +/// spends its whole baseline before the numbers say so, and the maintenance +/// window is gone. If deletion does not behave the way this whole account +/// assumes, this fails in seconds with the platform's own error. +pub async fn smoke_round(ctx: &WorkloadContext, config: &DeleteConfig) -> anyhow::Result<()> { + let agent = format!("{}-delete-smoke", ctx.key_prefix); + let mut value = 0; + for step in 0..config.increments_per_round.max(1) { + value = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, step as u64), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| { + anyhow::anyhow!("smoke round: increment {step} on {agent} did not answer") + })?; + } + + let agent_id = workload::counter_agent_id(ctx, &agent); + ctx.user + .delete_worker(&agent_id) + .await + .map_err(|e| anyhow::anyhow!("smoke round: deleting {agent} failed: {e:#}"))?; + + let after = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, u64::from(config.increments_per_round) + 1), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| anyhow::anyhow!("smoke round: {agent} did not answer after being deleted"))?; + + if after != COUNTER_OF_A_NEW_AGENT + 1 { + anyhow::bail!( + "smoke round: {agent} was worth {value}, was deleted, and its next increment \ + returned {after} rather than {}. Deleting an agent does not behave the way this \ + scenario's whole account assumes, so the run would report a resurrection on every \ + round.", + COUNTER_OF_A_NEW_AGENT + 1 + ); + } + info!("S6: smoke round passed — a deleted agent came back as a new one"); + Ok(()) +} + +/// Starts one emitter per agent slot. +pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos delete workload starting: {} emitters, {} increments then a delete per round, \ + {:?} between rounds", + config.agents, + config.increments_per_round, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Delete, index); + let mut round: u32 = 0; + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = workload::increment_counter( + &ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, seq), + ) + .await + .value + .map(u64::from); + + // The first increment of a round says what the previous + // round's delete left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(counter_left_by(observed)); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let (outcome, rejected_as_not_found) = delete_once(&ctx, &agent, round).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(DeleteRound { + agent: agent.clone(), + round, + before_delete: all_answered.then_some(last_value).flatten(), + outcome, + rejected_as_not_found, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + DeleteHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// The error code the platform returns when there is nothing to delete. +const AGENT_NOT_FOUND: &str = "AGENT_NOT_FOUND"; + +/// One delete, with retries switched off. See the module docs. +/// +/// Returns the outcome and whether a refusal was specifically "no such agent", +/// which is the signature of a delete that had already taken effect. +async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> (Outcome, bool) { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-delete-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let ctx2 = once.clone(); + + let result = workload::run_operation( + &once, + Stream::Delete, + agent.to_string(), + "delete", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user.delete_worker(&agent_id).await?; + Ok(None) + } + }, + ) + .await; + + let not_found = result + .error + .as_deref() + .is_some_and(|e| e.contains(AGENT_NOT_FOUND)); + (result.outcome, not_found) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The conversion the first S6 run got wrong, and which no test covered + /// because it lived in a spawned task rather than in the account. + /// + /// `observed_after` means the counter the delete left behind. An increment + /// reports the counter it just raised, so it is one more than that; a plain + /// read reports the counter directly and needs no conversion. Both probes + /// have to land on the same scale or every slot's last round — the only one + /// closed by a read — reads as a partial state. + #[test] + fn the_two_probes_agree_on_what_a_delete_left_behind() { + // A deleted agent: the next increment creates it and reports 1. + assert_eq!(counter_left_by(1), COUNTER_OF_A_NEW_AGENT); + // A survivor worth 3: the next increment reports 4. + assert_eq!(counter_left_by(4), 3); + // A plain read of the same two cases reports 0 and 3 with no + // conversion, which is what `close_last_rounds` stores. + for counter in [COUNTER_OF_A_NEW_AGENT, 3] { + assert_eq!(counter_left_by(counter + 1), counter); + } + } + + /// An increment that somehow reported nothing must not underflow into a + /// value the account would read as a survivor. + #[test] + fn a_zero_increment_does_not_wrap_around() { + assert_eq!(counter_left_by(0), 0); + } +} diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs new file mode 100644 index 0000000000..621f325053 --- /dev/null +++ b/integration-tests/src/chaos/fires.rs @@ -0,0 +1,980 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-fire account (GOL-378). +//! +//! Every other oracle in this suite reduces to arithmetic over counts: the +//! driver submitted N, the durable state says M, and the gap between them is +//! the finding. That works because an increment is interchangeable with every +//! other increment. A scheduled action is not. The question S10 asks is whether +//! *this* action, claimed by an executor that then died, fired — once — and how +//! long the lease took to hand it to somebody else. +//! +//! So the target agent records a token per fire rather than a tally, and this +//! module pairs those tokens against the registrations the driver made. Pairing +//! is what makes the verdicts facts about a named action rather than a +//! judgement about a distribution, which is the same reason S8 probes keys +//! individually. +//! +//! ## What fails a run, and what only gets reported +//! +//! Three things fail it, and all three are statements about one token: +//! +//! - a **confirmed** registration whose action never fired +//! - a token that fired **more than once** +//! - a registration the platform **refused** that fired anyway +//! +//! Everything else is reported. In particular an *indeterminate* registration +//! that never fired is not a finding: the driver never learned whether the +//! registration landed, so an action that never fired is one of the two +//! legitimate answers. Those are counted, so a clean verdict over many of them +//! reads as weaker than a clean verdict with none. +//! +//! The same care applies to the read itself. A target whose fire log could not +//! be read, or whose log hit the component's cap, cannot testify about its own +//! registrations — those become unverifiable rather than lost. Reporting an +//! unreadable agent as lost work would turn a failed read into a correctness +//! defect, which is the exact mistake this suite exists to avoid. +//! +//! ## Delay, and why it is grouped the way it is +//! +//! Fire delay is `observed - scheduled`: how far past its due time the platform +//! actually ran the action. It is grouped two ways at once, because either +//! alone is misleading. +//! +//! By **window**, because an action due while the executor was gone is the only +//! one whose delay says anything about recovery. By **group**, because on a +//! two-executor cluster roughly half the targets were never on the pod that +//! died: mixing them in drags the percentile down until a lease recovery that +//! took its full TTL looks like a healthy p99. +//! +//! Delays are measured across two clocks — the driver mints the due time, the +//! executor stamps the fire — so a small negative delay is skew rather than an +//! action that fired early. `minDelayMs` is reported per group so that skew is +//! visible instead of silently folded into the percentiles. +//! +//! ### Actions that were late before the scheduler saw them +//! +//! The due time is minted *before* the registering invocation goes out, so a +//! registration that takes longer than the lead to complete describes an action +//! that was already overdue the moment it was accepted. The platform runs it +//! immediately and correctly, and the arithmetic still reports it as tens of +//! seconds late. +//! +//! This is not hypothetical: killing an executor stalls the registrations that +//! were in flight to it, and a stall longer than the lead puts every one of +//! those actions into the delay cells as tens of seconds late even though each +//! fired the instant it was accepted. Worse, it lands in *both* groups, because +//! emitters are spread across executors independently of their targets — so an +//! untouched control group can be made to look exactly as damaged as the killed +//! executor's. +//! +//! So they are separated. A fire whose registration completed after its due time +//! is counted as `overdueOnArrival` and kept out of the delay cells, because the +//! delay cells answer "how late was the scheduler" and this is the answer to +//! "how late was the client". Both are reported; neither is allowed to +//! impersonate the other. + +use crate::chaos::history::{FireRecord, OperationRecord, Outcome, Stream, TargetFireLog}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, TimeDelta, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A scenario that lost every action would otherwise produce tens of thousands +/// of them and an artifact nobody can open. The count is reported separately, +/// so truncation is stated rather than inferred from a suspiciously round +/// number of findings. +const MAX_FINDINGS: usize = 200; + +/// The fault window, as the workflow reported it. See +/// [`crate::chaos::split::FaultWindow`]. +pub type FaultWindow = crate::chaos::split::FaultWindow; + +/// Which side of the fault an action was due on. +/// +/// An alias rather than a type of its own: every scenario that reports against a +/// fault window asks the same question, and `fireWindow` is the key an archived +/// S10 result already carries. +pub type FireWindow = crate::chaos::split::Window; + +/// Whether an action's target was on the executor the fault was aimed at. +/// +/// The names say "killed" because every scenario that read this report until +/// S19 killed a pod, and they are the names in the archived results and in the +/// report generator that reads them. S19 aims a clock skew at an executor +/// instead, and its targets land in [`TargetGroup::OnKilledExecutor`] having +/// survived: the group means "on the faulted pod", and renaming it would cost a +/// schema bump and stop rendering every run already in the bucket. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TargetGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl TargetGroup { + pub fn as_str(self) -> &'static str { + match self { + TargetGroup::OnKilledExecutor => "on-killed-executor", + TargetGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for TargetGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What a token did that it should not have. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FireViolation { + /// A registration the platform accepted whose action never ran. + NeverFired, + /// One registration, two or more fires. + FiredMoreThanOnce, + /// A registration the platform definitively refused, whose action ran + /// anyway. + FiredDespiteRejection, +} + +impl FireViolation { + pub fn as_str(self) -> &'static str { + match self { + FireViolation::NeverFired => "never-fired", + FireViolation::FiredMoreThanOnce => "fired-more-than-once", + FireViolation::FiredDespiteRejection => "fired-despite-rejection", + } + } +} + +impl std::fmt::Display for FireViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one token. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireFinding { + pub violation: FireViolation, + pub token: String, + pub agent: String, + pub window: FireWindow, + pub detail: String, +} + +/// Fire delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireDelayStats { + pub group: TargetGroup, + pub window: FireWindow, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than an action that fired early. + pub min_delay_ms: i64, + /// Fires whose delay exceeded the configured lease budget. + pub over_budget: u64, +} + +/// The scheduled-fire account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduleFireReport { + /// What a lease recovery is allowed to cost, from the suite YAML. Recorded + /// so a percentile in an archived result can be read years later against + /// the number it was judged by rather than against today's config. + pub lease_budget_ms: u64, + pub registrations_confirmed: u64, + pub registrations_indeterminate: u64, + pub registrations_rejected: u64, + /// Fires the targets recorded, including any whose token is unknown. + pub fires_recorded: u64, + /// Accepted registrations paired with exactly one fire. + pub fired_once: u64, + /// Registrations the driver was never sure of that fired anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_fired: u64, + /// Registrations the driver was never sure of that never fired. Not a + /// finding: the registration may never have landed. + pub inconclusive: u64, + /// Registrations whose target could not testify, because its log was + /// unreadable or truncated. + pub unverifiable: u64, + /// Fires whose token no registration claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Targets the read-back could not reach at all. + pub targets_unreadable: Vec, + /// Targets whose fire log hit the component's cap. + pub targets_truncated: Vec, + pub delay: Vec, + /// Fires whose registration only landed after the action was already due, so + /// the platform could not have run them on time whatever the scheduler did. + /// Held out of [`Self::delay`] — see the module docs. + pub overdue_on_arrival: u64, + /// How late those were, for the record. This is client-side registration + /// latency wearing a scheduler's clothes. + pub overdue_delay: LatencyStats, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +impl ScheduleFireReport { + /// Pairs registrations against fires. + /// + /// `records` is the whole history; only the scheduled stream is considered. + /// `lead` is how far ahead registrations were made, used to say when an + /// action that never fired was due — the fire log is the only place the + /// exact due time survives, and an action that never fired left no entry. + /// `targets_on_faulted_pod` is whichever targets the fault was aimed at, + /// killed or otherwise — see [`TargetGroup`]. + pub fn build( + records: &[OperationRecord], + logs: &[TargetFireLog], + lead: Duration, + fault: Option, + targets_on_faulted_pod: &BTreeSet, + lease_budget: Duration, + ) -> Self { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let budget_ms = lease_budget.as_millis().min(u64::MAX as u128) as u64; + + let mut fires_by_token: BTreeMap<&str, Vec<&FireRecord>> = BTreeMap::new(); + let mut complete: BTreeMap<&str, bool> = BTreeMap::new(); + let mut targets_unreadable = Vec::new(); + let mut targets_truncated = Vec::new(); + let mut fires_recorded = 0u64; + + for log in logs { + complete.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + targets_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + targets_truncated.push(log.agent.clone()); + } + for fire in &log.fires { + fires_recorded += 1; + fires_by_token + .entry(fire.token.as_str()) + .or_default() + .push(fire); + } + } + + let mut report = Self { + lease_budget_ms: budget_ms, + registrations_confirmed: 0, + registrations_indeterminate: 0, + registrations_rejected: 0, + fires_recorded, + fired_once: 0, + indeterminate_that_fired: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + targets_unreadable, + targets_truncated, + delay: Vec::new(), + overdue_on_arrival: 0, + overdue_delay: LatencyStats::default(), + findings: Vec::new(), + findings_omitted: 0, + }; + + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let mut findings: Vec = Vec::new(); + // When each registration actually landed, which is what decides whether + // its action still had a chance of being on time. + let mut registered_at: BTreeMap<&str, DateTime> = BTreeMap::new(); + + for record in records.iter().filter(|r| r.stream == Stream::Scheduled) { + let token = record.idempotency_key.as_str(); + claimed.insert(token); + if let Some(completed) = record.completed_at { + registered_at.insert(token, completed); + } + let fires = fires_by_token.get(token).map(Vec::as_slice).unwrap_or(&[]); + // The due time comes from the fire itself when there is one, because + // that is what the platform was actually told. Without a fire the + // driver only knows what it asked for. + let due = fires + .first() + .map(|f| f.scheduled_at) + .unwrap_or(record.submitted_at + lead); + let window = FireWindow::of(due, fault); + let can_testify = complete + .get(record.agent.as_str()) + .copied() + .unwrap_or(false); + + match record.outcome { + Outcome::Confirmed => report.registrations_confirmed += 1, + Outcome::Indeterminate => report.registrations_indeterminate += 1, + Outcome::Rejected => report.registrations_rejected += 1, + } + + match (record.outcome, fires.len()) { + (_, count) if count > 1 => findings.push(FireFinding { + violation: FireViolation::FiredMoreThanOnce, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "one registration, {count} fires at {} — the action ran more than once", + fires + .iter() + .map(|f| f.observed_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }), + (Outcome::Rejected, 1) => findings.push(FireFinding { + violation: FireViolation::FiredDespiteRejection, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: "the platform refused the registration, then ran the action" + .to_string(), + }), + (Outcome::Confirmed, 1) => report.fired_once += 1, + (Outcome::Indeterminate, 1) => { + report.fired_once += 1; + report.indeterminate_that_fired += 1; + } + (Outcome::Confirmed, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Confirmed, 0) => findings.push(FireFinding { + violation: FireViolation::NeverFired, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "accepted registration due at {} never fired", + due.to_rfc3339() + ), + }), + (Outcome::Indeterminate, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Indeterminate, 0) => report.inconclusive += 1, + (Outcome::Rejected, 0) => {} + (_, _) => {} + } + } + + report.unknown_tokens = fires_by_token + .iter() + .filter(|(token, _)| !claimed.contains(*token)) + .map(|(_, fires)| fires.len() as u64) + .sum(); + + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + let (delay, overdue) = delay_stats( + logs, + fault, + targets_on_faulted_pod, + budget_ms, + ®istered_at, + ); + report.delay = delay; + report.overdue_on_arrival = overdue.len() as u64; + report.overdue_delay = + LatencyStats::from_durations(overdue.iter().map(|d| (*d).max(0) as u64).collect()); + report + } + + /// The three conditions that fail the scenario. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() + } + + /// The p99 of the cell that the SLO is about: actions due while the + /// executor was gone, on targets it owned. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|d| { + d.group == TargetGroup::OnKilledExecutor && d.window == FireWindow::DuringFault + }) + .map(|d| d.delay.p99_ms) + } + + /// Lines an operator should see next to the read-back verdicts. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + for finding in &self.findings { + lines.push(format!( + "{}: token {} on target {} ({}) — {}", + finding.violation, finding.token, finding.agent, finding.window, finding.detail + )); + } + if self.findings_omitted > 0 { + lines.push(format!( + "scheduled-fire findings are a sample: {} more were dropped", + self.findings_omitted + )); + } + if !self.targets_unreadable.is_empty() { + lines.push(format!( + "{} scheduled targets could not be read back, so {} registrations are \ + unverifiable rather than accounted for", + self.targets_unreadable.len(), + self.unverifiable + )); + } + if !self.targets_truncated.is_empty() { + lines.push(format!( + "{} scheduled targets filled their fire log and dropped entries — raise the \ + cadence or shorten the run before reading this as exactly-once evidence", + self.targets_truncated.len() + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "{} fires carried a token no registration claims, which should be impossible \ + within one run nonce", + self.unknown_tokens + )); + } + if let Some(p99) = self.fault_window_p99_ms() + && p99 > self.lease_budget_ms + { + lines.push(format!( + "scheduled-fire p99 during the fault was {p99}ms against a {}ms lease budget", + self.lease_budget_ms + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + /// + /// Overdue-on-arrival lives here rather than in [`Self::attention_lines`] + /// because it describes the client, not the platform: the due time is + /// minted before the registering invocation goes out, so a slow + /// registration describes an action that was already late when it arrived + /// and which the scheduler then ran correctly and at once. + /// + /// Read its count together with the worst case, never alone. The + /// classification catches anything slower than the lead, so a fix that + /// shortens a two-minute stall to twelve seconds moves entries *into* this + /// count while making the platform strictly better. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.overdue_on_arrival > 0 { + lines.push(format!( + "{} already overdue when the registration landed (worst {}ms late). That is \ + client-side registration latency, not scheduler delay, so they are excluded \ + from the delay percentiles and counted here instead", + if self.overdue_on_arrival == 1 { + "1 action was".to_string() + } else { + format!("{} actions were", self.overdue_on_arrival) + }, + self.overdue_delay.max_ms + )); + } + lines + } +} + +/// Delay percentiles per (group, window) cell, and the fires held out of them. +/// +/// A fire is held out when its registration completed after the action was +/// already due: nothing the scheduler did could have made it on time, so +/// counting it as scheduler delay would blame the platform for the client. +fn delay_stats( + logs: &[TargetFireLog], + fault: Option, + targets_on_faulted_pod: &BTreeSet, + budget_ms: u64, + registered_at: &BTreeMap<&str, DateTime>, +) -> (Vec, Vec) { + let mut cells: BTreeMap<(TargetGroup, FireWindow), Vec> = BTreeMap::new(); + let mut overdue: Vec = Vec::new(); + + for log in logs { + let group = if targets_on_faulted_pod.contains(&log.agent) { + TargetGroup::OnKilledExecutor + } else { + TargetGroup::Elsewhere + }; + for fire in &log.fires { + let late_on_arrival = registered_at + .get(fire.token.as_str()) + .is_some_and(|landed| *landed > fire.scheduled_at); + if late_on_arrival { + overdue.push(fire.delay_ms()); + continue; + } + cells + .entry((group, FireWindow::of(fire.scheduled_at, fault))) + .or_default() + .push(fire.delay_ms()); + } + } + + let cells: Vec = cells + .into_iter() + .map(|((group, window), delays)| { + let min = delays.iter().copied().min().unwrap_or(0); + let over_budget = delays.iter().filter(|d| **d > budget_ms as i64).count() as u64; + FireDelayStats { + group, + window, + delay: LatencyStats::from_durations( + delays.iter().map(|d| (*d).max(0) as u64).collect(), + ), + min_delay_ms: min, + over_budget, + } + }) + .collect(); + (cells, overdue) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(30); + + fn record( + token: &str, + agent: &str, + outcome: Outcome, + submitted_at: DateTime, + ) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Fault, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 12, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 12, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn fire(token: &str, scheduled: DateTime, delay_ms: i64) -> FireRecord { + FireRecord { + token: token.to_string(), + scheduled_at: scheduled, + observed_at: scheduled + TimeDelta::milliseconds(delay_ms), + } + } + + /// A log the agent answered in full. + fn log(agent: &str, fires: Vec) -> TargetFireLog { + TargetFireLog { + agent: agent.to_string(), + polls: Some(fires.len() as u64), + fires, + error: None, + } + } + + fn build(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The healthy shape: one registration, one fire. + #[test] + fn a_registration_paired_with_one_fire_is_not_a_finding() { + let r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 40)])], + ); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.fired_once, 1); + assert_eq!(report.registrations_confirmed, 1); + } + + /// Accepted work has to happen. This is the loss half of the guarantee. + #[test] + fn a_confirmed_registration_that_never_fired_fails_the_run() { + let r = record("t-1", "target-0", Outcome::Confirmed, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(report.has_violations()); + assert_eq!(report.findings[0].violation, FireViolation::NeverFired); + assert_eq!(report.findings[0].token, "t-1"); + assert!( + report.findings[0].detail.contains(&at(10).to_rfc3339()), + "the finding must say when the action was due: {}", + report.findings[0].detail + ); + } + + /// The duplicate half, and the reason the target records tokens rather than + /// a tally: a lease recovered while the original claim was still running + /// runs the action twice, and no count over a busy target would show it. + #[test] + fn a_token_that_fired_twice_fails_the_run() { + let r = record("t-2", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-2", at(10), 40), fire("t-2", at(10), 31_000)], + )], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredMoreThanOnce + ); + } + + /// A definite refusal means nothing was accepted, so an action that runs + /// anyway is the platform contradicting its own answer. + #[test] + fn a_refused_registration_that_fired_anyway_fails_the_run() { + let r = record("t-3", "target-0", Outcome::Rejected, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-3", at(10), 5)])], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredDespiteRejection + ); + } + + /// The distinction the whole verdict rests on: the driver never learned + /// whether this registration landed, so an action that never fired is one + /// of two legitimate answers. + #[test] + fn an_indeterminate_registration_that_never_fired_is_counted_rather_than_failed() { + let r = record("t-4", "target-0", Outcome::Indeterminate, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!( + report.inconclusive, 1, + "but it must be counted, so a clean verdict over many of them reads as weaker" + ); + } + + /// ...and when it did fire, the doubt resolved in the platform's favour. + #[test] + fn an_indeterminate_registration_that_fired_resolves_the_doubt() { + let r = record("t-5", "target-0", Outcome::Indeterminate, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-5", at(10), 20)])], + ); + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 1); + assert_eq!(report.indeterminate_that_fired, 1); + } + + /// A read that failed is not evidence about the platform. Turning it into + /// one would report a network problem as lost durable work. + #[test] + fn an_unreadable_target_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-6", "target-0", Outcome::Confirmed, at(0)); + let unreadable = TargetFireLog { + agent: "target-0".to_string(), + polls: None, + fires: Vec::new(), + error: Some("timed out after 30s".to_string()), + }; + let report = build(std::slice::from_ref(&r), &[unreadable]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_unreadable, vec!["target-0".to_string()]); + } + + /// Same reasoning for a log that filled up: the fire may have happened and + /// been dropped. `polls` is what makes that detectable at all. + #[test] + fn a_truncated_fire_log_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-7", "target-0", Outcome::Confirmed, at(0)); + let truncated = TargetFireLog { + agent: "target-0".to_string(), + // The agent fired 500 actions and kept 2 of them. + polls: Some(500), + fires: vec![fire("t-other", at(10), 5), fire("t-more", at(11), 5)], + error: None, + }; + let report = build(std::slice::from_ref(&r), &[truncated]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_truncated, vec!["target-0".to_string()]); + } + + /// The grouping that makes the percentile mean anything. Both cells are + /// reported; folding them together would let the untouched half hide what + /// the recovery cost. + #[test] + fn delay_is_split_by_whether_the_target_was_on_the_killed_executor() { + let fault = FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }; + let killed = BTreeSet::from(["target-killed".to_string()]); + let logs = vec![ + log("target-killed", vec![fire("a", at(150), 28_000)]), + log("target-survivor", vec![fire("b", at(150), 40)]), + ]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, Some(fault), &killed, BUDGET); + + let killed_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::OnKilledExecutor) + .expect("the killed executor's targets need their own cell"); + assert_eq!(killed_cell.window, FireWindow::DuringFault); + assert_eq!(killed_cell.delay.p99_ms, 28_000); + + let survivor_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::Elsewhere) + .expect("the control group needs its own cell"); + assert_eq!(survivor_cell.delay.p99_ms, 40); + + assert_eq!(report.fault_window_p99_ms(), Some(28_000)); + } + + /// Actions are placed by when they were *due*, not when they fired: an + /// action due during the outage that landed afterwards is exactly the + /// population the scenario is about. + #[test] + fn an_action_is_placed_in_the_window_its_due_time_fell_in() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }); + assert_eq!(FireWindow::of(at(99), fault), FireWindow::BeforeFault); + assert_eq!(FireWindow::of(at(100), fault), FireWindow::DuringFault); + // Due mid-fault, fired long after it healed: still a fault-window action. + assert_eq!(FireWindow::of(at(199), fault), FireWindow::DuringFault); + assert_eq!(FireWindow::of(at(200), fault), FireWindow::AfterFault); + assert_eq!(FireWindow::of(at(150), None), FireWindow::Unknown); + } + + /// A fault that never cleared leaves everything after injection inside it, + /// rather than inventing an end. + #[test] + fn a_fault_that_never_cleared_has_no_after_window() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: None, + }); + assert_eq!(FireWindow::of(at(10_000), fault), FireWindow::DuringFault); + } + + /// Two clocks measure this, so a small negative delay is skew. It stays + /// visible as a minimum instead of being folded into the percentiles. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let logs = vec![log( + "target-0", + vec![fire("a", at(10), -35), fire("b", at(11), 60)], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -35); + assert_eq!(cell.delay.p50_ms, 0, "the negative sample clamps to zero"); + assert_eq!(cell.delay.max_ms, 60); + } + + /// Fires over the lease budget are counted per cell, which is what turns a + /// percentile into SLO evidence rather than a number. + #[test] + fn fires_past_the_lease_budget_are_counted() { + let logs = vec![log( + "target-0", + vec![ + fire("a", at(10), 29_000), + fire("b", at(11), 31_000), + fire("c", at(12), 45_000), + ], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.delay[0].over_budget, 2); + } + + /// An artifact nobody can open is not evidence. Truncation is stated. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 25) + .map(|i| record(&format!("t-{i}"), "target-0", Outcome::Confirmed, at(0))) + .collect(); + let report = build(&records, &[log("target-0", vec![])]); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 25); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("are a sample")), + "an operator must be told the list is partial" + ); + } + + /// Agent names carry the run nonce, so this cannot happen within a run. If + /// it ever does, the pairing is answering a different question than it + /// thinks it is, and the report says so. + #[test] + fn a_fire_whose_token_no_registration_claims_is_counted() { + let r = record("t-8", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-8", at(10), 5), fire("stranger", at(10), 5)], + )], + ); + assert_eq!(report.unknown_tokens, 1); + assert!(!report.has_violations()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("no registration claims")) + ); + } + + /// Only the scheduled stream is paired: a durable operation carries an + /// idempotency key too, and pairing it against a fire log would invent + /// findings out of an unrelated stream. + #[test] + fn operations_from_other_streams_are_not_paired() { + let mut durable = record("t-9", "target-0", Outcome::Confirmed, at(0)); + durable.stream = Stream::Durable; + let report = build(std::slice::from_ref(&durable), &[log("target-0", vec![])]); + assert_eq!(report.registrations_confirmed, 0); + assert!(!report.has_violations()); + } + + /// The correction the first S10 run forced. + /// + /// An executor kill stalls the invocations in flight to it. Those + /// registrations land late, describing actions that were already overdue, + /// and the platform then runs them immediately and correctly. Counting that + /// as scheduler delay put 13 fires over budget in the killed executor's + /// targets and 13 in the control group — which is the tell, since the + /// control group's shards never moved. + #[test] + fn a_fire_whose_registration_landed_late_is_held_out_of_the_delay_cells() { + // Registered at t=0 for t=10, but the invocation only returned at t=125 + // after a stalled attempt and a retry. The action fires at once. + let mut r = record("stalled", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(125)); + r.duration_ms = 125_000; + let prompt = record("prompt", "target-0", Outcome::Confirmed, at(20)); + + let report = build( + &[r, prompt], + &[log( + "target-0", + vec![ + fire("stalled", at(10), 115_200), + fire("prompt", at(30), 1_100), + ], + )], + ); + + assert_eq!(report.overdue_on_arrival, 1); + assert_eq!(report.overdue_delay.max_ms, 115_200); + assert_eq!( + report.delay.iter().map(|d| d.delay.count).sum::(), + 1, + "only the registration that landed before its due time belongs in the cells" + ); + assert_eq!(report.delay[0].delay.max_ms, 1_100); + assert_eq!( + report.delay[0].over_budget, 0, + "the stalled registration must not put the scheduler over budget" + ); + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("client-side registration latency")), + "an operator has to be told why the count is held out" + ); + assert!( + report.attention_lines().is_empty(), + "holding a fire out of the cells is context, not a finding: CI raises an \ + annotation on attention, and a run where the platform did nothing wrong must \ + not raise one" + ); + // Still exactly-once: both actions ran once. + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 2); + } + + /// The boundary: landing exactly on the due time is not late. + #[test] + fn a_registration_that_landed_on_its_due_time_stays_in_the_cells() { + let mut r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(10)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 900)])], + ); + assert_eq!(report.overdue_on_arrival, 0); + assert_eq!(report.delay[0].delay.count, 1); + } + + /// The p99 an operator is judged on is the fault-window cell for the + /// targets that were on the dead pod. With no such cell there is nothing to + /// judge, and the report says nothing rather than substituting another. + #[test] + fn the_reported_p99_is_absent_when_the_fault_window_produced_no_fires() { + let report = ScheduleFireReport::build(&[], &[], LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.fault_window_p99_ms(), None); + } +} diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 9a00ab5185..c160b4d14a 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -67,6 +67,45 @@ pub enum Stream { /// one known executor, so mixing them into the durable population would /// blur two different experiments. PinnedHttp, + /// `Counter.increment` rounds on agents that are then deleted outright + /// (GOL-372). + /// + /// Excluded from the generic read-back for the same reason as + /// [`Stream::Revert`], one step further: a deleted agent's counter is + /// *supposed* to be gone, so comparing it against everything confirmed + /// against it would report the whole agent as lost work. See + /// [`crate::chaos::resurrection`]. + Delete, + /// `Counter.increment` rounds that are deliberately taken back again, and + /// the `revert` calls that take them (GOL-371). + /// + /// Distinct from [`Stream::Durable`] even though both land on `Counter` + /// agents, and for a reason that is not cosmetic: a reverted increment is + /// acknowledged work that the platform was then *asked* to forget, so the + /// generic read-back — which compares a counter against everything + /// confirmed against it — would report every one of them as lost. S7 + /// computes each agent's exact expected value from its own round history + /// instead, which is strictly stronger than a range. + Revert, + /// `PromiseWaiter.arm` / `wait` / the external completion that resolves it + /// (GOL-377). Distinct from `Promise` even though both land on the promise + /// component: that stream creates and resolves a promise in one breath with + /// nobody suspended on it, and this one exists precisely to leave an agent + /// parked across a fault. + PromiseWait, + /// `Counter.increment_through_rpc` — one agent invoking another through + /// `golem:rpc` (GOL-368). + /// + /// The only stream whose work crosses an *executor* boundary of its own + /// accord. Every other stream is one client call landing on one agent; here + /// the caller agent is itself a client, and the agent it calls is chosen so + /// that a different executor owns it. + /// + /// What it records is the caller. The counter it advances belongs to the + /// callee, which the counters component derives as `{caller}-inner` — see + /// [`crate::chaos::scenarios::ReadKind::RpcInner`] for why read-back keeps + /// the two apart rather than recording the callee directly. + Rpc, } impl Stream { @@ -78,6 +117,10 @@ impl Stream { Stream::Promise => "promise", Stream::Quota => "quota", Stream::PinnedHttp => "pinned-http", + Stream::PromiseWait => "promise-wait", + Stream::Rpc => "rpc", + Stream::Delete => "delete", + Stream::Revert => "revert", } } @@ -91,20 +134,29 @@ impl Stream { /// - `Promise` operations resolve a one-shot promise rather than advancing a /// counter, so there is no accumulated number to read. They are reported /// on created/completed counts and latency. + /// - `PromiseWait` agents *do* keep a durable count, but comparing totals + /// would be strictly weaker than what S11 already does with them: every + /// completion carries a token into the waiter's wakeup log, so the report + /// pairs individual completions against individual wakeups instead of + /// arguing about sums. See [`crate::chaos::wakeups`]. pub fn has_readback(self) -> bool { matches!( self, - Stream::Durable | Stream::Scheduled | Stream::PinnedHttp | Stream::Quota + Stream::Durable | Stream::Scheduled | Stream::PinnedHttp | Stream::Quota | Stream::Rpc ) } - pub const ALL: [Stream; 6] = [ + pub const ALL: [Stream; 10] = [ Stream::Durable, Stream::Ephemeral, Stream::Scheduled, Stream::Promise, Stream::Quota, Stream::PinnedHttp, + Stream::PromiseWait, + Stream::Revert, + Stream::Delete, + Stream::Rpc, ]; } @@ -312,6 +364,88 @@ impl OperationRecord { pub fn had_successful_attempt(&self) -> bool { self.outcome == Outcome::Confirmed || self.attempt_log.iter().any(|a| a.succeeded) } + + /// Attempts at this operation that hit the client's attempt timeout rather + /// than answering. + /// + /// Matched on the message [`crate::chaos::workload`] writes for a timed-out + /// attempt. A structured flag would be better, but the attempt log is an + /// archived shape that older results already carry, and this reads it + /// without changing it. + /// + /// The distinction it draws is the one a stalled dependency needs: an + /// operation that timed out and then answered on its retry was rescued by + /// the caller, not returned by the platform, and the outcome alone cannot + /// say so. + pub fn attempts_timed_out(&self) -> u64 { + self.attempt_log + .iter() + .filter(|a| { + a.error + .as_deref() + .is_some_and(|e| e.contains("attempt timed out")) + }) + .count() as u64 + } +} + +/// What a scheduled action recorded when it ran, and the log it was recorded +/// in (GOL-378). +/// +/// These live here rather than in [`crate::chaos::fires`] for the same reason +/// [`OperationRecord`] does: they are what was *observed*, and the analysis +/// that reduces them is a separate thing that a later ticket may want to redo +/// over an archived run. The first S10 run learned that the expensive way — its +/// delay percentiles turned out to need a correction that could not be applied +/// afterwards, because only the reduced numbers had been archived. +/// +/// This one is a single fire, as the target agent recorded it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireRecord { + /// The registering invocation's idempotency key. + pub token: String, + /// When the action was due, as the driver asked for it. + pub scheduled_at: DateTime, + /// When the platform ran it, as the executor's clock saw it. + pub observed_at: DateTime, +} + +impl FireRecord { + /// How far past its due time the action ran. Negative means clock skew + /// between the driver and the executor, not an action that fired early. + pub fn delay_ms(&self) -> i64 { + (self.observed_at - self.scheduled_at).num_milliseconds() + } +} + +/// Everything read back from one target agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetFireLog { + pub agent: String, + /// `ScheduleCounter.polls`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub polls: Option, + pub fires: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl TargetFireLog { + /// Whether this log can testify about its own registrations. + /// + /// Two ways it cannot: the read failed outright, or the component's fire log + /// hit its cap and dropped entries. Both leave an absent fire ambiguous. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.polls) { + (true, _) => false, + (false, Some(polls)) => self.fires.len() as u64 >= polls, + // No `polls` read means no way to tell whether the log is whole. + (false, None) => false, + } + } } /// The persisted history document. @@ -324,6 +458,76 @@ pub struct HistoryDocument { /// so a reader never mistakes a partial history for a short one. pub partial: bool, pub operations: Vec, + /// Per-target fire logs, for the scenarios that drive scheduled actions. + /// Empty for every other scenario rather than absent, so a reader never + /// wonders whether the section was dropped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scheduled_fires: Vec, + /// Per-waiter wakeup logs, for the scenarios that park agents on promises. + /// Empty for every other scenario rather than absent, for the same reason as + /// `scheduled_fires`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub promise_wakeups: Vec, +} + +/// One wakeup, as the waiter agent recorded it (GOL-377). +/// +/// The times are the *cluster's*, stamped inside the agent, which is what makes +/// this log the authority on whether a completion landed. The driver's own view +/// is in the operation record for the `wait` invocation, and during the fault +/// that view is frequently just a broken connection. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupRecord { + /// The round's idempotency key, carried in by `arm` and back out by the + /// wakeup. This is what pairs a completion to the wakeup it caused. + pub token: String, + /// When the waiter armed the promise. + pub armed_at: DateTime, + /// When the platform resumed the waiter. + pub woken_at: DateTime, +} + +impl WakeupRecord { + /// How long the waiter was parked, on one clock. + /// + /// Both ends are stamped by the executor, so this is free of the driver ↔ + /// cluster skew that the completion-to-wakeup delay carries. It is not the + /// delay itself — it also contains the round's deliberate dwell — but it is + /// what lets a reader tell a slow wakeup from a skewed clock. + pub fn parked_ms(&self) -> i64 { + (self.woken_at - self.armed_at).num_milliseconds() + } +} + +/// Everything read back from one waiter agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WaiterWakeupLog { + pub agent: String, + /// `PromiseWaiter.wakes`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub wakes: Option, + pub wakeups: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl WaiterWakeupLog { + /// Whether this log can testify about its own completions. + /// + /// Same two ways it cannot as [`TargetFireLog::is_complete`], and the same + /// consequence: without a whole log an absent wakeup is ambiguous between a + /// lost completion and a dropped log entry, and S11 must not call the second + /// one a finding. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.wakes) { + (true, _) => false, + (false, Some(wakes)) => self.wakeups.len() as u64 >= wakes, + (false, None) => false, + } + } } /// Append-only operation log, shared across the concurrent workload streams. @@ -335,6 +539,8 @@ pub struct OperationHistory { scenario_code: String, inner: Arc>>, next_id: Arc, + fire_logs: Arc>>, + wakeup_logs: Arc>>, } impl OperationHistory { @@ -343,6 +549,8 @@ impl OperationHistory { scenario_code: scenario_code.into(), inner: Arc::new(Mutex::new(Vec::new())), next_id: Arc::new(std::sync::atomic::AtomicU64::new(0)), + fire_logs: Arc::new(Mutex::new(Vec::new())), + wakeup_logs: Arc::new(Mutex::new(Vec::new())), } } @@ -383,12 +591,27 @@ impl OperationHistory { .count() as u64 } + /// Archives the fire logs read back at the end of a scheduled scenario. + /// + /// Called once, after read-back, before the history is written. Kept here + /// rather than in the result because the result is the reduced report and + /// this is the raw material it was reduced from. + pub fn record_fire_logs(&self, logs: Vec) { + *self.fire_logs.lock().unwrap() = logs; + } + + pub fn record_wakeup_logs(&self, logs: Vec) { + *self.wakeup_logs.lock().unwrap() = logs; + } + pub fn document(&self, partial: bool) -> HistoryDocument { HistoryDocument { schema_version: HISTORY_SCHEMA_VERSION, scenario_code: self.scenario_code.clone(), partial, operations: self.snapshot(), + scheduled_fires: self.fire_logs.lock().unwrap().clone(), + promise_wakeups: self.wakeup_logs.lock().unwrap().clone(), } } diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 2404c937a8..ab14d7ef40 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -18,8 +18,10 @@ //! Cloud-mode Golem while a bounded fault is injected, then reports what //! happened. The suite's shape follows density: the workflow drives one scenario //! per invocation, each scenario is independently selectable through a YAML -//! `enabled` flag, and results are archived to S3 per scenario so an interrupted -//! run resumes rather than restarts. +//! `enabled` flag, and results are archived to the `golem-bench-results` bucket +//! per scenario so an interrupted run resumes rather than restarts. +//! +//! `S3` in this module always means the scenario code, never the bucket. //! //! Two boundaries define this module: //! @@ -32,18 +34,36 @@ //! engine here — see [`summary`] for what is measured and the narrow set of //! conditions that fail a run outright. +pub mod composed; +pub mod deletions; pub mod errors; +pub mod fires; pub mod history; +pub mod outage; pub mod ownership; pub mod pinned; pub mod prep; pub mod probe; +pub mod reachability; +pub mod relay; +pub mod resolution; pub mod result; +pub mod resurrection; +pub mod reverts; +pub mod rollback; pub mod scenarios; +pub mod scheduled; pub mod signal; +pub mod skew; +pub mod split; +pub mod steady; pub mod summary; +pub mod truncation; +pub mod waiters; +pub mod wakeups; pub mod workload; +use crate::chaos::history::Stream; use anyhow::Context; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -63,6 +83,126 @@ pub enum ScenarioCode { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, + /// Two executors cut off from each other while agents on one invoke agents + /// on the other. + /// + /// The control of the set, and the only scenario whose fault is expected to + /// change nothing. Executors hold no connection to each other: an + /// agent-to-agent call the local executor does not own is handed to + /// `worker_proxy`, which is a client of *worker-service*, not of the + /// executor that owns the callee. So the link this partition cuts carries + /// no traffic, and a run that degrades is reporting a path the architecture + /// says does not exist. + S2, + /// One executor unable to resolve the shard manager's name, while its + /// connection to it stays up. + /// + /// The second control of the set. The executor builds that connection once + /// with an infinite idle TTL, so DNS is consulted at connect time and never + /// again — a name that stops resolving should reach nothing. MF2 is the + /// companion that removes the cached connection first. + S4, + /// S4's DNS failure held across a shard-manager restart, so the executor + /// has to rebuild the connection and resolve a name that no longer + /// resolves. + /// + /// The composition is what makes it a different question. Both executors + /// lose the connection to the same restart; only one of them can resolve + /// the name to get it back, so the gap between them is the DNS failure with + /// everything else held constant. + MF2, + /// Executor pod kill while agents are having their state reverted. + S7, + /// Executor pod kill while agents are being deleted. + S6, + /// Executor pod kill while a component rollback is in flight. + S9, + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. + S16, + /// The same cut, held for longer than the key-value retry budget, so the + /// executors are expected to be replaced rather than to ride it out. + S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster, the other + /// Aurora cluster underneath them, for the length of a writer failover. + S14, + /// Executors cut off from the Redis cache that fronts the key-value layer, + /// for longer than a caller is willing to wait. + S18, + /// The same Redis cache still reachable but slowed, to ask whether the + /// platform degrades or breaks when its worker-status store gets slower. + S17, + /// The key-value PostgreSQL cluster still reachable but slowed. S17's + /// mirror on the other half of the split key-value layer. + S15, + /// S15 driving `ephemeral` alone. The control: that stream reaches none of + /// the delayed cluster, so nothing should move. + S15A, + /// S15A plus `durable`, the heaviest user of the delayed cluster. + S15B, + /// S15B plus `promise`. The remaining step, `scheduled`, is S15 itself. + S15C, + /// The indexed-oplog PostgreSQL cluster still reachable but slowed. S14's + /// mirror, and the last empty cell of the storage matrix. + /// + /// The one delay in the suite with no control stream. Every agent commits + /// its oplog to this cluster, `ephemeral` included: it never opens the + /// primary, but the first layer of the stack it does open is a compressed + /// archive on the same indexed storage. + S23, + /// The first composed fault: a worker-executor killed while the key-value + /// PostgreSQL cluster is unreachable. + /// + /// Not a harder S16 and not a slower S8. Each of those two faults on its + /// own leaves the platform a way out that the other one takes away: S16 + /// keeps every executor, so the shards nobody can read are at least still + /// owned by a process that will read them when the store comes back, and S8 + /// takes an executor away while the store that records the handover is + /// working. Held together, an executor's shards have to move to a survivor + /// that cannot reach the running-workers set, the promises or the + /// scheduler's schema. + MF1, + /// MF1 with the kill early enough that the shard-manager's `assign_shards` + /// call runs out of time before the storage comes back. + /// + /// The pair to MF1 rather than a harder version of it, the way S22 is to + /// S16. `ASSIGN_SHARDS_TIMEOUT` is 60s on golem-dev and golem-prod, raised + /// from the 5s default in August 2026 because 5s could not cover a + /// reassignment whose handler recovers every agent it was just handed. MF1 + /// keeps its overlap under that ceiling and asks whether the platform rides + /// the outage out. This one puts the overlap past it and asks what happens + /// when the call the whole handover depends on gives up mid-outage. + MF1B, + /// Worker-service CPU saturation while agents drive cross-pod RPC. + /// + /// S2's populations with S2's fault taken away and a different one put in + /// its place. S2 cut the link between two executors and expected nothing to + /// happen, because an agent-to-agent call the local executor does not own + /// is relayed through *worker-service*. This one saturates the CPU of the + /// relay itself, and expects the opposite. + /// + /// The measurement is the same premium S2 discovered it needed. Every call + /// in the workload crosses worker-service once; a cross-pod call crosses it + /// twice. So the gap between the two populations is exactly one + /// worker-service hop, and starving worker-service of CPU should widen it. + /// A run where both populations slow down by the same amount saturated + /// something else. + S21, + /// One executor's wall clock moved half a minute behind the rest of the + /// cluster. + /// + /// A uniformly wrong clock is invisible to every comparison a pod makes + /// between two of its own readings, so the fault can only be seen where one + /// machine's clock is judged against another's. On this platform that is the + /// quota lease: the shard-manager mints `expires_at` and the executor decides + /// against it when to renew and whether the lease is already dead. + S19, } impl ScenarioCode { @@ -73,18 +213,64 @@ impl ScenarioCode { ScenarioCode::S5 => "S5", ScenarioCode::S12 => "S12", ScenarioCode::S13 => "S13", + ScenarioCode::S10 => "S10", + ScenarioCode::S11 => "S11", + ScenarioCode::S3 => "S3", + ScenarioCode::S2 => "S2", + ScenarioCode::S4 => "S4", + ScenarioCode::MF2 => "MF2", + ScenarioCode::S7 => "S7", + ScenarioCode::S6 => "S6", + ScenarioCode::S9 => "S9", + ScenarioCode::S16 => "S16", + ScenarioCode::S22 => "S22", + ScenarioCode::S14 => "S14", + ScenarioCode::S18 => "S18", + ScenarioCode::S17 => "S17", + ScenarioCode::S15 => "S15", + ScenarioCode::S15A => "S15A", + ScenarioCode::S15B => "S15B", + ScenarioCode::S15C => "S15C", + ScenarioCode::S23 => "S23", + ScenarioCode::MF1 => "MF1", + ScenarioCode::MF1B => "MF1B", + ScenarioCode::S21 => "S21", + ScenarioCode::S19 => "S19", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 5] = [ + pub const ALL: [ScenarioCode; 28] = [ ScenarioCode::S1, + ScenarioCode::S2, + ScenarioCode::S3, + ScenarioCode::S4, ScenarioCode::S5, + ScenarioCode::S6, + ScenarioCode::S7, ScenarioCode::S8, + ScenarioCode::S9, + ScenarioCode::S10, + ScenarioCode::S11, ScenarioCode::S12, ScenarioCode::S13, + ScenarioCode::S14, + ScenarioCode::S15, + ScenarioCode::S15A, + ScenarioCode::S15B, + ScenarioCode::S15C, + ScenarioCode::S16, + ScenarioCode::S17, + ScenarioCode::S18, + ScenarioCode::S19, + ScenarioCode::S21, + ScenarioCode::S22, + ScenarioCode::S23, + ScenarioCode::MF1, + ScenarioCode::MF1B, + ScenarioCode::MF2, ]; pub fn parse(s: &str) -> Option { @@ -172,6 +358,17 @@ pub struct FaultConfig { /// half the executors" is only meaningful next to how many that was. #[serde(default, skip_serializing_if = "Option::is_none")] pub target_count: Option, + /// The Chaos Mesh manifest to apply, by file name, when it is not the one + /// the workflow would find from the scenario code. + /// + /// Exists for the elimination variants. S15A, S15B and S15C inject exactly + /// the fault S15 injects and differ only in which streams the workload + /// drives, so copying `networkchaos-s15.yaml` three times would make four + /// files that must be kept identical by hand — and a run whose fault + /// silently diverged from the run it is being compared against is worse + /// than no run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest: Option, pub duration_secs: u64, } @@ -215,9 +412,23 @@ pub struct WorkloadConfig { /// shard-manager↔executor traffic; see [`history::Stream::Quota`]. #[serde(default)] pub quota_agents: u32, + /// Caller agents that invoke another agent through `golem:rpc`. Zero for + /// scenarios that do not need agent-to-agent traffic; see + /// [`history::Stream::Rpc`]. + /// + /// Each caller implies a second agent, its callee, so a run configured with + /// `n` of these creates `2n` agents rather than `n`. + #[serde(default)] + pub rpc_agents: u32, /// Combined submission rate across all streams, in operations per second. /// The project caps this at 25% of measured per-pod capacity so the run /// measures fault recovery rather than saturation. + /// + /// The cap is a margin for faults that *remove* capacity: S13 kills + /// executors, so a rate comfortable at two of them has to stay survivable + /// at one. A scenario whose fault removes none may sit above it, provided + /// its suite entry says why. S21 does, because a CPU stress cannot saturate + /// a service that is spending 1% of a core. pub rate_per_sec: u32, } @@ -301,6 +512,779 @@ fn default_candidate_pool_multiplier() -> u32 { 8 } +/// Shape of the scheduled-registration workload (GOL-378). +/// +/// A third experiment shape next to [`WorkloadConfig`] and [`PinnedConfig`], not +/// a variation on either. The mixed workload asks what a stream of invocations +/// does when the platform is disturbed, and the pinned workload asks what +/// happens to specific invocations that were running on the pod that died. This +/// one asks about work the driver is not holding a connection to at all: an +/// action the platform promised to run later, whose executor died in between. +/// +/// The two numbers that decide whether the run measures anything are `leadSecs` +/// and `intervalMillis`. See [`crate::chaos::scheduled`] for why. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduledConfig { + /// Target agents, each with its own emitter. Also the resolution of the + /// report: a finding localises to one target out of this many. + pub targets: u32, + /// Milliseconds between registrations on one target. The offered rate is + /// `targets / interval`. + pub interval_millis: u64, + /// How far ahead each action is registered. With the cadence above this + /// sets how many actions stand accepted but not yet run at any instant, + /// which is the population a kill has to land in the middle of. + pub lead_secs: u64, + /// What recovering a scheduled action is allowed to cost, which the + /// fire-delay percentiles are reported against. An SLO the run records + /// rather than a threshold it fails on: the floor is a shard reassignment, + /// or the executor's `lease_ttl` for the rarer case of an action that was + /// already claimed, and how much more than that is acceptable is a + /// judgement. + pub lease_budget_secs: u64, +} + +impl ScheduledConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn lead(&self) -> Duration { + Duration::from_secs(self.lead_secs) + } + pub fn lease_budget(&self) -> Duration { + Duration::from_secs(self.lease_budget_secs) + } +} + +/// Shape of the suspended-waiter workload (GOL-377). +/// +/// The fourth experiment shape, and the only one whose agents are *asleep* when +/// the fault lands. [`ScheduledConfig`] leaves work with the platform and walks +/// away; this one leaves an agent parked mid-invocation on a promise, so the +/// thing that has to survive the kill is not a queued action but a suspended +/// worker and the completion on its way to it. +/// +/// The number that decides whether the run measures anything is `waiters`: each +/// one holds exactly one promise at a time, so the pool size *is* the population +/// standing parked at the instant the pod dies. `dwellMillis` decides how much +/// of that population is also mid-completion — see [`crate::chaos::waiters`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PromiseConfig { + /// Waiter agents, each parked on at most one promise at a time. Also the + /// resolution of the report: a finding localises to one waiter out of this + /// many. + pub waiters: u32, + /// How long a waiter stays parked before its promise is completed. + /// + /// Sets the completion rate — `waiters / dwell` — and, with it, how many + /// completions are genuinely in flight when the pod dies. It has to + /// comfortably exceed the workflow's inject-and-verify path (signal poll, + /// `kubectl apply`, waiting for `AllInjected`) or every promise armed before + /// the kill would already have been completed by the time it landed. + pub dwell_millis: u64, + /// What resuming a suspended waiter is allowed to cost, which the wakeup + /// delay percentiles are reported against. An SLO the run records rather + /// than a threshold it fails on: the floor is a shard reassignment plus the + /// worker recovery that replays the waiter's oplog, and how much more than + /// that is acceptable is a judgement. + pub wakeup_budget_secs: u64, +} + +impl PromiseConfig { + pub fn dwell(&self) -> Duration { + Duration::from_millis(self.dwell_millis) + } + pub fn wakeup_budget(&self) -> Duration { + Duration::from_secs(self.wakeup_budget_secs) + } +} + +/// Shape of the reachability workload (GOL-370). +/// +/// The fifth experiment shape, and the only one where nothing about the +/// platform is broken at all. [`PinnedConfig`] asks what happens to operations +/// running on a pod that dies; this one asks what happens to operations bound +/// for a pod that is perfectly healthy and simply cannot be reached from the +/// tier that routes to it. The executor keeps its shards for the whole fault, +/// because the link it needs in order to keep them — to the shard-manager — is +/// not the one that was cut. +/// +/// Every agent gets its own emitter holding at most one operation, rather than +/// the shared per-stream budget [`WorkloadConfig`] drives. That is load-bearing +/// here and not a style choice: the stall this scenario induces is bounded by +/// *which executor owns the agent*, not by which stream it belongs to, so a +/// shared budget would be drained by the isolated half and would stop the +/// reachable half submitting too. The run would then report the control group +/// degrading, and the cause would be the driver. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IsolationConfig { + /// Durable counter agents, split by shard ownership into the ones the + /// isolated executor holds and the ones it does not. Also the resolution of + /// the report: a finding localises to one agent out of this many. + pub agents: u32, + /// Milliseconds between one agent's operations, measured from the end of + /// the previous one. The offered rate is `agents / interval`, and an agent + /// whose operation is stalled offers nothing at all — which is the signal, + /// not a gap in it. + pub interval_millis: u64, + /// The most of its own baseline throughput the isolated group may keep + /// during the fault, as a percentage, for the partition to count as + /// observed. + /// + /// A run above this line did not cut the executor off, whatever the fault + /// status says, and every other number in the report is then a measurement + /// of an undisturbed cluster. That is reported as inconclusive rather than + /// clean: a healthy-looking result from a fault that never landed is the + /// worst artifact this suite can produce. + pub isolated_ceiling_percent: f64, + /// The least of its own baseline throughput the control group must keep + /// during the fault, as a percentage. + /// + /// The sharpest thing S3 can find. The agents on the reachable executor + /// have nothing to do with the partition, so a drop here is collateral + /// damage from how worker-service handles an unreachable pod — its routing + /// table is one process-wide entry, and every stalled caller invalidating it + /// costs every other caller a shard-manager round trip. + pub control_floor_percent: f64, + /// What resuming an isolated agent may cost once the link is back, and the + /// number the recovery gap is reported against. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +/// Latency may sit a tenth above its baseline after a heal without the run +/// calling the fault unbounded: the recovery window starts the instant the +/// fault clears, so its first minute is legitimately still draining. +fn default_recovery_floor_percent() -> f64 { + 110.0 +} + +/// Shape of a scenario built on the cross-pod RPC split (GOL-368, GOL-382). +/// +/// Two scenarios share these numbers and disagree about what they should say. +/// S2 drives agent-to-agent calls whose two halves live on different executors, +/// cuts those executors off from each other, and asserts that nothing moves: +/// the claim under test is architectural, that an executor reaches another +/// executor's agents only by asking worker-service. S21 keeps the same +/// populations and saturates worker-service instead, and asserts that the gap +/// between them widens. +/// +/// [`RelayExpectation`] is which of those two a run is, and it decides which +/// numbers below are findings and which are context. Everything here is shared +/// because the *measurement* is shared; only the verdict differs. +/// +/// A scenario is only worth running if it can fail, so most of these exist to +/// stop one passing for the wrong reason. A run where the pairs turned out to +/// be co-located would report "no degradation" under either expectation, and +/// that report would be worthless in both. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelayConfig { + /// The least share of configured callers that must have their callee on the + /// *other* executor, as a percentage, for the run to mean anything. + /// + /// Pairing is not something the driver can arrange. Both halves are placed + /// by hashing their agent ids onto shards, so the split is whatever the + /// hash gives — around half on a two-executor cluster, but not guaranteed. + /// Below this line the fault has almost nothing to cut and the run aborts + /// rather than reporting a clean result it did not earn. Same instinct as + /// S9's forward-leg gate. + pub cross_pod_floor_percent: f64, + /// The least of its own baseline throughput the cross-pod population must + /// keep during the fault, as a percentage. + /// + /// This is the assertion, and it is deliberately high. Every other scenario + /// sets a floor low enough to survive the disruption it injects; here there + /// is nothing to survive, so a real drop means an executor was talking to + /// an executor. + pub cross_pod_floor_throughput_percent: f64, + /// The same floor for the pairs that happened to land on one executor. + /// + /// They never leave the pod, so the partition cannot reach them even in + /// principle. Kept as a separate number because if *both* populations drop + /// the cause is not the link under test — it is the fault disturbing the + /// cluster some other way, and the run should say so rather than blame the + /// architecture. + pub co_located_floor_throughput_percent: f64, + /// The least a cross-pod call must cost over a co-located one at p50 on the + /// undisturbed baseline, in milliseconds. + /// + /// The guard against the scenario being vacuous. Throughput cannot tell the + /// two populations apart, because the driver sets the cadence and both run + /// at the rate they were asked to. Latency can: a cross-pod call pays + /// executor -> worker-service -> executor and a co-located one does not, so + /// a premium at least this large is what says the split is real. + /// + /// Small on purpose. The first run measured 50ms, but this is a floor + /// against *zero*, not a model of what the hop should cost, and a cluster + /// with faster links should not fail for being fast. + pub cross_pod_premium_floor_ms: u64, + /// How far above its baseline latency may still sit after the heal before + /// the run reports [`relay::RelayViolation::RelayDidNotRecover`]. Read + /// under [`relay::RelayExpectation::RelayDegraded`] only, where 100 is a + /// p50 that returned exactly. + /// + /// The only size this scenario judges, and deliberately the only one. It + /// carried two floors demanding the fault hurt by some minimum, and both + /// were wrong the same way: the question is what a saturated relay costs, + /// and a floor turns "the platform coped" into a failure of the test. They + /// were guesses too — the premium floor asked for 150% against a measured + /// 120%, and the run that cleared it turned out to have caught a bimodal + /// baseline on the right side of its swing. + /// + /// What the fault cost is a number to report. A bounded fault whose cost + /// outlives it is a finding. + /// + /// Defaulted so an inert scenario need not carry a number it never reads. + #[serde(default = "default_recovery_floor_percent")] + pub recovery_floor_percent: f64, + /// Whether this run's fault is supposed to reach the relay. + /// + /// Defaults to [`RelayExpectation::Inert`], which is what S2 has always + /// been. A scenario that expects damage has to say so, and saying so is + /// what turns the checks below from assertions into recorded context. + #[serde(default)] + pub expectation: relay::RelayExpectation, +} + +/// Shape of the clock-skew scenario (GOL-383). +/// +/// Three numbers and a count, and every one of them exists to stop the run +/// passing for the wrong reason. A clock skew is the least self-evident fault in +/// the suite: it changes nothing a pod can see about itself, so a run whose +/// injection silently failed looks exactly like a run whose injection landed and +/// did no harm. The tolerance and the probe count are what separate those two. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkewConfig { + /// What the workflow asks Chaos Mesh to do to the pod's clock, in + /// milliseconds. Negative for a clock set behind, which is the only + /// direction that produces a disagreement — see [`skew`]. + /// + /// Mirrored here rather than read from the manifest because the driver never + /// touches Kubernetes. It is what the report is judged against, so a value + /// that disagrees with the manifest fails the run on + /// [`skew::SkewViolation::ClockNeverMoved`] rather than passing quietly. + pub injected_offset_ms: i64, + /// How far the measured offset may sit from the injected one before the run + /// is treated as having failed to inject anything. + /// + /// Absorbs the probe's own round trip and the ordinary difference between a + /// GitHub runner's clock and an EC2 node's. It has to stay well under the + /// offset itself: a tolerance at or above it is met by a run where the clock + /// never moved at all, which is the one thing this number exists to catch. + pub tolerance_ms: i64, + /// Percentage of its own baseline the quota stream's post-fault p50 may + /// reach before the run reports + /// [`skew::SkewViolation::QuotaDidNotRecover`]. + /// + /// Losing a lease under skew is a legitimate response and is only recorded. + /// Never getting it back is not. + #[serde(default = "default_recovery_floor_percent")] + pub recovery_floor_percent: f64, + /// Clock readings to take per group, per round. + /// + /// A round is a handful rather than one, because the reading is a median and + /// a single probe that caught a slow invocation would decide the run on its + /// own. + pub probes_per_round: u32, +} + +/// Shape of the name-resolution scenario (GOL-373). +/// +/// Two ceilings and the name they are about. Both are recorded rather than +/// enforced, because "the quota stream got slower on the pod that could not +/// resolve the shard manager" is a result to read, not a contract to break — +/// what fails an S4 run is the exactly-once oracle and a shard assignment that +/// moved. +/// +/// The name is here rather than read from the Chaos Mesh manifest for the same +/// reason `skew.injectedOffsetMs` is: the driver never touches Kubernetes, and a +/// result that names what was poisoned is readable a year later without the +/// manifest beside it. Nothing checks the two agree, and nothing can — see the +/// workflow's DNS capability preflight, which is where that is established. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResolutionConfig { + /// The name the fault makes unresolvable on the target executor. + pub poisoned_name: String, + /// How far above the *control executor's* p50, in the same window, the + /// target executor's quota p50 may sit before + /// [`resolution::ResolutionViolation::QuotaDegraded`] is recorded. + /// + /// Against the other executor rather than against its own baseline, because + /// a cost here would be concurrent with the fault: both halves run the same + /// workload against the same shard manager at the same instant, so the + /// difference between them is the fault and not the hour. + pub degradation_ceiling_percent: f64, + /// How far above its own baseline the target executor's post-fault quota + /// p50 may sit before + /// [`resolution::ResolutionViolation::QuotaDidNotRecover`] is recorded. + #[serde(default = "default_recovery_floor_percent")] + pub recovery_floor_percent: f64, + /// Which way the ceiling is read. + /// + /// Not a cosmetic label, and the same split the relay account carries. + /// Under `survives` a target group that fell behind its control is the + /// finding. Under `degrades` that is the expected result and the finding is + /// the opposite one — a composition that changed nothing, which means the + /// run measured S4 wearing an MF code. + #[serde(default)] + pub expectation: resolution::ResolutionExpectation, +} + +impl IsolationConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + +/// Shape of the revert workload (GOL-371). +/// +/// The sixth experiment shape, and the only one that asks the platform to +/// *destroy* durable state on purpose. Every other scenario disturbs work that +/// is trying to happen; this one disturbs work that is trying to be undone. +/// +/// Each agent repeats a round: increment `increments_per_round` times, then +/// revert the last `revert_invocations` of them. Both numbers are exact, and +/// that is the point — the driver knows the counter's value before the revert +/// from the last increment's own return value, so the value afterwards has +/// exactly two legitimate answers and no band of doubt between them. +/// +/// Reverting needs the worker stopped (`lock_stopped_worker` in +/// `golem-worker-executor/src/worker/mod.rs`), so a revert is not one atomic +/// instant but a stop, a commit and a status reattach. The truncation itself is +/// a single oplog entry and cannot tear; the window worth killing into is the +/// one around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertConfig { + /// Counter agents running rounds, split by shard ownership around the + /// executor the kill is aimed at. Also the resolution of the report: a + /// torn revert localises to one agent out of this many. + pub agents: u32, + /// Increments before each revert. Must be at least `revert_invocations`, + /// or the revert would reach back into an already-deleted oplog region and + /// the platform would refuse it — see `find_nth_invocation_from_end`. + pub increments_per_round: u32, + /// How many of those increments each revert takes back. + pub revert_invocations: u32, + /// Milliseconds an agent waits between rounds. The share of the population + /// standing mid-revert at any instant is roughly one round-step in + /// `increments_per_round + 1`, so this and the round length together decide + /// how much of the mechanism a kill can land in. + pub interval_millis: u64, + /// What recovering a reverted agent may cost, and the number the + /// resume delay is reported against. Recorded rather than asserted, like + /// every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl RevertConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } + /// What one completed round adds to a counter. + pub fn net_per_round(&self) -> u32 { + self.increments_per_round + .saturating_sub(self.revert_invocations) + } +} + +/// Shape of the deletion workload (GOL-372). +/// +/// The seventh experiment shape, and one step past [`RevertConfig`]. A revert +/// asks the platform to forget some of an agent's work; this asks it to forget +/// the agent. Each slot builds a counter up, deletes it, and is used again — +/// invoking a deleted id creates a new agent, so the next round's first +/// increment says which of the two things happened. +/// +/// The failure mode it is named for has a defence in the executor already: +/// `start_deleting` stops a background status flush from "resurrecting the +/// cached status" after the durable removal. So the question is not whether +/// anyone thought about it, but whether the defence survives the pod dying +/// between the mark and the removal. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteConfig { + /// Agent slots running rounds, split by shard ownership around the executor + /// the kill is aimed at. Also the resolution of the report: a resurrection + /// localises to one slot out of this many. + pub agents: u32, + /// Increments before each delete. + /// + /// More than one, so that a *partial* survival is observable at all. The + /// two legal answers are always distinguishable — a fresh agent reports 1 + /// and a survivor reports `before + 1`, which never collide — but at one + /// increment there is no value *between* them, so a slot that came back + /// carrying some of a state it should have lost has nowhere to land and + /// [`crate::chaos::resurrection::ResurrectionViolation::PartialState`] can + /// never fire. Three of them leaves room for it. + pub increments_per_round: u32, + /// Milliseconds a slot waits between rounds. + pub interval_millis: u64, + /// What recovering a deleted agent's slot may cost. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl DeleteConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + +/// Shape of the component rollback (GOL-369). +/// +/// S5 moves agents forward onto a new build and kills an executor while that is +/// happening. S9 moves them forward, waits for that to land, and then moves them +/// **back**, killing an executor during the return leg. +/// +/// The rollback is a redeploy: the original artifact is uploaded again as a new +/// revision, and every agent is asked to move to it. That is what "rollback" +/// means operationally, and it is what makes the evidence unambiguous — +/// `Counter::component_version` is compiled into each build, so an agent that +/// has genuinely returned reports `1` from the code that is actually running, +/// not from metadata about what the platform believes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackConfig { + /// How long to let the roll-forward settle before rolling back. + /// + /// Load-bearing. If the agents never reached the new build, the rollback + /// returns them to a build they never left and the run proves nothing, so + /// this has to be long enough for the forward leg to finish on a healthy + /// cluster. + pub settle_secs: u64, + /// The share of agents that must actually report the new build before the + /// rollback is worth attempting, as a percentage. + /// + /// Below it the driver stops rather than spending the maintenance window on + /// a return journey nobody made. The same instinct as S6's smoke round. + pub rolled_forward_floor_percent: f64, + /// Retries for the rollback *control plane* — the per-agent update + /// requests — recorded apart from the workload's own retries. + /// + /// Separate because they answer different questions. The workload's retry + /// exists to expose duplicate execution and is deliberately one attempt. + /// This one exists so that a refused rollback request is distinguishable + /// from a rollback that was never asked for, which matters when the + /// executor owning an agent is about to be killed. + pub control_retries: u32, + pub control_retry_delay_secs: u64, + /// How far into the rollback to ask for the kill. + pub kill_delay_secs: u64, +} + +impl RollbackConfig { + pub fn settle(&self) -> Duration { + Duration::from_secs(self.settle_secs) + } + pub fn control_retry_delay(&self) -> Duration { + Duration::from_secs(self.control_retry_delay_secs) + } + pub fn kill_delay(&self) -> Duration { + Duration::from_secs(self.kill_delay_secs) + } +} + +/// Settings for a storage-outage scenario (GOL-379). +/// +/// Not a workload shape, unlike the blocks above it: S16 drives the mixed +/// workload and the scheduled-registration workload that already exist, because +/// the fault is not about which agents are driven but about which *dependency* +/// is taken away from all of them. What this block carries is the thing being +/// taken away and the two numbers the account is judged by. +/// +/// The endpoint is recorded rather than acted on, like every other entry in +/// [`FaultConfig`]. The driver never resolves it, connects to it or cuts +/// anything off from it; the workflow does that, and this exists so an archived +/// result says which storage the run was about rather than leaving a reader to +/// infer it from the scenario name. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageConfig { + /// The storage endpoint the workflow partitions the executors from, as a + /// hostname. Chaos Mesh resolves it in the controller, so this is the same + /// string that appears in the NetworkChaos manifest's `externalTargets`. + pub endpoint: String, + /// What this scenario expects its cut to do to the workload, and therefore + /// what the run treats as evidence the cut landed. + /// + /// Deliberately per-scenario rather than one shared rule. The *account* the + /// driver produces is shared because it is factual: throughput per stream + /// per window, quiet time, latency, what was caught in flight. The + /// *verdict* is not, because what a cut is supposed to do depends on what + /// it cut, and a rule that reads correctly for one arrangement can be + /// confidently wrong about another. See [`OutageExpectation`]. + pub expect: OutageExpectation, + /// What serving again may cost once the storage is reachable, and the + /// number each stream's recovery gap is reported against. Recorded rather + /// than asserted, like every other budget in the suite: how long a + /// connection pool may take to notice its database came back is a + /// judgement, and the number is in the result either way. + pub recovery_budget_secs: u64, +} + +/// What a storage cut is expected to do to the workload. +/// +/// The two variants exist because the storage scenarios cut two different +/// *kinds* of thing, not two instances of one thing. S16, S22 and S14 each take +/// away a database every stream depends on, so "did anything keep serving" is a +/// sound test of whether the fault landed. S18 takes away a cache only part of +/// the workload touches, and under that arrangement the same rule is not merely +/// too strict — it is backwards. It reported S18's first run as an outage that +/// never happened while the partition had plainly landed, because `durable` +/// held 100% of its baseline throughout, exactly as the design says it should. +/// +/// Parameterising the old rule with a stream list would have fixed that one +/// message and left the deeper problem: for a partial cut, the streams that +/// *keep working* carry as much information as the ones that stop. If `durable` +/// had gone quiet under S18, the status blob would not be off the commit path +/// the way `AgentStatusFlusher` claims, and the shared rule would have called +/// that a clean pass. So the partial variant asserts both halves. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum OutageExpectation { + /// Everything is behind the cut, so every stream that was serving before it + /// must fall silent. S16, S22 and S14. + WholeWorkload { + /// The least of the fault window each stream must answer nothing at all + /// for, as a percentage of that window. + /// + /// This used to be a ceiling on during-fault throughput as a share of + /// baseline, and that number is still recorded. It stopped being the + /// verdict because it is a rate averaged over the whole window while + /// all the serving in an absorbed outage happens in the seconds at its + /// edges, so the same handful of edge confirmations reads as 8% of a + /// 180s window and 26% of a 60s one. The threshold then tracks the + /// window length rather than the platform, and shortening S16's window + /// to 60s duly tripped it on a partition that had plainly landed. + /// + /// Quiet time has no such coupling: it is measured against the window's + /// own edges, so it means the same thing whatever the window is. + quiet_floor_percent: f64, + }, + /// Only part of the workload is behind the cut. The named streams must fall + /// silent and every other stream that was serving before it must keep + /// serving. S18. + PartialWorkload { + /// The streams this cut is expected to stop. Their silence is the + /// evidence the fault landed, so naming a stream the workload never + /// drives would leave the run with no such evidence and no complaint — + /// which `require_storage` refuses at load time rather than discovering + /// during a maintenance window. + silenced: Vec, + /// As [`Self::WholeWorkload::quiet_floor_percent`], but applied only to + /// the streams named above. + quiet_floor_percent: f64, + /// The least of its own baseline rate an *unnamed* stream must still + /// hold during the cut. + /// + /// This is the half a shared rule cannot express. A partial cut makes a + /// claim about the streams it does not touch, and that claim is worth + /// asserting: it is derived from where the code routes each namespace, + /// so a stream stalling here says the routing is not what the source + /// says it is. + serving_floor_percent: f64, + }, + /// The store is still reachable, only slow. Nothing is expected to fall + /// silent at all, so the evidence the fault landed is latency rather than + /// absence. S15 and S17. + /// + /// Worth being explicit that this is a different *question*, not a milder + /// version of the other two. Those ask whether the platform survives losing + /// a store. This asks whether it degrades or breaks when one gets slower, + /// and "it went quiet" would be a failure here rather than the expected + /// result. + LatencyDegradation { + /// The streams whose latency must rise for the injected delay to have + /// landed. Their slowdown plays the part silence plays elsewhere: it is + /// the only positive evidence the run has that anything happened. + /// + /// May be empty, and empty is a claim rather than an omission: a run + /// that delays a store and names no stream to feel it is a control, so + /// the absence of a slowdown is its result. S15A. + slowed: Vec, + /// The least each named stream's during-fault median latency must be as + /// a multiple of its own before-fault median. + /// + /// A multiple rather than a millisecond figure, so the threshold does + /// not have to be re-derived per stream. Streams differ by an order of + /// magnitude at rest — S18's baselines ran from 56ms on `durable` to + /// 240ms on `ephemeral` — and a fixed ceiling would be slack for one + /// and impossible for the other. + slowdown_floor: f64, + /// The streams the delay is expected *not* to reach, and whose + /// steadiness is therefore the run's evidence that it reached only what + /// it aimed at. + /// + /// The latency counterpart to what + /// [`Self::PartialWorkload::serving_floor_percent`] does for a cut, and + /// it earns its place for the same reason. Both halves of the key-value + /// layer are delayed by a scenario in this suite, and each predicts a + /// different split: S17 slows the Redis half, where only a lifecycle + /// boundary crosses synchronously, so `ephemeral` moves and the other + /// three do not; S15 slows the PostgreSQL half, where the running-workers + /// recovery index, the promise keys and the scheduler's own schema all + /// live, so those three move and `ephemeral` does not. Naming the far + /// side turns each run into a test of that routing rather than a + /// measurement taken on faith. + /// + /// Optional, and empty means the run asserts nothing about the streams + /// it did not name. + #[serde(default)] + steady: Vec, + /// The most a steady stream's during-fault median may be as a multiple + /// of its own before-fault median. + /// + /// Not 1.0. A stream on the far side of the delay still shares + /// executors, connection pools and a tokio runtime with the streams on + /// the near side, so some spill is expected and is not itself a finding. + /// The number has to sit above that spill and below the multiple a real + /// dependency would produce, and those are far apart: S17 measured 6.59x + /// on the stream it delayed and no movement at all — 61ms, 100ms and + /// 91ms, identical across all three windows — on the three it did not. + #[serde(default = "default_steady_ceiling")] + steady_ceiling: f64, + /// The least of its own baseline rate *every* stream must still hold, + /// including the slowed ones. + /// + /// The claim that separates degradation from breakage. A delay should + /// make work take longer, not stop it, so a stream falling to nothing + /// here means the added latency cost more than time — a timeout fired, + /// a pool drained, a queue filled — and that is worth failing on. + serving_floor_percent: f64, + }, +} + +/// Applied when a `latency-degradation` expectation names steady streams +/// without saying how steady. Loose enough that ordinary run-to-run spread on +/// an undelayed stream does not trip it, which matters because the finding it +/// raises says the routing is wrong. +fn default_steady_ceiling() -> f64 { + 1.5 +} + +impl OutageExpectation { + /// The quiet floor, which both variants carry and apply to a different set. + pub fn quiet_floor_percent(&self) -> f64 { + match self { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } + | OutageExpectation::PartialWorkload { + quiet_floor_percent, + .. + } => *quiet_floor_percent, + // Nothing is expected to go quiet, so there is no floor to apply + // and reporting one would invite a reader to check it. + OutageExpectation::LatencyDegradation { .. } => 0.0, + } + } + + /// Whether this stream is one whose slowdown would prove a delay landed. + pub fn expects_slowdown(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { slowed, .. } => slowed.contains(&stream), + _ => false, + } + } + + /// Whether this is a delay that names no stream to feel it. + /// + /// True only of a control run. Worth its own accessor because + /// [`Self::slowdown_floor`] is `Some` for every delay, empty `slowed` or + /// not, so testing the floor cannot tell a control run apart from a run + /// whose slowed streams produced no latency to compare. + pub fn names_no_slowed_stream(&self) -> bool { + matches!( + self, + OutageExpectation::LatencyDegradation { slowed, .. } if slowed.is_empty() + ) + } + + /// The multiple of its own baseline median a slowed stream must reach, or + /// `None` where the expectation makes no claim about latency. + pub fn slowdown_floor(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { slowdown_floor, .. } => Some(*slowdown_floor), + _ => None, + } + } + + /// Whether this stream is one the delay is expected not to reach. + pub fn expects_steady(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { steady, .. } => steady.contains(&stream), + _ => false, + } + } + + /// The multiple of its own baseline median a steady stream may not exceed, + /// or `None` where the expectation names no steady streams to hold to it. + pub fn steady_ceiling(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { + steady, + steady_ceiling, + .. + } if !steady.is_empty() => Some(*steady_ceiling), + _ => None, + } + } + + /// Whether this stream is one whose silence would prove the cut landed. + /// + /// `WholeWorkload` says yes to everything, which is what makes it the + /// stricter rule rather than merely a different one. + pub fn expects_silence(&self, stream: Stream) -> bool { + match self { + OutageExpectation::WholeWorkload { .. } => true, + OutageExpectation::PartialWorkload { silenced, .. } => silenced.contains(&stream), + OutageExpectation::LatencyDegradation { .. } => false, + } + } + + /// The floor an unnamed stream's during-fault throughput must clear, or + /// `None` where the expectation makes no claim about streams still serving. + pub fn serving_floor_percent(&self) -> Option { + match self { + OutageExpectation::WholeWorkload { .. } => None, + OutageExpectation::PartialWorkload { + serving_floor_percent, + .. + } + | OutageExpectation::LatencyDegradation { + serving_floor_percent, + .. + } => Some(*serving_floor_percent), + } + } +} + +impl StorageConfig { + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -333,6 +1317,56 @@ pub struct ScaleDuringFaultConfig { pub steps: Vec, } +/// The second fault of a composed scenario, and where in the first one's window +/// it lands (GOL-381). +/// +/// The suite's one-fault scenarios describe their fault entirely in +/// [`FaultConfig`], which the workflow turns into one manifest applied for +/// `phases.faultSecs`. A composed scenario keeps all of that for its *enclosing* +/// fault and adds this: a second manifest, applied part way through, and left in +/// force until the enclosing fault heals. +/// +/// The asymmetry is deliberate. Both faults could have been listed as peers, +/// and then every phase boundary in the driver would have had two candidate +/// windows to follow and neither would obviously be the right one. One fault +/// owning the window keeps the phases meaning what they mean everywhere else in +/// the suite, and reduces the second fault to a point inside them. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComposedConfig { + /// Fault kind of the second fault, e.g. `pod-kill`. Recorded so the result + /// says what the run was configured to compose; the driver does not act on + /// it. + pub kind: String, + /// What the second fault is aimed at, e.g. `worker-executor`. + pub target: String, + /// The manifest the workflow applies for it, by file name. + /// + /// Named rather than derived. The workflow finds a scenario's manifest by + /// its code, and a composed scenario has two files sharing that code, so + /// leaving the choice to a glob would make the second fault depend on + /// alphabetical order. + pub manifest: String, + /// How far into the enclosing fault's window the second fault lands, as a + /// fraction of it. The same knob [`ScaleStep`] uses, and for the same + /// reason: the workflow holds the window, so a fraction of it is the only + /// thing either side can agree on without a second clock. + pub after_fraction: f64, + /// The least time both faults must be in force at once for the run to be a + /// composition rather than two faults in a row. + /// + /// Judged after the fact rather than enforced: the driver cannot refuse the + /// second fault, because by the time it is due the fault window already + /// belongs to the workflow. See [`crate::chaos::composed`]. + pub min_overlap_secs: u64, +} + +impl ComposedConfig { + pub fn min_overlap(&self) -> Duration { + Duration::from_secs(self.min_overlap_secs) + } +} + /// Settings for the shard-ownership oracle (GOL-364). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -372,6 +1406,38 @@ pub struct ScenarioConfig { /// The pinned in-flight workload. Absent for scenarios that do not run one. #[serde(default)] pub pinned: Option, + /// The scheduled-registration workload. Absent for scenarios that do not + /// run one. + #[serde(default)] + pub scheduled: Option, + /// The suspended-waiter workload. Absent for scenarios that do not run one. + #[serde(default)] + pub promise: Option, + /// The reachability workload. Absent for scenarios that do not run one. + #[serde(default)] + pub isolation: Option, + /// The revert workload. Absent for scenarios that do not run one. + #[serde(default)] + pub revert: Option, + /// The deletion workload. Absent for scenarios that do not run one. + #[serde(default)] + pub delete: Option, + /// The component rollback. Absent for scenarios that do not run one. + #[serde(default)] + pub rollback: Option, + /// The cross-pod RPC control. Absent for scenarios that do not run one. + #[serde(default)] + pub relay: Option, + /// Clock-skew settings. Absent for scenarios that do not move a clock. + #[serde(default)] + pub skew: Option, + /// Name-resolution settings. Absent for scenarios that do not break DNS. + #[serde(default)] + pub resolution: Option, + /// Storage-outage settings. Absent for scenarios that do not take a + /// storage dependency away. + #[serde(default)] + pub storage: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -380,6 +1446,10 @@ pub struct ScenarioConfig { /// that do not. #[serde(default)] pub scale_during_fault: Option, + /// The second fault, for the `MF` codes. Absent for every scenario that + /// injects one fault. + #[serde(default)] + pub composed: Option, #[serde(default)] pub retry_policy: RetryPolicy, /// How long the driver waits for each workflow signal before aborting. @@ -425,55 +1495,580 @@ impl ScenarioConfig { }) } - /// The pinned workload block. See [`Self::require_workload`]. - pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { - self.pinned.as_ref().ok_or_else(|| { + /// The scheduled-registration block. See [`Self::require_workload`]. + pub fn require_scheduled(&self) -> anyhow::Result<&ScheduledConfig> { + self.scheduled.as_ref().ok_or_else(|| { anyhow::anyhow!( - "chaos scenario {} needs a `pinned` block in the suite YAML", + "chaos scenario {} needs a `scheduled` block in the suite YAML", self.code ) }) } -} -/// The suite YAML: the authoritative operational switchboard. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ChaosSuite { - pub name: String, - pub scenarios: Vec, -} + /// The suspended-waiter block. See [`Self::require_workload`]. + pub fn require_promise(&self) -> anyhow::Result<&PromiseConfig> { + self.promise.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `promise` block in the suite YAML", + self.code + ) + }) + } -impl ChaosSuite { - pub fn load(path: impl AsRef) -> anyhow::Result { - let raw = std::fs::read_to_string(path.as_ref()) - .with_context(|| format!("reading chaos suite {:?}", path.as_ref()))?; - let suite: ChaosSuite = serde_yaml::from_str(&raw) - .with_context(|| format!("parsing chaos suite {:?}", path.as_ref()))?; - Ok(suite) + /// The reachability workload block. See [`Self::require_workload`]. + pub fn require_isolation(&self) -> anyhow::Result<&IsolationConfig> { + self.isolation.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs an `isolation` block in the suite YAML", + self.code + ) + }) } - /// The entry for `code`, which must exist and be enabled — the workflow is - /// expected to have filtered disabled scenarios out already, so reaching one - /// here means the two disagree and that is worth failing on. - /// Looks a scenario up, refusing one the suite has switched off. - /// - /// `allow_disabled` is the caller saying an operator named this scenario - /// deliberately. It exists for prototype scenarios, which are `enabled: - /// false` so no ordinary run picks them up but must still be runnable on - /// demand. Without it the two gates — this one and the workflow's — would - /// disagree, and the workflow's would silently lose. - pub fn scenario( - &self, - code: ScenarioCode, - allow_disabled: bool, - ) -> anyhow::Result<&ScenarioConfig> { - let entry = self - .scenarios - .iter() - .find(|s| s.code.eq_ignore_ascii_case(code.as_str())) - .ok_or_else(|| anyhow::anyhow!("chaos suite has no entry for scenario {code}"))?; - if !entry.enabled && !allow_disabled { + /// The revert workload block. See [`Self::require_workload`]. + pub fn require_revert(&self) -> anyhow::Result<&RevertConfig> { + let config = self.revert.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `revert` block in the suite YAML", + self.code + ) + })?; + // Checked here rather than discovered mid-run: a revert reaching past + // its own round lands in an already-deleted oplog region and the + // platform refuses it, so every round would fail and the scenario would + // measure nothing. + if config.revert_invocations > config.increments_per_round { + anyhow::bail!( + "chaos scenario {}: revertInvocations ({}) exceeds incrementsPerRound ({}), \ + so every revert would reach into an already-deleted oplog region", + self.code, + config.revert_invocations, + config.increments_per_round + ); + } + if config.revert_invocations == 0 { + anyhow::bail!( + "chaos scenario {}: revertInvocations is 0, so the scenario would revert nothing", + self.code + ); + } + Ok(config) + } + + /// The deletion workload block. See [`Self::require_workload`]. + pub fn require_delete(&self) -> anyhow::Result<&DeleteConfig> { + let config = self.delete.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `delete` block in the suite YAML", + self.code + ) + })?; + // At one increment the two legal answers are adjacent — 1 and 2 — so + // nothing can land between them and the partial-state violation is + // structurally unobservable. A third of the oracle would be blind, and + // every run would look clean on that axis by construction. + if config.increments_per_round < 2 { + anyhow::bail!( + "chaos scenario {}: incrementsPerRound is {}, which leaves no value between \ + a fresh agent and a survivor, so a partial state could never be observed", + self.code, + config.increments_per_round + ); + } + Ok(config) + } + + /// The rollback block. See [`Self::require_workload`]. + pub fn require_rollback(&self) -> anyhow::Result<&RollbackConfig> { + self.rollback.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `rollback` block in the suite YAML", + self.code + ) + }) + } + + /// The cross-pod RPC control block. See [`Self::require_workload`]. + pub fn require_relay(&self) -> anyhow::Result<&RelayConfig> { + let config = self.relay.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `relay` block in the suite YAML", + self.code + ) + })?; + // One number, and only under the expectation that reads it. + // + // Two floors used to live here demanding the fault degrade by some + // minimum before the run counted. Both are gone: this scenario reports + // what the fault cost and fails only when the cost outlasts the fault. + if config.expectation == relay::RelayExpectation::RelayDegraded + && config.recovery_floor_percent <= 100.0 + { + anyhow::bail!( + "chaos scenario {}: relay.recoveryFloorPercent is {}, and 100 is latency that \ + returned to its baseline exactly — a floor at or below it fails every run \ + that recovered", + self.code, + config.recovery_floor_percent + ) + } + Ok(config) + } + + /// The composed-fault block. See [`Self::require_workload`]. + /// + /// Checked rather than merely fetched, because every one of these mistakes + /// costs a maintenance window and produces a result that reads as a clean + /// composed run. A fraction outside the window puts the second fault + /// outside the first one; an overlap floor the window cannot supply fails + /// every run on a gate no configuration could ever satisfy. + pub fn require_composed(&self) -> anyhow::Result<&ComposedConfig> { + let config = self.composed.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `composed` block in the suite YAML", + self.code + ) + })?; + if !(0.0..1.0).contains(&config.after_fraction) || config.after_fraction <= 0.0 { + anyhow::bail!( + "chaos scenario {}: composed.afterFraction is {}, and only a fraction strictly inside the fault window puts the second fault inside the first one", + self.code, + config.after_fraction + ); + } + if config.manifest.trim().is_empty() { + anyhow::bail!( + "chaos scenario {}: composed.manifest is empty, so the workflow would have to guess which of this scenario's manifests is the second fault", + self.code + ); + } + if config.min_overlap_secs == 0 { + anyhow::bail!( + "chaos scenario {}: composed.minOverlapSecs is 0, which is met by a second fault landing at the last instant of the window", + self.code + ); + } + // The kill has to be aimed, and the driver aims it at the executor + // owning the largest share of the schedule targets. That is a coupling + // between this block and the `scheduled` one, so it is checked where a + // mistake costs a build rather than where it costs the maintenance + // window the driver would spend discovering there was nothing to aim + // at. + // + // Only when the second fault is aimed at an executor, which is not the + // same as "always". MF2 restarts the *shard manager*, and there is only + // one of those: Chaos Mesh selects it by label and the driver has + // nothing to pin, so requiring a schedule population would demand a + // workload the run has no use for. What matters for that shape is that + // the second fault lands inside the first one's window, which the + // checks above already cover. + if config.target == "worker-executor" && !self.drives_stream(Stream::Scheduled) { + anyhow::bail!( + "chaos scenario {}: a composed run aims its second fault at the executor owning the most schedule targets, and this entry registers none", + self.code + ); + } + // What the schedule can actually deliver, assuming both injections are + // instant. They are not — Chaos Mesh has to report `AllInjected` first + // — so a floor equal to this is already unreachable in practice, and + // the check is a floor on nonsense rather than a tight bound. + let available = self.phases.fault_secs as f64 * (1.0 - config.after_fraction); + if (config.min_overlap_secs as f64) > available { + anyhow::bail!( + "chaos scenario {}: composed.minOverlapSecs is {}, but a second fault injected at {} of a {}s window leaves at most {available:.0}s of overlap, so no run could ever meet it", + self.code, + config.min_overlap_secs, + config.after_fraction, + self.phases.fault_secs + ); + } + Ok(config) + } + + /// Whether this scenario's configuration actually produces operations on a + /// stream. + /// + /// Counts the blocks that drive traffic rather than the streams a run + /// happens to record, so it can be answered from the YAML alone before any + /// cluster time is spent. `Scheduled` has two possible sources and either + /// one counts; the streams that belong to other scenario shapes are not + /// reachable from a `storage` scenario's config at all. + fn drives_stream(&self, stream: Stream) -> bool { + let workload = self.workload.as_ref(); + match stream { + Stream::Durable => workload.is_some_and(|w| w.durable_agents > 0), + Stream::Ephemeral => workload.is_some_and(|w| w.ephemeral_agents > 0), + Stream::Promise => workload.is_some_and(|w| w.promise_agents > 0), + Stream::Quota => workload.is_some_and(|w| w.quota_agents > 0), + Stream::Rpc => workload.is_some_and(|w| w.rpc_agents > 0), + Stream::Scheduled => { + workload.is_some_and(|w| w.scheduled_agents > 0) + || self.scheduled.as_ref().is_some_and(|s| s.targets > 0) + } + Stream::PinnedHttp | Stream::PromiseWait | Stream::Delete | Stream::Revert => false, + } + } + + /// A floor of 100% can never be met: a stream is only quiet between the + /// answers it did give, and an operation submitted just before the heal + /// confirms just after it, inside the window it was submitted in. Every run + /// would then report the outage as not observed, and the one verdict that + /// exists to catch a fault which never landed would be stuck on. A floor of + /// zero is the opposite failure: every run passes, including the ones where + /// nothing was ever cut off. + fn check_quiet_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.quietFloorPercent is {floor}, which is not a share \ + of the fault window a real outage could be judged by", + self.code + ); + } + Ok(()) + } + + fn check_serving_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..=100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.servingFloorPercent is {floor}, which is not a share \ + of a stream's own baseline it could be held to", + self.code + ); + } + Ok(()) + } + + /// The streams whose behaviour is the run's only positive evidence that the + /// fault landed at all. + /// + /// Naming one the workload never drives leaves the run with no such + /// evidence and nothing to complain about — the exact failure the floors + /// exist to prevent, reintroduced one level up. Refused at load time so it + /// costs a build rather than a maintenance window. + fn check_evidence_streams( + &self, + field: &str, + streams: &[Stream], + evidence: &str, + ) -> anyhow::Result<()> { + if streams.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.{field} is empty, so nothing in the run would show \ + whether the fault landed at all", + self.code + ); + } + for stream in streams { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.{field} names `{stream}`, which this scenario's \ + workload never drives, so its {evidence} during the fault would prove \ + nothing", + self.code + ); + } + } + Ok(()) + } + + /// The storage-outage block. See [`Self::require_workload`]. + pub fn require_storage(&self) -> anyhow::Result<&StorageConfig> { + let config = self.storage.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `storage` block in the suite YAML", + self.code + ) + })?; + // A floor of 100% can never be met: a stream is only quiet between the + // answers it did give, and an operation submitted just before the heal + // confirms just after it, inside the window it was submitted in. Every + // run would then report the outage as not observed, and the one verdict + // that exists to catch a fault which never landed would be stuck on. A + // floor of zero is the opposite failure: every run passes, including + // the ones where nothing was ever cut off. + // Checked per variant rather than through the accessors, because the + // variants do not share a set of knobs and a single pass over "whatever + // is present" would silently skip whichever one this scenario actually + // relies on. + match &config.expect { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; + } + OutageExpectation::PartialWorkload { + silenced, + quiet_floor_percent, + serving_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; + self.check_serving_floor(*serving_floor_percent)?; + self.check_evidence_streams("silenced", silenced, "silence")?; + } + OutageExpectation::LatencyDegradation { + slowed, + slowdown_floor, + steady, + steady_ceiling, + serving_floor_percent, + } => { + // A floor of 1.0 or less asks a stream to be no slower than it + // already was, which every run satisfies including one where + // the delay never applied. That is the same hole a zero quiet + // floor leaves, in the units this variant uses. + if !slowdown_floor.is_finite() || *slowdown_floor <= 1.0 { + anyhow::bail!( + "chaos scenario {}: expect.slowdownFloor is {slowdown_floor}, and a \ + factor at or below 1.0 is met by a run where the delay never applied", + self.code + ); + } + self.check_serving_floor(*serving_floor_percent)?; + // A run may name no slowed streams at all, but only if it names + // steady ones — and then it is a control rather than an + // experiment. S15A drives `ephemeral` alone, which reaches none + // of the delayed store, so *nothing* in its workload should + // move and there is no slowdown available as evidence that the + // fault landed. + // + // The cost is real and worth stating: such a run cannot fail on + // `slowdown-not-observed`, so it cannot tell a delay that + // landed and did not matter from one that never applied. That + // evidence has to come from outside the workload, and for S15A + // it does — the scheduler polls its own schema on the delayed + // cluster whether or not anything is scheduled, so + // `db_success_seconds{svc="scheduler_storage"}` moves under the + // fault regardless. The runbook says to check it. + if slowed.is_empty() && steady.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.slowed and expect.steady are both empty, so \ + nothing in the run would show whether the fault landed or whether it \ + mattered", + self.code + ); + } + if !slowed.is_empty() { + self.check_evidence_streams("slowed", slowed, "slowdown")?; + } + // The steady list is optional, so it is checked only when + // present — but a present one carries the same two ways of + // being useless as the slowed list, plus one of its own. + for stream in steady { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.steady names `{stream}`, which this \ + scenario's workload never drives, so its steadiness during the \ + fault would prove nothing", + self.code + ); + } + if slowed.contains(stream) { + anyhow::bail!( + "chaos scenario {}: expect names `{stream}` as both slowed and \ + steady, so the run would demand the same stream both move and \ + stay put", + self.code + ); + } + } + // A ceiling at or below 1.0 asks an undelayed stream to run no + // slower than its own baseline median, which run-to-run spread + // alone breaks. The finding it would then raise says the + // platform routes a namespace somewhere other than where the + // source says, and that is far too strong a claim to make on + // noise. + if !steady.is_empty() && (!steady_ceiling.is_finite() || *steady_ceiling <= 1.0) { + anyhow::bail!( + "chaos scenario {}: expect.steadyCeiling is {steady_ceiling}, and a \ + factor at or below 1.0 is broken by ordinary run-to-run spread on a \ + stream the delay never reached", + self.code + ); + } + } + } + if config.endpoint.trim().is_empty() { + anyhow::bail!( + "chaos scenario {}: storage.endpoint is empty, so the result could not say which \ + storage the run took away", + self.code + ); + } + self.check_one_scheduled_writer()?; + Ok(config) + } + + /// Only one of the two blocks that write to the scheduled stream may be + /// present. + /// + /// Checked at load time so a bad YAML fails the build instead of a + /// maintenance window. Both blocks register scheduled actions, but only the + /// `scheduled` block's registrations carry a token into the target's fire + /// log — see `ScheduleEmitter::schedule_fire_at` in the counters component. + /// Driving both would leave the fire account pairing the mixed workload's + /// tokenless registrations against nothing and reporting every one of them + /// as an action that never ran. + fn check_one_scheduled_writer(&self) -> anyhow::Result<()> { + if let (Some(workload), Some(_)) = (&self.workload, &self.scheduled) + && workload.scheduled_agents > 0 + { + anyhow::bail!( + "chaos scenario {}: the mixed workload drives {} scheduled agents while a \ + `scheduled` block is also present, and only the latter's registrations carry a \ + token into the fire log, so the fire account would report every mixed-workload \ + registration as one that never fired", + self.code, + workload.scheduled_agents + ); + } + Ok(()) + } + + /// The name-resolution block. See [`Self::require_workload`]. + pub fn require_resolution(&self) -> anyhow::Result<&ResolutionConfig> { + let config = self.resolution.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `resolution` block in the suite YAML", + self.code + ) + })?; + // A ceiling at or below 100 is met by two executors performing + // identically, which is the expected result — so every clean run would + // report a finding and the one number that matters would stop meaning + // anything. + if config.degradation_ceiling_percent <= 100.0 { + anyhow::bail!( + "chaos scenario {}: resolution.degradationCeilingPercent is {}, and anything at \ + or below 100 is breached by two executors performing identically", + self.code, + config.degradation_ceiling_percent + ); + } + if config.poisoned_name.trim().is_empty() { + anyhow::bail!( + "chaos scenario {}: resolution.poisonedName is empty, so an archived result \ + would not say what the run made unresolvable", + self.code + ); + } + Ok(config) + } + + /// The clock-skew block. See [`Self::require_workload`]. + pub fn require_skew(&self) -> anyhow::Result<&SkewConfig> { + let config = self.skew.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `skew` block in the suite YAML", + self.code + ) + })?; + // A skew of zero is a run with no fault in it. The direction matters + // just as much: a clock that runs *fast* makes an executor renew its + // lease early, which costs an extra RPC and produces no disagreement at + // all. Only a clock set behind can make a pod act on a lease the + // granting authority has already retired. + if config.injected_offset_ms >= 0 { + anyhow::bail!( + "chaos scenario {}: skew.injectedOffsetMs is {}, and a clock set forward (or not \ + at all) only makes an executor renew early, which no oracle here can see", + self.code, + config.injected_offset_ms + ); + } + // The same hole `slowdownFloor <= 1.0` leaves, in this scenario's units: + // a tolerance at or above the offset is satisfied by a measured offset + // of zero, so the one check that proves the fault landed would pass on a + // run where nothing was injected. + if config.tolerance_ms <= 0 || config.tolerance_ms >= config.injected_offset_ms.abs() { + anyhow::bail!( + "chaos scenario {}: skew.toleranceMs is {} against an offset of {}ms, and a \ + tolerance that wide is met by a run where the clock never moved", + self.code, + config.tolerance_ms, + config.injected_offset_ms + ); + } + // At or below 100 the run would demand the quota stream come back + // *faster* than its own baseline, which run-to-run spread alone breaks. + if !config.recovery_floor_percent.is_finite() || config.recovery_floor_percent <= 100.0 { + anyhow::bail!( + "chaos scenario {}: skew.recoveryFloorPercent is {}, and a floor at or below 100 \ + asks the quota stream to end up quicker than it started", + self.code, + config.recovery_floor_percent + ); + } + if config.probes_per_round == 0 { + anyhow::bail!( + "chaos scenario {}: skew.probesPerRound is zero, so nothing in the run would \ + show whether the clock ever moved", + self.code + ); + } + // The quota lease is the only cross-clock contract on this platform, so + // a skew run that drives no quota agents has nothing the fault can + // reach and no cell to judge. + if !self.drives_stream(Stream::Quota) { + anyhow::bail!( + "chaos scenario {}: a `skew` block with no quota agents in the workload, and the \ + quota lease is the only thing on this platform one machine mints and another \ + judges — nothing else in the run can see a clock move", + self.code + ); + } + self.check_one_scheduled_writer()?; + Ok(config) + } + + /// The pinned workload block. See [`Self::require_workload`]. + pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { + self.pinned.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `pinned` block in the suite YAML", + self.code + ) + }) + } +} + +/// The suite YAML: the authoritative operational switchboard. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChaosSuite { + pub name: String, + pub scenarios: Vec, +} + +impl ChaosSuite { + pub fn load(path: impl AsRef) -> anyhow::Result { + let raw = std::fs::read_to_string(path.as_ref()) + .with_context(|| format!("reading chaos suite {:?}", path.as_ref()))?; + let suite: ChaosSuite = serde_yaml::from_str(&raw) + .with_context(|| format!("parsing chaos suite {:?}", path.as_ref()))?; + Ok(suite) + } + + /// The entry for `code`, which must exist and be enabled — the workflow is + /// expected to have filtered disabled scenarios out already, so reaching one + /// here means the two disagree and that is worth failing on. + /// Looks a scenario up, refusing one the suite has switched off. + /// + /// `allow_disabled` is the caller saying an operator named this scenario + /// deliberately. It exists for prototype scenarios, which are `enabled: + /// false` so no ordinary run picks them up but must still be runnable on + /// demand. Without it the two gates — this one and the workflow's — would + /// disagree, and the workflow's would silently lose. + pub fn scenario( + &self, + code: ScenarioCode, + allow_disabled: bool, + ) -> anyhow::Result<&ScenarioConfig> { + let entry = self + .scenarios + .iter() + .find(|s| s.code.eq_ignore_ascii_case(code.as_str())) + .ok_or_else(|| anyhow::anyhow!("chaos suite has no entry for scenario {code}"))?; + if !entry.enabled && !allow_disabled { anyhow::bail!( "chaos scenario {code} is disabled in the suite YAML \ (pass --allow-disabled to run it anyway)" @@ -486,6 +2081,7 @@ impl ChaosSuite { #[cfg(test)] mod tests { use super::*; + use golem_test_framework::benchmark::ChaosScenarioArg; use test_r::test; /// Path of the checked-in suite, resolved from the crate root so the test @@ -519,6 +2115,34 @@ mod tests { ); } + /// A `pod-kill` is instantaneous and its `duration` only governs how long + /// the object lingers, so those two numbers are free to differ. A + /// `network-partition` is not: Chaos Mesh takes the iptables rules down on + /// the object's own `duration`, while the workflow holds the phase for + /// `faultSecs` and only then deletes it. A shorter `durationSecs` therefore + /// heals the cluster part-way through the window the driver is still + /// attributing to the fault, and every during-fault number is measured + /// across a mix of the two states. + #[test] + fn a_partition_lasts_at_least_as_long_as_the_fault_phase_it_is_measured_over() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + if entry.fault.kind != "network-partition" { + continue; + } + assert!( + entry.fault.duration_secs >= entry.phases.fault_secs, + "{}: fault.durationSecs ({}) is shorter than phases.faultSecs ({}), \ + so the partition lifts {}s before the fault phase ends and the rest \ + of that phase measures a healed cluster", + entry.code, + entry.fault.duration_secs, + entry.phases.fault_secs, + entry.phases.fault_secs - entry.fault.duration_secs, + ); + } + } + #[test] fn every_suite_entry_resolves_to_an_implemented_scenario() { let suite = ChaosSuite::load(suite_path()).unwrap(); @@ -542,6 +2166,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, phases: PhaseConfig { @@ -555,9 +2180,21 @@ mod tests { scheduled_agents: 1, promise_agents: 1, quota_agents: 1, + rpc_agents: 0, rate_per_sec: 1, }), pinned: None, + scheduled: None, + promise: None, + isolation: None, + revert: None, + delete: None, + rollback: None, + relay: None, + skew: None, + resolution: None, + storage: None, + composed: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -570,6 +2207,247 @@ mod tests { assert!(suite.scenario(ScenarioCode::S12, true).is_ok()); } + fn revert_config(increments: u32, revert: u32) -> ScenarioConfig { + ScenarioConfig { + code: "S7".to_string(), + name: "executor-crash-during-revert".to_string(), + enabled: true, + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + manifest: None, + duration_secs: 60, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: None, + pinned: None, + scheduled: None, + promise: None, + isolation: None, + delete: None, + rollback: None, + relay: None, + skew: None, + resolution: None, + storage: None, + composed: None, + revert: Some(RevertConfig { + agents: 10, + increments_per_round: increments, + revert_invocations: revert, + interval_millis: 500, + recovery_budget_secs: 60, + }), + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A revert reaching further back than its own round lands in the region an + /// earlier revert already deleted, and the platform refuses it outright. + /// Every round would fail and the run would measure nothing, so this is + /// caught before the maintenance window is spent rather than per round. + #[test] + fn a_revert_deeper_than_its_own_round_is_refused_before_the_run_starts() { + let error = revert_config(2, 3) + .require_revert() + .unwrap_err() + .to_string(); + assert!( + error.contains("already-deleted oplog region"), + "the message has to say why, got: {error}" + ); + } + + /// Reverting nothing would leave a scenario that builds state up and takes + /// none of it back, which is S8 with extra steps. + #[test] + fn a_revert_of_nothing_is_refused() { + assert!(revert_config(4, 0).require_revert().is_err()); + } + + /// The boundary case is legal: a round may take back everything it added. + #[test] + fn a_revert_of_exactly_one_round_is_allowed() { + let config = revert_config(3, 3); + assert_eq!(config.require_revert().unwrap().net_per_round(), 0); + } + + fn storage_config( + endpoint: &str, + quiet_floor: f64, + scheduled_agents: u32, + scheduled_block: bool, + ) -> ScenarioConfig { + ScenarioConfig { + code: "S16".to_string(), + name: "keyvalue-postgres-outage".to_string(), + enabled: true, + fault: FaultConfig { + kind: "network-partition".to_string(), + target: "worker-executor".to_string(), + mode: "all".to_string(), + target_count: None, + manifest: None, + duration_secs: 180, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: Some(WorkloadConfig { + durable_agents: 10, + ephemeral_agents: 0, + scheduled_agents, + promise_agents: 0, + quota_agents: 0, + rpc_agents: 0, + rate_per_sec: 10, + }), + pinned: None, + scheduled: scheduled_block.then_some(ScheduledConfig { + targets: 10, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 240, + }), + promise: None, + isolation: None, + delete: None, + rollback: None, + relay: None, + skew: None, + resolution: None, + storage: Some(StorageConfig { + endpoint: endpoint.to_string(), + expect: OutageExpectation::WholeWorkload { + quiet_floor_percent: quiet_floor, + }, + recovery_budget_secs: 120, + }), + composed: None, + revert: None, + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A floor of zero passes every run, including one where nothing was cut + /// off, so the verdict that exists to catch a fault which never landed + /// would never fire. + #[test] + fn a_storage_fault_quiet_floor_of_zero_is_refused() { + let error = storage_config("db.example", 0.0, 0, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("expect.quietFloorPercent"), + "the message has to name the knob, got: {error}" + ); + } + + /// A stream is only quiet between the answers it did give, so no real + /// outage can be quiet for the whole window and the verdict would be stuck + /// on for every run. + #[test] + fn a_storage_fault_quiet_floor_of_a_whole_window_is_refused() { + assert!( + storage_config("db.example", 100.0, 0, true) + .require_storage() + .is_err(), + "a floor of the whole window can never be met" + ); + } + + /// A control run: no stream is expected to slow, and the one named steady + /// is what the run is actually asking about. Allowed, because a series that + /// isolates an interaction needs a run with nothing in it to interact. + #[test] + fn a_latency_run_that_names_only_steady_streams_is_allowed() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.workload.as_mut().unwrap().ephemeral_agents = 10; + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: vec![Stream::Ephemeral], + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + assert!( + config.require_storage().is_ok(), + "a run may assert that nothing moved, as long as it names what should not have" + ); + } + + /// Both lists empty is the one shape that says nothing at all: no stream + /// would show the fault landing, and none would show it mattering either. + #[test] + fn a_latency_run_that_names_no_streams_at_all_is_refused() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: Vec::new(), + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + let error = config.require_storage().unwrap_err().to_string(); + assert!( + error.contains("expect.slowed and expect.steady are both empty"), + "the message has to say which pair is missing, got: {error}" + ); + } + + /// Without an endpoint the archived result could not say which storage the + /// run took away. + #[test] + fn a_storage_block_with_no_endpoint_is_refused() { + assert!( + storage_config(" ", 50.0, 0, true) + .require_storage() + .is_err() + ); + } + + /// Both blocks write to the scheduled stream and only one of them carries a + /// token into the fire log, so driving both would report every tokenless + /// registration as an action that never ran. Caught at load time rather + /// than discovered in the report. + #[test] + fn driving_the_scheduled_stream_from_both_blocks_is_refused() { + let error = storage_config("db.example", 15.0, 50, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("token into the fire log"), + "the message has to say why, got: {error}" + ); + } + + /// The same mixed-workload setting is fine on its own: without a + /// `scheduled` block there is only one writer to the stream. + #[test] + fn scheduled_agents_alone_are_allowed() { + assert!( + storage_config("db.example", 15.0, 50, false) + .require_storage() + .is_ok() + ); + } + /// The retry defaults are load-bearing for correctness, not just for load. #[test] fn retry_policy_defaults_to_one_same_key_transport_only_retry() { @@ -630,9 +2508,306 @@ mod tests { assert_eq!(ScenarioCode::parse("S12"), Some(ScenarioCode::S12)); assert_eq!(ScenarioCode::parse("s8"), Some(ScenarioCode::S8)); assert_eq!(ScenarioCode::parse("s1"), Some(ScenarioCode::S1)); + assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); + assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); + assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); + assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); + assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); + assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); + assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); + assert_eq!(ScenarioCode::parse("s17"), Some(ScenarioCode::S17)); + assert_eq!(ScenarioCode::parse("s15"), Some(ScenarioCode::S15)); + assert_eq!(ScenarioCode::parse("s15a"), Some(ScenarioCode::S15A)); + assert_eq!(ScenarioCode::parse("s15b"), Some(ScenarioCode::S15B)); + assert_eq!(ScenarioCode::parse("s15c"), Some(ScenarioCode::S15C)); + assert_eq!(ScenarioCode::parse("s23"), Some(ScenarioCode::S23)); + assert_eq!(ScenarioCode::parse("s21"), Some(ScenarioCode::S21)); assert_eq!(ScenarioCode::parse("S99"), None); } + /// A scenario drives agent-to-agent traffic exactly when it carries the + /// block that says what to do with it. + /// + /// Asserted from the checked-in suite rather than a fixture because the + /// failure this guards against is a configuration one, and it can happen + /// from either side. An entry that lost its `rpcAgents` would still run, + /// still produce two throughput cells, and still come back clean — of a + /// fault that had nothing to reach. An entry that gained `rpcAgents` without + /// a `relay` block would drive twice the agents and report nothing about + /// them. The pairing gate catches the first at run time; this catches both + /// at build time. + /// + /// Stated as an equivalence rather than against a list of codes, so a third + /// scenario built on this pairing is covered without anyone remembering to + /// come back here. + #[test] + fn exactly_the_relay_scenarios_drive_the_rpc_stream() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + let code = entry.scenario_code().unwrap(); + assert_eq!( + entry.drives_stream(Stream::Rpc), + entry.relay.is_some(), + "{code} drives cross-pod RPC without a relay block, or carries one without \ + driving the stream it reads" + ); + } + } + + /// Exactly one scenario moves a clock, and it is the one that carries the + /// block saying by how much. + /// + /// The same shape as the relay pairing check above, and for a sharper + /// version of the same reason. A `skew` block on a scenario that injects no + /// clock fault would judge every run of it against an offset nobody applied + /// and fail them all on `clock-never-moved`. A clock fault with no block + /// would inject a skew and report a run with no evidence it landed, which is + /// the artifact this whole scenario is built to avoid producing. + #[test] + fn exactly_the_clock_skew_scenario_carries_a_skew_block() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + let code = entry.scenario_code().unwrap(); + assert_eq!( + entry.skew.is_some(), + entry.fault.kind == "clock-skew", + "{code} injects a clock skew without a skew block, or carries one without a \ + clock fault to judge against" + ); + } + } + + /// A DNS fault and a `resolution` block must arrive together. + /// + /// The same shape as the skew pairing above, and the failure it prevents is + /// worse here. A DNS fault with no block would run the one scenario in the + /// suite whose expected result is that nothing changed, and archive it with + /// no line saying what was compared — which is indistinguishable from a run + /// that injected nothing. A block on a scenario that breaks no name would + /// judge two executor groups against a ceiling for a fault neither of them + /// was under. + #[test] + fn exactly_the_dns_scenario_carries_a_resolution_block() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + let code = entry.scenario_code().unwrap(); + assert_eq!( + entry.resolution.is_some(), + entry.fault.kind == "dns-failure", + "{code} injects a DNS failure without a resolution block, or carries one \ + without a DNS fault to judge against" + ); + } + } + + /// The two DNS scenarios must disagree about what their fault does. + /// + /// Both read the same cells and the same ceiling, and the only thing that + /// separates a control from a composition is `expectation`. A copy-paste + /// that left MF2 on the default would report `quota-degraded` as a defect + /// for a fault composed on purpose to cause it, and would file no finding + /// at all if the shard-manager restart never forced a re-resolution — which + /// is the one outcome MF2 exists to catch. + #[test] + fn the_two_dns_scenarios_expect_opposite_things() { + use crate::chaos::resolution::ResolutionExpectation; + let suite = ChaosSuite::load(suite_path()).unwrap(); + let expectation = |code: ScenarioCode| { + suite + .scenarios + .iter() + .find(|entry| entry.scenario_code().ok() == Some(code)) + .unwrap_or_else(|| panic!("{code} is missing from the suite")) + .require_resolution() + .unwrap_or_else(|e| panic!("{code} has an unusable resolution block: {e}")) + .expectation + }; + assert_eq!( + expectation(ScenarioCode::S4), + ResolutionExpectation::Survives + ); + assert_eq!( + expectation(ScenarioCode::MF2), + ResolutionExpectation::Degrades + ); + } + + /// The poisoned name a DNS scenario reports must be the one the executor + /// actually dials. + /// + /// The suite mirrors it rather than reading the Chaos Mesh manifest, so + /// nothing else can catch a name that drifted. A mismatch would archive a + /// result claiming a name was made unresolvable when a different one was, + /// and the run would still pass — the executor never re-resolves either. + #[test] + fn the_poisoned_name_is_the_one_the_executor_dials() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + let Some(resolution) = entry.resolution.as_ref() else { + continue; + }; + let code = entry.scenario_code().unwrap(); + assert!( + resolution.poisoned_name.starts_with("shard-manager."), + "{code} poisons {}, but the only name the executor resolves towards the shard \ + manager is shard-manager..svc.cluster.local — see \ + GOLEM__SHARD_MANAGER__HOST on the worker-executor deployment", + resolution.poisoned_name + ); + } + } + + /// The two relay scenarios must disagree about what their fault does. + /// + /// Both read the same cells and the same premium, and the only thing that + /// separates a control from a load test is `expectation`. A copy-paste that + /// left S21 on the default would produce a run reporting `cross-pod-degraded` + /// as a defect for a fault injected on purpose to cause it, and would file + /// no finding at all if the stress missed worker-service entirely. + #[test] + fn the_two_relay_scenarios_expect_opposite_things() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + let expectation = |code: ScenarioCode| { + suite + .scenarios + .iter() + .find(|entry| entry.scenario_code().ok() == Some(code)) + .unwrap_or_else(|| panic!("{code} is missing from the suite")) + .require_relay() + .unwrap_or_else(|e| panic!("{code} has an unusable relay block: {e}")) + .expectation + }; + assert_eq!( + expectation(ScenarioCode::S2), + relay::RelayExpectation::Inert, + "S2 is the control: its fault is supposed to change nothing" + ); + assert_eq!( + expectation(ScenarioCode::S21), + relay::RelayExpectation::RelayDegraded, + "S21 saturates the relay both populations depend on, so it is supposed to hurt" + ); + } + + /// The workflow selects a scenario with `--scenario "${CODE,,}"`, so every + /// code in the registry has to be accepted by the CLI in exactly that form. + /// + /// This is a trap rather than a formality. `clap` derives the accepted + /// spelling from the *Rust* variant name by kebab-casing it, and the two + /// enums are in different crates with no compiler-visible link between + /// them. A variant whose name kebab-cases to something other than its own + /// lowercased code — or a code added to the registry with no CLI variant at + /// all — fails at the moment the runner invokes the driver, which is after + /// the cluster has been wiped and scaled up. + #[test] + fn every_scenario_code_is_selectable_the_way_the_workflow_spells_it() { + use clap::ValueEnum; + for code in ScenarioCode::ALL { + let spelling = code.as_str().to_lowercase(); + assert!( + ChaosScenarioArg::from_str(&spelling, false).is_ok(), + "the CLI does not accept --scenario {spelling}, which is how the workflow spells {code}" + ); + } + } + + /// MF1 and MF1B differ in exactly one thing — how much of the enclosing + /// outage runs after the kill — and the pair is worthless if that + /// difference ever narrows to nothing. + /// + /// The number they straddle is `ASSIGN_SHARDS_TIMEOUT`, which lives in the + /// golem-cloud deployment rather than here, so this cannot check the + /// straddle directly without hard-coding another repository's value. It + /// checks the weaker property that survives a change to that value: the + /// least overlap MF1B accepts is more than the most MF1 can deliver, so no + /// pair of runs can describe the same window. If someone retunes either + /// entry until they overlap, the two results stop being comparable and this + /// says so at build time. + #[test] + fn the_composed_pair_cannot_describe_the_same_window() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + let entry = |code: ScenarioCode| { + suite + .scenarios + .iter() + .find(|e| e.scenario_code().ok() == Some(code)) + .unwrap_or_else(|| panic!("{code} is missing from the suite")) + }; + + let mf1 = entry(ScenarioCode::MF1); + let mf1b = entry(ScenarioCode::MF1B); + let composed = |e: &ScenarioConfig| e.require_composed().unwrap().clone(); + + // The workflow measures the remainder of the window from the wall + // clock, so time spent confirming the injection comes out of the + // overlap. What a fraction implies is therefore a ceiling, never a + // floor. + let mf1_ceiling = mf1.phases.fault_secs as f64 * (1.0 - composed(mf1).after_fraction); + let mf1b_floor = composed(mf1b).min_overlap_secs as f64; + + assert!( + mf1b_floor > mf1_ceiling, + "MF1 can deliver up to {mf1_ceiling}s of overlap and MF1B accepts as little as \ + {mf1b_floor}s, so the two entries can describe the same experiment" + ); + } + + /// A composed run is a different kind of result from a single-fault one: + /// every number in it was measured on a cluster under two faults at once, + /// and comparing it against a scenario that had one would be wrong. The + /// `MF` prefix is how a reader tells them apart at a glance, in the suite + /// YAML, in the S3 bucket and in the runbook index. + /// + /// Nothing in the driver enforces that: `storage_fault` will happily + /// compose a fault for any code that declares one. So the convention is + /// held here instead, where breaking it costs a build. + #[test] + fn only_an_mf_code_composes_two_faults() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + if entry.composed.is_none() { + continue; + } + let code = entry.scenario_code().unwrap(); + assert!( + code.as_str().starts_with("MF"), + "{code} composes a second fault but does not carry an MF code, so its result would be read as a single-fault run" + ); + // Every rule the accessor holds, applied to the checked-in entry + // rather than only to a hand-built one in a test. + entry.require_composed().unwrap(); + } + } + + /// The gate that fails a run when nothing fired has to tell "no fire was + /// observed" apart from "no fire was asked for". S15A's first run was lost + /// to that distinction: it registers no targets, so the gate aborted it + /// during the baseline, before the fault it exists to measure was injected. + #[test] + fn a_run_that_registers_no_scheduled_targets_does_not_drive_that_stream() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + let drives = |code: ScenarioCode| { + suite + .scenarios + .iter() + .find(|entry| entry.scenario_code().ok() == Some(code)) + .unwrap_or_else(|| panic!("{code} is missing from the suite")) + .drives_stream(Stream::Scheduled) + }; + for code in [ScenarioCode::S15A, ScenarioCode::S15B, ScenarioCode::S15C] { + assert!( + !drives(code), + "{code} registers no targets, so the fire gate has nothing to gate" + ); + } + for code in [ScenarioCode::S15, ScenarioCode::S23] { + assert!( + drives(code), + "{code} drives the scheduled loop, so the fire gate still applies to it" + ); + } + } + /// A scenario whose YAML entry is missing the workload block it needs must /// say so rather than quietly running against nothing. #[test] @@ -656,6 +2831,66 @@ mod tests { ScenarioCode::S13 => { entry.require_workload().unwrap(); } + ScenarioCode::S10 => { + entry.require_scheduled().unwrap(); + } + ScenarioCode::S11 => { + entry.require_promise().unwrap(); + } + ScenarioCode::S3 => { + entry.require_isolation().unwrap(); + } + ScenarioCode::S19 => { + entry.require_workload().unwrap(); + entry.require_scheduled().unwrap(); + entry.require_skew().unwrap(); + } + ScenarioCode::S4 => { + entry.require_workload().unwrap(); + entry.require_resolution().unwrap(); + entry.require_ownership().unwrap(); + } + ScenarioCode::MF2 => { + entry.require_workload().unwrap(); + entry.require_resolution().unwrap(); + entry.require_ownership().unwrap(); + entry.require_composed().unwrap(); + } + ScenarioCode::S2 | ScenarioCode::S21 => { + entry.require_workload().unwrap(); + entry.require_relay().unwrap(); + entry.require_ownership().unwrap(); + } + ScenarioCode::S7 => { + entry.require_revert().unwrap(); + } + ScenarioCode::S6 => { + entry.require_delete().unwrap(); + } + ScenarioCode::S9 => { + entry.require_workload().unwrap(); + entry.require_rollback().unwrap(); + } + ScenarioCode::S14 + | ScenarioCode::S15 + | ScenarioCode::S15A + | ScenarioCode::S15B + | ScenarioCode::S15C + | ScenarioCode::S16 + | ScenarioCode::S17 + | ScenarioCode::S18 + | ScenarioCode::S22 + | ScenarioCode::S23 => { + entry.require_workload().unwrap(); + entry.require_scheduled().unwrap(); + entry.require_storage().unwrap(); + } + ScenarioCode::MF1 | ScenarioCode::MF1B => { + entry.require_workload().unwrap(); + entry.require_scheduled().unwrap(); + entry.require_storage().unwrap(); + entry.require_composed().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs new file mode 100644 index 0000000000..8c79c245e6 --- /dev/null +++ b/integration-tests/src/chaos/outage.rs @@ -0,0 +1,2024 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a storage outage cost, and whether the platform came back from it +//! (GOL-379). +//! +//! ### The control is time, not a pod +//! +//! Every other partition scenario in this suite keeps a group of agents on the +//! healthy side of the cut and reads the verdict off the disagreement between +//! the two groups. A storage outage has no healthy side. All three executors +//! share one key-value cluster, so cutting them off from it cuts off everything +//! at once, and an agent that happened to live elsewhere would be no better +//! served. +//! +//! So the comparison runs along the other axis. Each stream is measured against +//! **its own before-fault rate**, and the question the report answers first is +//! whether that rate collapsed at all. It has to be asked explicitly, because a +//! storage partition that silently failed to take hold produces a report full +//! of healthy numbers, and that must read as "this run tested nothing" rather +//! than as a pass. +//! +//! ### Throughput, not success rate +//! +//! The same reason [`crate::chaos::reachability`] gives, arriving by a +//! different route. The mixed workload gives every stream its own in-flight +//! budget ([`crate::chaos::workload::start`]), so a stream whose operations +//! stop returning does not fail over and over: it fills its budget, stops +//! submitting, and offers nothing further. A success rate would read that as a +//! handful of failures out of a handful of attempts. Confirmed operations per +//! second is the number that collapses. +//! +//! The per-stream budget is also what makes a per-stream row worth printing. +//! With one shared pool a single stalled stream drains the budget for all of +//! them and every row degrades together, which is exactly the unattributable +//! result S1 produced before the budgets were split. +//! +//! ### What the streams are actually testing +//! +//! The key-value cluster holds more than its name suggests. Reading the +//! executor's deployment: promises, the running-workers set and user key-value +//! data go to its `KeyValue` namespaces, and the scheduler keeps its own schema +//! on the same cluster. The oplog is on a different Aurora cluster and the +//! worker-status hot cache is in Redis, and neither is touched here. +//! +//! That is why every stream degrades under that cut rather than only the +//! obviously storage-shaped ones. A durable increment needs the running-workers +//! set before it can run at all, so `durable` is not a control group there and +//! must not be read as one. +//! +//! S18 cuts the Redis half instead, and the picture inverts. Only `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` live there, the status blob is +//! written off the commit path by a background sweeper, and what still crosses +//! the cache synchronously is a lifecycle boundary or a `get_agent_mode` miss. +//! So `ephemeral` stops and `durable`, `scheduled` and `promise` carry on, and +//! the streams still answering are evidence rather than noise. +//! +//! S17 and S15 delay the two halves rather than cutting them, and the same +//! split decides which streams move. Under S17 that is `ephemeral` alone, which +//! its run confirmed at 6.59x against three streams that did not shift by a +//! millisecond. Under S15 it is the other three: a durable invocation flips its +//! agent between tracked and untracked in the running-workers index and waits +//! for the write, a promise operation reads and writes promise keys, and +//! registering a schedule writes to the scheduler's schema — all on the +//! PostgreSQL half — while an ephemeral agent is excluded from that index by +//! `AgentStatusFlusher::on_status_changed` and never touches it. +//! +//! ### What fails the run +//! +//! Five things, and all are statements about the experiment rather than about +//! latency: +//! +//! * [`OutageViolation::OutageNotObserved`] — a stream the cut was supposed to +//! stop kept working, so the fault did not land where the run says it did. +//! * [`OutageViolation::UnexpectedStall`] — a stream the cut was *not* supposed +//! to touch stopped anyway. +//! * [`OutageViolation::SlowdownNotObserved`] — a stream a delay was aimed at +//! ran no slower than at rest, so the netem rule proved nothing. +//! * [`OutageViolation::UnexpectedSlowdown`] — a stream a delay was *not* aimed +//! at slowed with it, which says the routing is not what the source says. +//! * [`OutageViolation::StreamNeverRecovered`] — a stream that was working +//! before the outage produced nothing at all after the heal. +//! +//! ### Why the verdict is per scenario and the account is not +//! +//! Everything above the verdict is factual and shared: throughput per stream +//! per window, quiet time, latency, what the fault caught in flight. The +//! verdict is supplied by the scenario, as an +//! [`OutageExpectation`](crate::chaos::OutageExpectation), because what a cut +//! is supposed to do depends on what it cut. +//! +//! This was one rule until S18. That rule — every stream must fall silent — +//! is sound for S16, S22 and S14, where a database every stream depends on goes +//! away. S18 cuts a cache only part of the workload touches, and there the same +//! rule is not merely too strict but backwards: it reported S18's first run as +//! an outage that never landed, on the grounds that `durable` held 100% of its +//! baseline, which is exactly what the routing in +//! `NamespaceRoutedKeyValueStorage` says should happen. +//! +//! Parameterising the old rule with a list of streams to exempt would have +//! silenced that message and left the deeper gap. Under a partial cut the +//! streams that keep working carry as much information as the ones that stop: +//! had `durable` gone quiet under S18, the status blob would not be off the +//! commit path the way `AgentStatusFlusher` describes, and a rule that only +//! looks for silence would have called that its cleanest possible pass. So the +//! partial expectation asserts both halves and the whole-workload one keeps +//! asserting the single half that is all it can know. +//! +//! Recovery time is recorded against the configured budget and never asserted +//! on, like every other budget in the suite. How long a connection pool may +//! take to notice its database is back is a judgement, and the number is in the +//! result either way. + +use crate::chaos::OutageExpectation; +use crate::chaos::errors::ErrorClass; +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{ + FaultWindow, Window, longest_silence_ms, round2, window_end, window_secs, window_start, +}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// What an outage finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OutageViolation { + /// The workload kept most of its baseline throughput while the storage was + /// supposed to be unreachable. Whatever the fault status said, the + /// executors could still reach the database, and every other number in this + /// report describes an undisturbed cluster. + OutageNotObserved, + /// A stream a partial cut was *not* supposed to touch stopped serving + /// anyway. + /// + /// The counterpart to [`Self::OutageNotObserved`], and the reason the + /// storage scenarios stopped sharing one rule. A partial cut is a claim + /// about routing: these namespaces go to the store being cut and those go + /// elsewhere. If a stream on the far side stalls, the routing is not what + /// the source says it is, and the older rule — which asked only whether + /// everything went quiet — would have called that its cleanest possible + /// pass. + UnexpectedStall, + /// A stream a delay was aimed at ran no slower than it did at rest. + /// + /// The latency equivalent of [`Self::OutageNotObserved`]. Where a cut + /// proves it landed by silence, a delay can only prove it by time, and a + /// netem rule that failed to apply leaves a run full of healthy numbers and + /// no error anywhere — the worst artifact this suite can produce. + SlowdownNotObserved, + /// A stream a delay was expected to leave alone slowed down with it. + /// + /// The counterpart to [`Self::SlowdownNotObserved`], and the same argument + /// [`Self::UnexpectedStall`] makes for a cut. A delay aimed at one half of + /// the key-value layer is a claim about routing: these namespaces go to the + /// store being slowed and those go elsewhere. A stream on the far side + /// moving with it says something is wrong with that picture — and a rule + /// that only looked for slowdown would have read it as its cleanest + /// possible pass, because everything the run aimed at did indeed get + /// slower. + /// + /// It says *something* rather than *what*, and S15's first run is why the + /// wording is careful. `ephemeral` came out at 234.83x there, which cannot + /// be a route it takes itself: 46 seconds is ninety 500ms round trips, and + /// the stream makes none. It was queueing behind the three streams that do, + /// through a gate the executor shares across an account. So the three + /// readings are that the routing is wrong, that the fault landed wider than + /// the manifest names, or that the platform couples streams that the + /// storage layer keeps apart — and the third is not the least interesting. + UnexpectedSlowdown, + /// A stream that was confirming operations before the outage confirmed + /// nothing at all after the heal. + StreamNeverRecovered, +} + +impl OutageViolation { + pub fn as_str(self) -> &'static str { + match self { + OutageViolation::OutageNotObserved => "outage-not-observed", + OutageViolation::UnexpectedStall => "unexpected-stall", + OutageViolation::SlowdownNotObserved => "slowdown-not-observed", + OutageViolation::UnexpectedSlowdown => "unexpected-slowdown", + OutageViolation::StreamNeverRecovered => "stream-never-recovered", + } + } +} + +impl std::fmt::Display for OutageViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one stream. The aggregate verdict carries no stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OutageFinding { + pub violation: OutageViolation, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + pub detail: String, +} + +/// What one stream managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamThroughputCell { + pub stream: Stream, + pub window: Window, + /// Agents of this stream that offered at least one operation in this + /// window. Below the stream's pool size means emitters were stalled across + /// the whole window rather than merely slowed. + pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the platform was serving, and it is + /// deliberately not `confirmed`: during an outage the two differ by exactly + /// the work that was accepted while the storage was gone and answered only + /// once it came back. + pub served: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub served_per_sec: f64, + /// This cell's rate against the same stream's own before-fault rate. `None` + /// for the before-fault cell itself, and for a stream that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The longest the stream answered nothing at all, anywhere in this window. + /// + /// The number that stops a small non-zero during-fault rate being read as + /// residual service. Measured against the window's own edges, so a stream + /// that fell silent at the start or stayed silent to the end is caught by + /// it too. A `quietMs` close to `windowSecs` is a total outage however the + /// rate arithmetic came out, and it needs no threshold to say so. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, + pub latency: LatencyStats, +} + +/// The operations the outage began underneath. +/// +/// These were submitted before the storage went away and were still running +/// when it did, which makes them the population at risk: the platform may have +/// executed them, may have half-executed them, and cannot tell the client +/// which. They cannot be read off the cells, because every trace of them is +/// attributed to the `before-fault` row they were submitted in. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamCaughtInFlight { + pub stream: Stream, + pub operations: u64, + /// Distinct agents they belonged to. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. An operation that stalled and + /// then answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform, and the outcome alone cannot say so. + pub max_attempts: u32, + /// How many were still unresolved when the storage was reported reachable + /// again. + pub outlived_the_fault: u64, +} + +/// How long one stream took to serve anything again. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamRecovery { + pub stream: Stream, + /// Milliseconds from the heal to this stream's first confirmed operation. + /// `None` means it never confirmed one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_confirmed_ms: Option, + /// Whether that exceeded the configured budget. Recorded, not asserted. + pub over_budget: bool, +} + +/// How operations failed while the storage was unreachable. +/// +/// The acceptance criteria ask for fault-window failures, and a count alone +/// does not answer the question an operator has: whether the platform refused +/// the work definitively, or accepted it and then lost the ability to say what +/// happened. That is exactly the [`ErrorClass`] split, so the histogram is +/// keyed on it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FaultWindowErrors { + pub stream: Stream, + pub class: ErrorClass, + pub operations: u64, + /// One message, for the operator to paste into a log query. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// The storage-outage account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageFaultReport { + /// The endpoint the workflow was asked to cut the executors off from, + /// recorded so an archived result says which storage the run was about + /// rather than leaving it to the scenario name. + pub endpoint: String, + /// The rule from the suite YAML, recorded whole so an archived cell can be + /// read years later against what it was judged by rather than against + /// today's config — including *which* rule, since the storage scenarios no + /// longer share one. + pub expect: OutageExpectation, + pub recovery_budget_ms: u64, + /// The whole workload's during-fault rate as a share of its own baseline. + /// `None` for a run that never learned when the fault was. + /// + /// Recorded, no longer the verdict. It is a rate averaged over the whole + /// fault window, so it moves with the window length even when the platform + /// behaves identically. See `outage_quiet_floor_percent`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The least any stream *expected to stop* stayed silent during the fault, + /// as a share of that window. This is what the quiet half of the verdict is + /// drawn from. + /// + /// Under `WholeWorkload` that is every stream, so this is the run-wide + /// minimum and means what it always did. Under `PartialWorkload` it covers + /// only the named streams, because the others are expected to keep + /// answering and their quiet time says nothing about whether the cut + /// landed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quietest_stream_percent: Option, + /// The least of its own baseline any stream *expected to keep serving* held + /// during the fault. `None` under `WholeWorkload`, which expects none to. + /// + /// The other half of a partial cut's verdict, and the half no shared rule + /// could state: a partial cut claims the streams it does not touch carry on, + /// and that claim comes from where the code routes each namespace rather + /// than from an assumption about faults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_serving_stream_percent: Option, + /// The least any stream *expected to slow down* did, as a multiple of its + /// own before-fault median latency. `None` unless the expectation is + /// [`OutageExpectation::LatencyDegradation`]. + /// + /// A multiple of the stream's own baseline rather than a millisecond + /// figure, because the streams differ by an order of magnitude at rest and + /// a single absolute threshold would be slack for one and unreachable for + /// another. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_slowdown_factor: Option, + /// The most any stream the delay was expected *not* to reach moved, as a + /// multiple of its own before-fault median. `None` unless the expectation + /// names steady streams. + /// + /// Read together with `least_slowdown_factor`, the pair is the whole + /// experiment in two numbers: how far the streams behind the delayed store + /// moved, and how far the ones that should not be behind it did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub greatest_steady_factor: Option, + pub cells: Vec, + /// What the outage began underneath, per stream. Empty for a run that never + /// learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, + pub recovery: Vec, + pub fault_window_errors: Vec, + pub findings: Vec, +} + +/// One stream's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, + /// When this stream actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, +} + +impl StorageFaultReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// both thresholds are defined relative to a before-and-during comparison + /// that cannot be made. + pub fn build( + records: &[OperationRecord], + fault: Option, + endpoint: &str, + expect: OutageExpectation, + recovery_budget: Duration, + ) -> Self { + let mut tallies: BTreeMap<(Stream, Window), Tally> = BTreeMap::new(); + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records { + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((record.stream, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record.attempts_timed_out(); + + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered during the outage and answered after the heal is service + // the fault window did not get, and counting it there is how a total + // outage reads as partial service. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((record.stream, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // stream's before-fault rate, so a stream the workload drives rarely + // cannot make the picture look better or worse than it was. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((stream, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; + let rate = if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*stream, rate); + } + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); + cells.push(StreamThroughputCell { + stream: *stream, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + served, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + served_per_sec: round2(rate), + share_of_baseline_percent: None, + quiet_ms, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(base) = baseline_rate.get(&cell.stream).copied() + && base > 0.0 + { + cell.share_of_baseline_percent = Some(round2(cell.served_per_sec / base * 100.0)); + } + } + cells.sort_by_key(|c| (c.stream, c.window)); + + let mut report = Self { + endpoint: endpoint.to_string(), + expect, + recovery_budget_ms: recovery_budget.as_millis().min(u64::MAX as u128) as u64, + share_of_baseline_percent: None, + quietest_stream_percent: None, + least_serving_stream_percent: None, + least_slowdown_factor: None, + greatest_steady_factor: None, + cells, + caught_in_flight: caught_in_flight(records, fault), + recovery: Vec::new(), + fault_window_errors: fault_window_errors(records, fault), + findings: Vec::new(), + }; + + report.judge_outage(fault, first_submitted, last_completed); + report.judge_recovery(records, fault, recovery_budget, &baseline_rate); + report + } + + /// Did the outage land? + /// + /// Judged on how long each stream answered *nothing*, as a share of the + /// fault window, and on the stream that managed it least. A stream still + /// serving is a fault that did not land on it, and averaging would let the + /// quiet ones outvote it. + /// + /// Quiet time rather than throughput because throughput is a rate over the + /// whole window while an absorbed outage does all its serving in the + /// seconds at the window's edges: the same handful of confirmations reads + /// as a small share of a long window and a large share of a short one, so a + /// throughput threshold tracks the window length rather than the platform. + /// Quiet time is measured against the window's own edges and does not move + /// when the window does. The share is still computed, and still reported, + /// as context. + /// + /// Streams with no before-fault serving are skipped. A stream the run never + /// got going says nothing about whether the storage went away, and the + /// driver's baseline gate already reports it. + fn judge_outage( + &mut self, + fault: Option, + first_submitted: Option>, + last_completed: Option>, + ) { + if fault.is_none() { + return; + } + let total = |window: Window| -> f64 { + let served: u64 = self + .cells + .iter() + .filter(|c| c.window == window) + .map(|c| c.served) + .sum(); + let secs = window_secs(window, fault, first_submitted, last_completed); + if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + } + }; + + let baseline = total(Window::BeforeFault); + if baseline <= 0.0 { + return; + } + self.share_of_baseline_percent = + Some(round2(total(Window::DuringFault) / baseline * 100.0)); + + let served_before: BTreeSet = self + .cells + .iter() + .filter(|c| c.window == Window::BeforeFault && c.served > 0) + .map(|c| c.stream) + .collect(); + + // Split by what this scenario claims the cut does, not by what a cut + // does in general. The two sets are judged by opposite tests, and a + // stream in neither — one that never served before the fault — is + // judged by neither, because it has no baseline to be read against. + let during: Vec<&StreamThroughputCell> = self + .cells + .iter() + .filter(|c| c.window == Window::DuringFault && c.window_secs > 0.0) + .filter(|c| served_before.contains(&c.stream)) + .collect(); + + // Driven from `served_before` for the same reason as the serving list + // below: a stream that answered nothing at all leaves no during-fault + // cell, and that is total silence rather than missing data. Scoring the + // two sides of one fact differently — absence as 0% served but as + // "unknown" quiet — would let the same run read as a stall on one check + // and as no evidence on the other. + let mut quiet: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_silence(*stream)) + .filter_map(|stream| match during.iter().find(|c| c.stream == stream) { + Some(cell) => cell + .quiet_ms + .map(|ms| (stream, round2(ms as f64 / (cell.window_secs * 10.0)))), + None => Some((stream, 100.0)), + }) + .collect(); + quiet.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, quietest)) = quiet.first() { + self.quietest_stream_percent = Some(quietest); + let floor = self.expect.quiet_floor_percent(); + if quietest < floor { + self.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to stop while {} was unreachable and kept \ + answering instead, silent for only {quietest}% of the fault window \ + against a {floor}% floor — so this run has no evidence the cut landed", + self.endpoint + ), + }); + } + } + + // The latency half, before the serving one, because a delay that never + // applied makes everything below it uninteresting: the streams would + // all be serving normally and the run would read as a clean pass of an + // experiment that never happened. + if let Some(slowdown_floor) = self.expect.slowdown_floor() { + let mut slowdowns: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_slowdown(*stream)) + .filter_map(|stream| { + // Each stream against its own before-fault median. Medians + // rather than means because a handful of retried operations + // drag a mean far enough to hide whether the typical + // operation moved at all, which is the question. + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + slowdowns.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = slowdowns.first() { + self.least_slowdown_factor = Some(least); + if least < slowdown_floor { + self.findings.push(OutageFinding { + violation: OutageViolation::SlowdownNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to slow down while {} was delayed and ran at \ + {least}x its own baseline median against a {slowdown_floor}x floor \ + — so this run has no evidence the delay reached the executors", + self.endpoint + ), + }); + } + } + } + + // The far side of the same claim. A delay aimed at one store predicts + // both which streams move and which do not, and only the second half + // can catch a delay that landed somewhere wider than the run says. + if let Some(steady_ceiling) = self.expect.steady_ceiling() { + let mut steadies: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_steady(*stream)) + .filter_map(|stream| { + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + steadies.sort_by(|a, b| b.1.total_cmp(&a.1)); + + if let Some(&(stream, most)) = steadies.first() { + self.greatest_steady_factor = Some(most); + if most > steady_ceiling { + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedSlowdown, + stream: Some(stream), + detail: format!( + "{stream} does not depend on {} and was expected to run at its own \ + pace through the delay, but ran at {most}x its own baseline median \ + against a {steady_ceiling}x ceiling — so either it reaches that \ + storage after all, or it is queueing behind the streams that do", + self.endpoint + ), + }); + } + } + } + + let Some(serving_floor) = self.expect.serving_floor_percent() else { + return; + }; + // Driven from the streams that served *before* the cut rather than from + // the during-fault cells, because the worst case leaves no such cell at + // all: a stream whose operations all stall produces nothing to tally, + // and reading the cells alone would let total silence escape the one + // check that exists to catch it. Absence is the strongest evidence + // here, so it is scored as zero rather than skipped. + let mut serving: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| !self.expect.expects_silence(*stream)) + .map(|stream| { + let share = during + .iter() + .find(|c| c.stream == stream) + .and_then(|c| c.share_of_baseline_percent) + .unwrap_or(0.0); + (stream, share) + }) + .collect(); + serving.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = serving.first() { + self.least_serving_stream_percent = Some(least); + // Under a delay the two floors are not independent, and treating + // them as though they were reports the same fact twice. + // + // Throughput is concurrency over latency, and the workload offers a + // fixed in-flight budget per stream. So once a stream's latency + // rises far enough to saturate that budget, its throughput *must* + // fall in proportion, and the serving floor is then measuring the + // slowdown the run already reported rather than anything new. S15's + // first run made this concrete: promise held 11.88% of baseline + // while running 939x slower, which is 108 times more work in flight + // than at rest, not a stall. + // + // The invariant that separates them: share × slowdown is the ratio + // of during-fault concurrency to baseline concurrency. At or above + // 100% the stream carried more work than it did at rest and did not + // stop, whatever its throughput did. Below it, work is genuinely + // being lost, which is what this finding exists for. + let explained_by_slowdown = self + .expect + .slowdown_floor() + .and(self.median_ms(stream, Window::BeforeFault)) + .zip(self.median_ms(stream, Window::DuringFault)) + .map(|(before, during)| before > 0.0 && least * (during / before) >= 100.0) + .unwrap_or(false); + if least < serving_floor && !explained_by_slowdown { + let detail = if self.expect.slowdown_floor().is_some() { + format!( + "{stream} was expected to slow down while {} was delayed rather than \ + stop, but held only {least}% of its own baseline against a \ + {serving_floor}% floor, and its own latency does not account for the \ + shortfall — so the delay cost this stream work and not only time", + self.endpoint + ) + } else { + format!( + "{stream} does not depend on {} and was expected to carry on through \ + the cut, but held only {least}% of its own baseline against a \ + {serving_floor}% floor — the storage this stream actually reaches is \ + not what the routing says it is", + self.endpoint + ) + }; + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedStall, + stream: Some(stream), + detail, + }); + } + } + } + + /// How long each stream took to serve again, and which never did. + /// + /// Only streams that were working *before* the outage are judged. A stream + /// the run never got going is a different problem, already reported by the + /// baseline gate in the driver, and calling it unrecovered here would put + /// the same failure in two places under two names. + fn judge_recovery( + &mut self, + records: &[OperationRecord], + fault: Option, + budget: Duration, + baseline_rate: &BTreeMap, + ) { + let Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) = fault + else { + return; + }; + let budget_ms = budget.as_millis().min(i64::MAX as u128) as i64; + + for (stream, base) in baseline_rate { + if *base <= 0.0 { + continue; + } + let first = records + .iter() + .filter(|r| r.stream == *stream && r.outcome == Outcome::Confirmed) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= recovered) + .min(); + + let first_confirmed_ms = + first.map(|at| (at - recovered).num_milliseconds().max(0) as u64); + let over_budget = first + .map(|at| (at - recovered).num_milliseconds() > budget_ms) + .unwrap_or(false); + + if first.is_none() { + self.findings.push(OutageFinding { + violation: OutageViolation::StreamNeverRecovered, + stream: Some(*stream), + detail: format!( + "the {stream} stream confirmed operations before the outage and none \ + after {} became reachable again", + self.endpoint + ), + }); + } + self.recovery.push(StreamRecovery { + stream: *stream, + first_confirmed_ms, + over_budget, + }); + } + } + + /// One stream's cell for one window, for the tests and for anything reading + /// the report without wanting to scan the whole list. + pub fn cell(&self, stream: Stream, window: Window) -> Option<&StreamThroughputCell> { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + } + + /// Whether the report found anything a human has to act on. + pub fn has_findings(&self) -> bool { + !self.findings.is_empty() + } + + /// Lines that need a human. Empty on a run where the outage landed and + /// everything came back. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| match f.stream { + Some(stream) => format!("{}: {stream}: {}", f.violation, f.detail), + None => format!("{}: {}", f.violation, f.detail), + }) + .collect() + } + + /// One stream's median latency in a window, if it has a cell with samples. + fn median_ms(&self, stream: Stream, window: Window) -> Option { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + .filter(|c| c.latency.count > 0) + .map(|c| c.latency.p50_ms as f64) + } + + /// Context a reader needs in order to read the cells, which is not itself a + /// problem. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if let Some(share) = self.share_of_baseline_percent { + // Says which streams the numbers are about, because under a partial + // cut they are about a subset and a reader who assumes otherwise + // draws the opposite conclusion from the same figure. + // Under a latency expectation nothing is meant to go quiet, so the + // quiet figure is absent by design rather than missing, and the + // slowdown takes its place as what says the fault landed. + let quiet = match (self.quietest_stream_percent, self.least_slowdown_factor) { + // A control run: the expectation is a delay, but it names no + // stream that should feel it, so there is no slowdown to report + // and its absence is the result rather than missing data. + (None, None) if self.expect.names_no_slowed_stream() => { + "no stream in this run was expected to slow down, so whether the delay \ + landed has to be read from the storage metrics rather than from here" + .to_string() + } + (_, Some(factor)) => format!( + "the streams expected to slow down ran at least {factor}x their own baseline \ + median (floor {}x)", + self.expect.slowdown_floor().unwrap_or_default() + ), + (Some(q), None) => format!( + "the streams expected to stop were silent for at least {q}% of the fault \ + window (floor {}%)", + self.expect.quiet_floor_percent() + ), + // A delay that did name streams to slow down, none of which + // produced a pair of medians to divide. Under a severe enough + // delay that is the expected shape rather than missing data: a + // stream that stops answering inside the fault window has a + // baseline and no during-fault median. + (None, None) if self.expect.slowdown_floor().is_some() => { + "no stream expected to slow down produced both a before-fault and a \ + during-fault median, so the delay's effect cannot be read from here" + .to_string() + } + (None, None) => { + "no stream had a before-fault baseline to be judged against".to_string() + } + }; + // The far side of a delay, when the scenario named one. Printed + // next to the slowdown rather than left in the cells, because the + // two numbers only mean anything together: a run where everything + // moved by the same multiple measured a slower cluster, not a + // slower store. + let steady = match (self.greatest_steady_factor, self.expect.steady_ceiling()) { + (Some(most), Some(ceiling)) => format!( + ", the streams expected to be left alone ran at no more than {most}x theirs \ + (ceiling {ceiling}x)" + ), + _ => String::new(), + }; + let serving = match ( + self.least_serving_stream_percent, + self.expect.serving_floor_percent(), + ) { + (Some(least), Some(floor)) => format!( + ", the streams expected to carry on held at least {least}% of their own \ + baseline (floor {floor}%)" + ), + _ => String::new(), + }; + lines.push(format!( + "Storage outage: {quiet}{steady}{serving} while {} was unreachable, and the workload held \ + {share}% of its baseline throughput across that window", + self.endpoint + )); + } else { + lines.push(format!( + "Storage outage: no fault window was reported, so the {} cells carry counts but \ + no verdict", + self.endpoint + )); + } + let over: Vec = self + .recovery + .iter() + .filter(|r| r.over_budget) + .map(|r| { + format!( + "{} ({}ms)", + r.stream, + r.first_confirmed_ms.unwrap_or_default() + ) + }) + .collect(); + if !over.is_empty() { + lines.push(format!( + "Storage outage: {} took longer than the {}ms recovery budget to serve again: {}", + over.len(), + self.recovery_budget_ms, + over.join(", ") + )); + } + lines + } +} + +/// The operations that were submitted before the outage and still running when +/// it started. +fn caught_in_flight( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let Some(window) = fault else { + return Vec::new(); + }; + let mut by_stream: BTreeMap> = BTreeMap::new(); + for record in records { + // Submitted before the cut, and either still unfinished when it landed + // or finished after it. An operation with no completion at all is + // included: the driver never learned how it ended, which is the same + // doubt in a starker form. + if record.submitted_at >= window.injected_at { + continue; + } + let still_running = record + .completed_at + .map(|at| at >= window.injected_at) + .unwrap_or(true); + if still_running { + by_stream.entry(record.stream).or_default().push(record); + } + } + + by_stream + .into_iter() + .map(|(stream, caught)| StreamCaughtInFlight { + stream, + operations: caught.len() as u64, + agents: caught + .iter() + .map(|r| r.agent.as_str()) + .collect::>() + .len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect::>(), + ), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + (Some(at), Some(recovered)) => at >= recovered, + (None, _) => true, + _ => false, + }) + .count() as u64, + }) + .collect() +} + +/// How the operations submitted during the outage failed, by stream and class. +fn fault_window_errors( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let mut tallies: BTreeMap<(Stream, ErrorClass), (u64, Option)> = BTreeMap::new(); + for record in records { + if Window::of(record.submitted_at, fault) != Window::DuringFault { + continue; + } + let Some(class) = record.error_class else { + continue; + }; + let entry = tallies.entry((record.stream, class)).or_insert((0, None)); + entry.0 += 1; + if entry.1.is_none() { + entry.1.clone_from(&record.error); + } + } + + let mut rows: Vec = tallies + .into_iter() + .map( + |((stream, class), (operations, example))| FaultWindowErrors { + stream, + class, + operations, + example, + }, + ) + .collect(); + // Commonest first: an operator reading this wants the dominant failure mode + // before the long tail. + rows.sort_by(|a, b| { + b.operations + .cmp(&a.operations) + .then((a.stream, a.class).cmp(&(b.stream, b.class))) + }); + rows +} + +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use test_r::test; + + const ENDPOINT: &str = "golem-postgres-dev-keyvalue.cluster-example.rds.amazonaws.com"; + const QUIET_FLOOR: f64 = 50.0; + const AGENT: &str = "chaos-s16-durable-0000"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the storage + /// was taken away. Negative offsets are the baseline. + fn op(stream: Stream, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream, + phase: Phase::Baseline, + agent: AGENT.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{stream}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// invocation takes while the database is unreachable. + fn stalled(stream: Stream, offset_secs: i64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.error = Some("attempt timed out after 120s".to_string()); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// 300s of baseline on both streams, then `during` confirmations while the + /// storage is gone, then 240s of recovery. + fn history(during: usize, recovery: bool) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + for i in 0..during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, i as i64, Outcome::Confirmed)); + } + if recovery { + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + } + records + } + + const SERVING_FLOOR: f64 = 50.0; + + /// A workload shaped like S18's: `durable` on the far side of the cut and + /// `ephemeral` behind it. `durable_during` and `ephemeral_during` say how + /// many confirmations each managed while the storage was gone, so one + /// helper covers "behaved as designed", "the wrong stream stalled" and + /// "nothing stopped at all". + fn partial_history(durable_during: usize, ephemeral_during: usize) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for i in 0..durable_during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + } + for i in 0..ephemeral_during { + records.push(op(Stream::Ephemeral, i as i64, Outcome::Confirmed)); + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + /// The same shape as [`partial_history`], but with the operations that did + /// not confirm still *recorded* as stalled attempts. + /// + /// This is what a real run looks like: the workload keeps submitting into a + /// stall, so the stream has a during-fault cell showing what it offered and + /// nothing served. S18's first run recorded 321 such ephemeral operations. + /// [`partial_history`] covers the other shape, where a stream produced no + /// during-fault record at all, and the two must reach the same verdict. + fn partial_history_with_stalls( + durable_confirmed: usize, + ephemeral_confirmed: usize, + ) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for (stream, confirmed) in [ + (Stream::Durable, durable_confirmed), + (Stream::Ephemeral, ephemeral_confirmed), + ] { + for i in 0..180 { + if i < confirmed { + records.push(op(stream, i as i64, Outcome::Confirmed)); + } else { + records.push(stalled(stream, i as i64)); + } + } + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + const SLOWDOWN_FLOOR: f64 = 2.0; + const STEADY_CEILING: f64 = 1.5; + + /// A workload where every stream keeps serving and `ephemeral` is the one + /// the delay is aimed at. `ephemeral_ms` is what its operations take during + /// the fault, against a 200ms baseline. + fn latency_history(ephemeral_ms: u64, ephemeral_during: usize) -> Vec { + latency_history_with(ephemeral_ms, ephemeral_during, 50) + } + + /// As [`latency_history`], with the far-side stream's during-fault duration + /// under the test's control too, so a run where the delay reached more than + /// it was aimed at can be built. + fn latency_history_with( + ephemeral_ms: u64, + ephemeral_during: usize, + durable_during_ms: u64, + ) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(timed(Stream::Durable, -second, 50)); + records.push(timed(Stream::Ephemeral, -second, 200)); + } + for i in 0..180 { + records.push(timed(Stream::Durable, i as i64, durable_during_ms)); + } + for i in 0..ephemeral_during { + records.push(timed(Stream::Ephemeral, i as i64, ephemeral_ms)); + } + for second in 181..=420 { + records.push(timed(Stream::Durable, second, 50)); + records.push(timed(Stream::Ephemeral, second, 200)); + } + records + } + + /// A confirmed operation that took a stated time, so a window's median is + /// something the test controls rather than something it inherits. + fn timed(stream: Stream, offset_secs: i64, duration_ms: u64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Confirmed); + record.duration_ms = duration_ms; + record.completed_at = + Some(record.submitted_at + TimeDelta::milliseconds(duration_ms as i64)); + record + } + + fn build_latency(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + steady: Vec::new(), + steady_ceiling: STEADY_CEILING, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The same delay, with the claim S15 and S17 both make about the streams + /// on the far side of the store written down. + fn build_latency_steady(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + steady: vec![Stream::Durable], + steady_ceiling: STEADY_CEILING, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + fn build_partial(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::PartialWorkload { + silenced: vec![Stream::Ephemeral], + quiet_floor_percent: QUIET_FLOOR, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + fn build(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The healthy shape: throughput collapses while the database is gone and + /// comes back afterwards, and the report raises nothing. + #[test] + fn an_outage_that_lands_and_heals_has_no_findings() { + let report = build(&history(2, true)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the streams should have gone silent, got {:?} from {during:?}", + report.quietest_stream_percent + ); + } + + /// The one thing this scenario cannot afford to be quiet about. A partition + /// that never took hold leaves every other number describing a cluster + /// nothing happened to, and a reader has to be told rather than left to + /// infer it from cells that all look fine. + #[test] + fn a_workload_that_kept_working_reports_the_outage_as_not_observed() { + // 180 confirmations per stream across the 180s fault window is the + // baseline cadence carrying straight on through it. + let report = build(&history(180, true)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "expected an outage-not-observed finding, got {:?}", + report.findings + ); + assert!( + report.quietest_stream_percent.unwrap() < QUIET_FLOOR, + "a stream answering every second is never quiet for long, got {:?}", + report.quietest_stream_percent + ); + } + + /// The regression S18's first run produced, expressed as a test. + /// + /// Under a partial cut one stream stops and the others carry on by design. + /// The whole-workload rule reads that as a fault that never landed, because + /// the only question it can ask is whether *everything* went quiet. Run + /// 33130077355 hit exactly this: `durable` held 100% of its baseline while + /// the Redis partition had plainly landed on `ephemeral`. + #[test] + fn a_partial_cut_is_not_judged_by_the_streams_it_was_never_going_to_stop() { + let records = partial_history(180, 0); + + let shared = build(&records); + assert!( + shared + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "the whole-workload rule is expected to misjudge this, or the test guards nothing" + ); + + let report = build_partial(&records); + assert!( + report.findings.is_empty(), + "one stream stopping and the rest carrying on is the expected outcome here, got {:?}", + report.findings + ); + } + + /// The half no shared rule could state. + /// + /// A partial cut claims the streams it does not touch keep working, and + /// that claim comes from where the code routes each namespace rather than + /// from an assumption about faults. A stall on the far side means the + /// routing is not what the source says, and the older rule would have + /// called it the cleanest possible pass: everything went quiet. + #[test] + fn a_stream_the_cut_should_not_have_touched_stalling_is_a_finding() { + let report = build_partial(&partial_history(0, 0)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "durable does not depend on this endpoint and stopped anyway, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "ephemeral did stop, so the cut was observed; got {:?}", + report.findings + ); + } + + /// The named stream refusing to stop still fails, which is the check the + /// partial rule inherits rather than replaces. + #[test] + fn a_partial_cut_whose_named_stream_kept_serving_is_still_not_observed() { + let report = build_partial(&partial_history(180, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved + && f.stream == Some(Stream::Ephemeral)), + "ephemeral was the stream expected to stop, got {:?}", + report.findings + ); + } + + /// Both numbers are reported, and each covers only the streams its own rule + /// judges. Reading the quiet figure as run-wide is how the first S18 report + /// told its reader to draw the wrong conclusion. + #[test] + fn each_reported_share_covers_only_the_streams_its_rule_judges() { + let report = build_partial(&partial_history(180, 0)); + + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the quiet figure must describe ephemeral, which stopped, not durable, got {:?}", + report.quietest_stream_percent + ); + assert!( + report.least_serving_stream_percent.unwrap() > SERVING_FLOOR, + "the serving figure must describe durable, which carried on, got {:?}", + report.least_serving_stream_percent + ); + } + + /// The same two verdicts against the shape a real run produces, where the + /// stalled operations are recorded rather than absent. A stream that + /// offered work and served none of it must read the same as one that + /// offered nothing at all. + #[test] + fn a_recorded_stall_reads_the_same_as_a_stream_that_went_missing() { + let as_designed = build_partial(&partial_history_with_stalls(180, 0)); + assert!( + as_designed.findings.is_empty(), + "ephemeral stalling and durable carrying on is the expected outcome, got {:?}", + as_designed.findings + ); + + let wrong_stream = build_partial(&partial_history_with_stalls(0, 0)); + assert!( + wrong_stream + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall + && f.stream == Some(Stream::Durable)), + "durable served nothing while off the fault path, got {:?}", + wrong_stream.findings + ); + } + + /// The delay landed and the platform degraded rather than broke, which is + /// the outcome S15 and S17 are written to confirm. + #[test] + fn a_slowed_stream_that_kept_serving_is_the_expected_outcome() { + // 200ms at rest, 1s under the delay, and still completing throughout. + let report = build_latency(&latency_history(1_000, 180)); + + assert!( + report.findings.is_empty(), + "slower but still serving is what a delay is supposed to produce, got {:?}", + report.findings + ); + assert_eq!( + report.least_slowdown_factor, + Some(5.0), + "1000ms against a 200ms baseline is 5x" + ); + assert!( + report.quietest_stream_percent.is_none(), + "nothing is expected to go quiet under a delay, so there is no quiet verdict to draw" + ); + } + + /// The netem rule that never applied. Without this the run is full of + /// healthy numbers and no error anywhere, which is the worst artifact this + /// suite can produce — the same failure `outage-not-observed` exists to + /// catch, in the units a delay is measured in. + #[test] + fn a_delay_that_did_not_reach_the_executors_is_a_finding() { + // 240ms against a 200ms baseline: 1.2x, well under the 2x floor. + let report = build_latency(&latency_history(240, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved + && f.stream == Some(Stream::Ephemeral)), + "a stream that barely moved cannot evidence a delay, got {:?}", + report.findings + ); + } + + /// A delay is supposed to cost time, not work. A slowed stream that stops + /// entirely means the added latency broke something — a timeout fired, a + /// pool drained — and that is a different and worse outcome than slowness. + #[test] + fn a_delay_that_stopped_a_stream_rather_than_slowing_it_is_a_finding() { + // Slow enough to clear the slowdown floor, but only a trickle of + // operations got through, so it degraded past degradation. + let report = build_latency(&latency_history(1_000, 2)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream serving 2 operations where it served 180 has stopped, got {:?}", + report.findings + ); + } + + /// The throughput drop a delay forces on its own, which is not a second + /// finding. The workload offers each stream a fixed in-flight budget, so a + /// stream running a hundred times slower cannot hold its baseline rate no + /// matter how healthy it is — and reporting that as a stall restates the + /// slowdown under another name. S15's first run hit exactly this: promise + /// held 11.88% of baseline at 939x, which is a hundred times more work in + /// flight than at rest. + #[test] + fn a_stream_too_slow_to_hold_its_rate_has_not_stopped() { + // 100x slower, and serving 5 operations where it served 180. That is + // 2.8% of baseline, far under the 50% floor — and 2.8% x 100 is 278% of + // its baseline concurrency, so it carried more work rather than less. + let report = build_latency(&latency_history(20_000, 5)); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream whose own latency accounts for its rate has not stalled, got {:?}", + report.findings + ); + } + + /// Both halves of a delay's claim, held at once: the stream behind the + /// slowed store moved and the stream that is not behind it did not. This is + /// the shape S17 measured and the one S15 predicts with the streams + /// swapped, and it is what makes either run attributable to the store + /// rather than to a slower cluster. + #[test] + fn a_steady_stream_that_stayed_steady_is_what_makes_the_slowdown_attributable() { + let report = build_latency_steady(&latency_history_with(1_000, 180, 50)); + + assert!( + report.findings.is_empty(), + "one stream slower and the other unchanged is the expected outcome, got {:?}", + report.findings + ); + assert_eq!(report.least_slowdown_factor, Some(5.0)); + assert_eq!( + report.greatest_steady_factor, + Some(1.0), + "50ms against a 50ms baseline has not moved" + ); + } + + /// The finding a rule that only looks for slowdown cannot raise. Everything + /// the delay was aimed at did get slower, so the slowdown check passes and + /// the run reads as clean — while a stream the routing says is nowhere near + /// the delayed store moved with it, which means the routing is wrong or the + /// rule landed wider than the manifest says. + #[test] + fn a_stream_the_delay_should_not_have_touched_slowing_with_it_is_a_finding() { + // The delayed stream still clears its floor at 5x, so nothing else in + // the report objects. + let report = build_latency_steady(&latency_history_with(1_000, 180, 550)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedSlowdown + && f.stream == Some(Stream::Durable)), + "11x on a stream that does not depend on the delayed store is a finding, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved), + "the aimed-at stream did slow down, so that check should be quiet: {:?}", + report.findings + ); + } + + /// A stream the run barely drives cannot flip the verdict on its own. It is + /// judged on how long it was silent, not on its rate against a baseline, so + /// a trickle reads as the near-total silence it is rather than as a stream + /// holding up. + #[test] + fn a_trickle_stream_does_not_make_the_outage_look_unobserved() { + let mut records = history(0, true); + // A trickle of a third stream that keeps working throughout: two + // operations in the whole fault window against a baseline of hundreds. + for i in 0..2 { + records.push(op(Stream::Promise, i, Outcome::Confirmed)); + } + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "a trickle should not make the outage look unobserved, got {:?}", + report.findings + ); + } + + /// The inverse of the trickle case, and the one the old aggregate verdict + /// could miss. A single stream still answering all the way through is a + /// fault that did not land on it, and the silence of the others must not + /// vote it down. + #[test] + fn a_stream_still_answering_is_a_finding_even_when_the_others_are_silent() { + let mut records = history(0, true); + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + // Answering every second of the fault window, unlike durable and + // scheduled which say nothing at all. + for second in 0..180 { + records.push(op(Stream::Promise, second, Outcome::Confirmed)); + } + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::OutageNotObserved) + .expect("a stream answering throughout is a fault that did not land on it"); + assert_eq!(finding.stream, Some(Stream::Promise)); + } + + /// An absorbed outage: silence across the window with the serving bunched + /// into the seconds at its two edges, which is the shape the platform + /// produces once storage failures are retried rather than fatal. + fn absorbed(fault_secs: i64) -> StorageFaultReport { + let mut records = Vec::new(); + for second in 1..=300 { + for _ in 0..10 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + } + for edge in [0, fault_secs - 1] { + for _ in 0..60 { + records.push(op(Stream::Durable, edge, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, edge, Outcome::Confirmed)); + } + } + for second in fault_secs + 1..=fault_secs + 240 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + StorageFaultReport::build( + &records, + Some(FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(fault_secs)), + }), + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The regression this floor exists for. + /// + /// An absorbed outage does all its serving in the seconds at the window's + /// edges, so a during-fault *rate* divides one fixed burst by the window + /// length: identical platform behaviour reads as a small share of a long + /// window and a large share of a short one. Shortening S16's window from + /// 180s to 60s duly tripped the old 15% ceiling on a partition that had + /// plainly landed. Quiet time is measured against the window's own edges + /// and does not move with it. + #[test] + fn the_verdict_does_not_move_when_the_fault_window_does() { + let long = absorbed(180); + let short = absorbed(60); + + for (secs, report) in [(180, &long), (60, &short)] { + assert!( + report + .findings + .iter() + .all(|f| f.violation != OutageViolation::OutageNotObserved), + "a {secs}s absorbed outage did land, got {:?}", + report.findings + ); + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "a {secs}s absorbed outage is silent for most of its window, got {:?}", + report.quietest_stream_percent + ); + } + + // And the number that used to decide it, kept as the demonstration: + // the same shape reads very differently at the two lengths, which is + // exactly why it could not stay the verdict. + let long_share = long.share_of_baseline_percent.unwrap(); + let short_share = short.share_of_baseline_percent.unwrap(); + // 15% was the old ceiling. The long window sits under it and the short + // one over it, on the same behaviour, which is the whole defect. + assert!( + long_share < 15.0 && short_share > 15.0, + "this test only means something if the old ceiling would have flipped between the \ + two windows, got {long_share}% over 180s against {short_share}% over 60s" + ); + } + + /// A stream that was working before the outage and confirmed nothing after + /// the heal is the platform losing a mechanism, not a slow recovery. + #[test] + fn a_stream_that_never_came_back_is_a_finding() { + let mut records = history(0, true); + records.retain(|r| !(r.stream == Stream::Scheduled && r.submitted_at > t0())); + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::StreamNeverRecovered) + .expect("expected a stream-never-recovered finding"); + assert_eq!(finding.stream, Some(Stream::Scheduled)); + } + + /// A stream the run never got going is the driver's problem, already + /// reported by the baseline gate. Calling it unrecovered here would put one + /// failure in two places under two names. + #[test] + fn a_stream_that_never_worked_at_all_is_not_reported_as_unrecovered() { + let mut records = history(0, true); + records.retain(|r| r.stream != Stream::Scheduled); + // Present throughout, never confirming. + for second in 1..=300 { + records.push(op(Stream::Scheduled, -second, Outcome::Rejected)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.stream == Some(Stream::Scheduled)), + "a stream that never worked should not be reported as unrecovered, got {:?}", + report.findings + ); + } + + /// Recovery is measured and reported against the budget, never asserted on. + #[test] + fn a_slow_recovery_is_recorded_rather_than_a_finding() { + let mut records = history(0, false); + // First confirmation 200s after the heal, past the 120s budget. + records.push(op(Stream::Durable, 380, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, 380, Outcome::Confirmed)); + let report = build(&records); + + assert!( + report.findings.is_empty(), + "a slow recovery must not be a finding, got {:?}", + report.findings + ); + assert!( + report.recovery.iter().all(|r| r.over_budget), + "both streams should be recorded as over budget, got {:?}", + report.recovery + ); + } + + /// Without a fault window nothing can be placed either side of the outage, + /// so the report carries counts and refuses to reach a verdict. + #[test] + fn without_a_fault_window_there_is_no_verdict() { + let report = StorageFaultReport::build( + &history(180, true), + None, + ENDPOINT, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, + Duration::from_secs(120), + ); + + assert!(report.findings.is_empty()); + assert_eq!(report.share_of_baseline_percent, None); + assert!(report.caught_in_flight.is_empty()); + assert!( + report + .cells + .iter() + .all(|c| c.window == Window::Unknown && c.share_of_baseline_percent.is_none()), + "every cell should be unplaceable, got {:?}", + report.cells + ); + } + + /// The number that stops a small non-zero during-fault cell reading as + /// residual service: it says the stream was silent for almost the whole + /// window and answered only at the end. + #[test] + fn the_during_fault_cell_says_how_long_the_stream_stayed_quiet() { + let mut records = history(0, true); + // One confirmation, answered 170s into the 180s outage. + records.push(op(Stream::Durable, 170, Outcome::Confirmed)); + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.served, 1); + // Silent from the cut until that one answer, which is nearly the whole + // window — not the 10s that remained after it. + assert_eq!(during.quiet_ms, Some(170_020)); + } + + /// A stream answering steadily right through the window is the shape the + /// quiet number has to be able to tell apart from the one above. Both have + /// a non-zero during-fault count; only the silence separates them. + #[test] + fn a_stream_answering_throughout_the_window_is_never_quiet_for_long() { + let report = build(&history(180, true)); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.served, 180); + assert!( + during.quiet_ms.is_some_and(|ms| ms < 2_000), + "a stream answering once a second should show no real silence, got {:?}", + during.quiet_ms + ); + } + + /// The regression the first S16 run turned up. + /// + /// The workload keeps offering work all through the outage, so anything + /// derived from submission times is busy no matter what the platform is + /// doing. Every one of these operations is offered during the fault and + /// answered only after the heal: the fault window served nothing, and the + /// report has to say so rather than crediting the window with work it did + /// not do. + #[test] + fn work_offered_during_the_outage_and_answered_after_it_is_not_during_fault_service() { + let mut records = history(0, true); + for second in 0..120 { + let mut record = op(Stream::Durable, second, Outcome::Confirmed); + // Offered inside the outage, answered once the storage returned. + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - second) as u64 * 1_000; + records.push(record); + } + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.submitted, 120, "they were offered during the fault"); + assert_eq!(during.confirmed, 120, "and they did all eventually confirm"); + assert_eq!( + during.served, 0, + "but none of it was served during the fault" + ); + assert_eq!(during.served_per_sec, 0.0); + assert_eq!( + during.quiet_ms, + Some(180_000), + "silent for the whole window" + ); + assert!( + report.findings.is_empty(), + "a total outage must not be reported as one that never landed, got {:?}", + report.findings + ); + } + + /// The operations at risk. They were submitted before the cut and are + /// attributed to the before-fault row, which is the last place anyone looks + /// for the damage. + #[test] + fn operations_running_when_the_storage_went_away_are_reported_apart() { + let mut records = history(0, true); + // Submitted 30s before the cut, still running when it landed. + for _ in 0..3 { + records.push(stalled(Stream::Durable, -30)); + } + let report = build(&records); + + let caught = report + .caught_in_flight + .iter() + .find(|c| c.stream == Stream::Durable) + .expect("expected a caught-in-flight row for the durable stream"); + assert_eq!(caught.operations, 3); + assert_eq!(caught.indeterminate, 3); + assert_eq!(caught.attempts_timed_out, 6); + assert_eq!(caught.max_attempts, 2); + // 245s from 30s before the cut lands 35s past the 180s heal. + assert_eq!(caught.outlived_the_fault, 3); + } + + /// An operator reading the fault window wants the dominant failure mode + /// first, and wants to know whether the platform refused the work or lost + /// track of it. + #[test] + fn fault_window_failures_are_grouped_by_class_commonest_first() { + let mut records = history(0, true); + for i in 0..5 { + records.push(stalled(Stream::Durable, i)); + } + let mut rejected = op(Stream::Durable, 10, Outcome::Rejected); + rejected.error_class = Some(ErrorClass::Response); + rejected.error = Some("agent not found".to_string()); + records.push(rejected); + let report = build(&records); + + assert_eq!(report.fault_window_errors.len(), 2); + assert_eq!(report.fault_window_errors[0].class, ErrorClass::Transport); + assert_eq!(report.fault_window_errors[0].operations, 5); + assert_eq!( + report.fault_window_errors[0].example.as_deref(), + Some("attempt timed out after 120s") + ); + assert_eq!(report.fault_window_errors[1].class, ErrorClass::Response); + assert_eq!(report.fault_window_errors[1].operations, 1); + } + + /// Findings are hoisted for a human; the share line is context and belongs + /// with the notes, not with the things to act on. + #[test] + fn the_share_line_is_a_note_and_the_findings_are_attention() { + let report = build(&history(180, true)); + + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("outage-not-observed")) + ); + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("of its baseline throughput")) + ); + } +} diff --git a/integration-tests/src/chaos/pinned.rs b/integration-tests/src/chaos/pinned.rs index 95f148f61a..010aa80a17 100644 --- a/integration-tests/src/chaos/pinned.rs +++ b/integration-tests/src/chaos/pinned.rs @@ -63,6 +63,7 @@ use crate::chaos::history::Stream; use crate::chaos::workload::{self, WorkloadContext}; use anyhow::Context; use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::component::ComponentDto; use golem_common::model::{AgentId, RoutingTable}; use golem_common::{agent_id, data_value}; use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; @@ -116,29 +117,56 @@ pub fn candidate_agent_name(key_prefix: &str, index: u32) -> String { format!("{key_prefix}-{}-{index:04}", Stream::PinnedHttp) } -/// The routing-table `AgentId` for one pinned agent. +/// The routing-table `AgentId` for one agent of `agent_type`. /// /// This has to match how the worker-service builds the id it routes on — /// the component id plus the *string form* of the parsed agent id — or the /// ownership calculation would be answering a different question from the one /// the platform answers. -fn routing_agent_id(ctx: &WorkloadContext, agent: &str) -> AgentId { - let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); +pub fn routing_agent_id(ctx: &WorkloadContext, agent_type: &str, agent: &str) -> AgentId { + routing_agent_id_in(&ctx.counters, agent_type, agent) +} + +/// [`routing_agent_id`] against a named component rather than the counters one. +/// +/// Ownership is per agent id and an agent id contains its component, so an agent +/// type that lives in a different component — S11's waiters are in the promise +/// component — hashes to a different shard than the same name would under +/// `counters`. Looking one up against the wrong component would silently aim a +/// kill at whichever executor happened to own a name nothing uses. +pub fn routing_agent_id_in(component: &ComponentDto, agent_type: &str, agent: &str) -> AgentId { + let parsed: ParsedAgentId = agent_id!(agent_type, agent.to_string()); AgentId { - component_id: ctx.counters.id, + component_id: component.id, agent_id: parsed.to_string(), } } -/// Groups candidate agents by the executor that owns them. -fn owners( +/// Groups agents of `agent_type` by the executor that owns them. +/// +/// Takes the type rather than assuming `Counter` because S10 asks the same +/// question about its schedule targets, and ownership is per agent id: two +/// agent types with the same name are two different agents on two possibly +/// different executors. +pub fn owners_by_pod( ctx: &WorkloadContext, table: &RoutingTable, - candidates: &[String], + agent_type: &str, + agents: &[String], +) -> BTreeMap> { + owners_by_pod_in(&ctx.counters, table, agent_type, agents) +} + +/// [`owners_by_pod`] against a named component. See [`routing_agent_id_in`]. +pub fn owners_by_pod_in( + component: &ComponentDto, + table: &RoutingTable, + agent_type: &str, + agents: &[String], ) -> BTreeMap> { let mut by_pod: BTreeMap> = BTreeMap::new(); - for agent in candidates { - if let Some(pod) = table.lookup(&routing_agent_id(ctx, agent)) { + for agent in agents { + if let Some(pod) = table.lookup(&routing_agent_id_in(component, agent_type, agent)) { by_pod .entry(pod.to_string()) .or_default() @@ -148,6 +176,27 @@ fn owners( by_pod } +/// Groups candidate pinned agents by the executor that owns them. +fn owners( + ctx: &WorkloadContext, + table: &RoutingTable, + candidates: &[String], +) -> BTreeMap> { + owners_by_pod(ctx, table, COUNTER_AGENT, candidates) +} + +/// Host part of an executor address, which is what a Kubernetes `status.podIP` +/// field selector matches. +/// +/// No port at all is not a shape the shard-manager produces, but degrading to +/// the whole string beats panicking mid-run. +pub fn pod_ip_of(pod_address: &str) -> String { + pod_address + .rsplit_once(':') + .map(|(host, _)| host.to_string()) + .unwrap_or_else(|| pod_address.to_string()) +} + /// Chooses an executor and the agents it owns. /// /// Fails rather than falling back to an unpinned run: a scenario that quietly @@ -201,10 +250,7 @@ pub async fn select( } let agents: Vec = owned.into_iter().take(config.agents as usize).collect(); - let pod_ip = pod_address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| pod_address.clone()); + let pod_ip = pod_ip_of(&pod_address); info!( "S8: pinned {} agents to executor {pod_address} (scanned {pool} candidates across {} executors)", @@ -242,7 +288,7 @@ pub async fn verify_ownership( let mut drifted: Vec = Vec::new(); for agent in &selection.agents { let owner = table - .lookup(&routing_agent_id(ctx, agent)) + .lookup(&routing_agent_id(ctx, COUNTER_AGENT, agent)) .map(|pod| pod.to_string()); if owner.as_deref() != Some(selection.pod_address.as_str()) { drifted.push(format!( @@ -379,16 +425,10 @@ mod tests { /// hold for the `ip:port` form the routing table actually produces. #[test] fn a_pod_address_splits_into_an_ip_the_workflow_can_select_on() { - let split = |address: &str| { - address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| address.to_string()) - }; - assert_eq!(split("10.0.14.207:9000"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207:9000"), "10.0.14.207"); // No port at all is not a shape the shard-manager produces, but // degrading to the whole string beats panicking mid-run. - assert_eq!(split("10.0.14.207"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207"), "10.0.14.207"); } /// Candidate names carry the run prefix and the stream, and are zero-padded diff --git a/integration-tests/src/chaos/probe.rs b/integration-tests/src/chaos/probe.rs index 8577b175b5..942fdd052e 100644 --- a/integration-tests/src/chaos/probe.rs +++ b/integration-tests/src/chaos/probe.rs @@ -33,6 +33,15 @@ //! the probe's own footprint: the delta is exactly how many keys had never run, //! which is what makes "replayed a stored result" and "executed fresh work" //! distinguishable in aggregate. +//! +//! ### The pass is bounded, and says so when it runs into a bound +//! +//! A pass covers every key a run accepted, which is six figures, so it is the +//! one place where a single unresponsive agent can outlast the whole job. It +//! ends within [`PROBE_BUDGET`] either way, and every key it decided not to ask +//! about comes back as a [`SkipReason`] rather than going missing. That keeps a +//! shortened pass legible: the verdict is computed over a population with a +//! named gap in it, instead of over a silently smaller one. use crate::chaos::errors::ErrorClass; use crate::chaos::history::{OperationRecord, Stream}; @@ -41,7 +50,8 @@ use golem_common::base_model::agent::ParsedAgentId; use golem_common::{agent_id, data_value}; use golem_test_framework::dsl::TestDsl; use serde::{Deserialize, Serialize}; -use std::time::Duration; +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; use tokio::task::JoinSet; use tracing::{info, warn}; @@ -62,6 +72,11 @@ const PROBE_MILLIS: u32 = 0; /// wants some concurrency — but it runs against a cluster that has just been /// through a fault, and hammering it would be a second experiment nobody asked /// for. +/// +/// A live ceiling, not a batch size. The pass used to join a whole batch before +/// starting the next one, which let one key that took [`PROBE_TIMEOUT`] hold the +/// other thirty-one slots idle behind it; a run where one agent stopped +/// answering spent hours replaying keys that came back in milliseconds. const PROBE_CONCURRENCY: usize = 32; /// Ceiling on a single probe invocation. @@ -76,6 +91,30 @@ const PROBE_CONCURRENCY: usize = 32; /// not ask, which says nothing about whether the platform holds the answer. const PROBE_TIMEOUT: Duration = Duration::from_secs(30); +/// How many timed-out probes an agent gets before the pass stops asking it. +/// +/// A wedged agent holds every one of its keys for [`PROBE_TIMEOUT`], twice over +/// once the same-key retry has had its turn, and a scenario hands each agent +/// hundreds of keys. Past the second refusal there is nothing left to learn and +/// the only thing still being spent is the run's remaining time. +/// +/// Two rather than one because [`probe_one`] already burns its retry before it +/// reports a timeout, so a strike is an agent that ignored two requests, not one +/// dropped connection. +const AGENT_STRIKE_LIMIT: u32 = 2; + +/// Wall-clock ceiling on a whole pass. +/// +/// A healthy pass over a hundred thousand keys finishes in a few minutes at +/// [`PROBE_CONCURRENCY`], so this leaves several times the room it needs. What +/// it is really guarding is the job timeout: a pass that runs into that takes +/// the whole run down and writes no result at all, which is strictly worse than +/// a verdict over a population with a named gap in it. +/// +/// Soft by up to one [`PROBE_TIMEOUT`] and a retry, because probes already in +/// flight when the budget runs out are waited for rather than cancelled. +const PROBE_BUDGET: Duration = Duration::from_secs(900); + /// How often to report progress through a probe pass. /// /// A pass covers tens of thousands of keys and logs nothing between its opening @@ -99,6 +138,54 @@ pub struct KeyProbe { pub error: Option, #[serde(skip_serializing_if = "Option::is_none")] pub error_class: Option, + /// Set when the pass declined to ask about this key at all. Distinct from + /// an ordinary failed probe: the exchange did not fail, it never happened. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub skipped: Option, +} + +/// Why a pass declined to ask about a key. +/// +/// Both reasons mean the same thing for the verdict — the driver did not ask, +/// so the key is inconclusive — and different things for whoever reads the run. +/// One names a platform problem, the other names a pass that was too slow to +/// finish; keeping them apart is the difference between chasing an agent and +/// chasing a budget. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SkipReason { + /// The key's agent had already left [`AGENT_STRIKE_LIMIT`] probes + /// unanswered. + AgentUnresponsive, + /// The pass ran out of [`PROBE_BUDGET`] before it reached this key. + BudgetExhausted, +} + +impl SkipReason { + /// The phrase a report uses when it names a group of skipped keys. + pub fn describe(self) -> String { + match self { + SkipReason::AgentUnresponsive => format!( + "their agents each left {AGENT_STRIKE_LIMIT} probes unanswered, so the pass \ + stopped asking them" + ), + SkipReason::BudgetExhausted => { + format!("the pass ran out of its {PROBE_BUDGET:?} budget before reaching them") + } + } + } + + /// The per-key line stored in the artifact. + fn detail(self, agent: &str) -> String { + match self { + SkipReason::AgentUnresponsive => { + format!("not asked: agent {agent} left {AGENT_STRIKE_LIMIT} probes unanswered") + } + SkipReason::BudgetExhausted => { + format!("not asked: the probe pass exhausted its {PROBE_BUDGET:?} budget") + } + } + } } /// Re-invokes every recorded key under its original idempotency key. @@ -106,6 +193,10 @@ pub struct KeyProbe { /// Read the module documentation before changing anything here: the pass only /// means what it means because the key is reused verbatim and because the /// counters are read on either side of it. +/// +/// Slots are kept full rather than refilled in batches, and the two ways a pass +/// gives up early — an agent that stopped answering, the budget running out — +/// both produce a [`SkipReason`] against the key instead of dropping it. pub async fn probe_keys( ctx: &WorkloadContext, records: &[OperationRecord], @@ -117,38 +208,126 @@ pub async fn probe_keys( .map(|r| (r.idempotency_key.clone(), r.agent.clone())) .collect(); - info!( - "Chaos: probing {} {stream} keys for their final results", - keys.len() - ); - let total = keys.len(); + info!("Chaos: probing {total} {stream} keys for their final results"); + + let deadline = Instant::now() + PROBE_BUDGET; + let mut queue = keys.into_iter(); + let mut running: JoinSet = JoinSet::new(); + let mut strikes: BTreeMap = BTreeMap::new(); + let mut skipped: BTreeMap = BTreeMap::new(); let mut probes = Vec::with_capacity(total); let mut next_report = PROBE_PROGRESS_EVERY; - for chunk in keys.chunks(PROBE_CONCURRENCY) { - let mut batch = JoinSet::new(); - for (key, agent) in chunk { - let ctx = ctx.clone(); - let key = key.clone(); - let agent = agent.clone(); - batch.spawn(async move { probe_one(&ctx, &agent, &key).await }); + + loop { + while running.len() < PROBE_CONCURRENCY { + let Some((key, agent)) = queue.next() else { + break; + }; + match next_step(&strikes, &agent, Instant::now() >= deadline) { + Step::Ask => { + let ctx = ctx.clone(); + running.spawn(async move { probe_one(&ctx, &agent, &key).await }); + } + Step::Skip(reason) => { + *skipped.entry(reason).or_default() += 1; + probes.push(skipped_probe(key, agent, reason)); + } + } } - while let Some(joined) = batch.join_next().await { - match joined { - Ok(probe) => probes.push(probe), - Err(e) => warn!("Chaos: a probe task panicked: {e}"), + + // Empty only once the queue is drained, since the fill above runs first. + let Some(joined) = running.join_next().await else { + break; + }; + match joined { + Ok(outcome) => { + if outcome.timed_out { + let strikes = strikes.entry(outcome.probe.agent.clone()).or_default(); + *strikes += 1; + if *strikes == AGENT_STRIKE_LIMIT { + warn!( + "Chaos: agent {} left {AGENT_STRIKE_LIMIT} probes unanswered — \ + leaving its remaining keys inconclusive rather than spending \ + {PROBE_TIMEOUT:?} apiece on them", + outcome.probe.agent + ); + } + } + probes.push(outcome.probe); } + Err(e) => warn!("Chaos: a probe task panicked: {e}"), } + if probes.len() >= next_report { info!("Chaos: probed {} of {total} {stream} keys", probes.len()); - next_report += PROBE_PROGRESS_EVERY; + // Counted from here rather than stepped, so a pass that skips a + // whole agent's keys at once does not then emit a line per join + // catching up with the thresholds it jumped over. + next_report = probes.len() + PROBE_PROGRESS_EVERY; } } + for (reason, count) in &skipped { + warn!( + "Chaos: left {count} of {total} {stream} keys unasked — {}", + reason.describe() + ); + } + probes.sort_by(|a, b| a.idempotency_key.cmp(&b.idempotency_key)); probes } +/// What a pass should do with the next key it takes off the queue. +/// +/// Split out and kept pure so the two conditions that shorten a pass can be +/// tested without a cluster to wedge. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Step { + Ask, + Skip(SkipReason), +} + +/// The budget is checked first: once a pass is out of time it stops asking +/// about everything, healthy agents included. +fn next_step(strikes: &BTreeMap, agent: &str, out_of_budget: bool) -> Step { + if out_of_budget { + return Step::Skip(SkipReason::BudgetExhausted); + } + if strikes.get(agent).copied().unwrap_or(0) >= AGENT_STRIKE_LIMIT { + return Step::Skip(SkipReason::AgentUnresponsive); + } + Step::Ask +} + +/// Records a key the pass never asked about. +/// +/// Carries a transport class deliberately. Nothing was refused, so this must not +/// read as a definite answer: it lands in the exactly-once account as +/// *inconclusive*, which is what "the driver could not ask" has always meant +/// there, and [`KeyProbe::skipped`] says why this one was never even attempted. +fn skipped_probe(key: String, agent: String, reason: SkipReason) -> KeyProbe { + let error = reason.detail(&agent); + KeyProbe { + idempotency_key: key, + agent, + final_value: None, + error: Some(error), + error_class: Some(ErrorClass::Transport), + skipped: Some(reason), + } +} + +/// A finished probe, plus the one fact the pass needs that the result itself +/// does not carry: whether the last attempt ran out of time rather than +/// answering. A timeout is the only outcome that costs the pass real +/// wall-clock, so it is the only one that counts against an agent. +struct ProbeOutcome { + probe: KeyProbe, + timed_out: bool, +} + /// Probes one key, retrying once on a transport failure. /// /// The retry follows the same rule as the workload itself — one attempt, same @@ -157,7 +336,7 @@ pub async fn probe_keys( /// leaves the key inconclusive, and an inconclusive key weakens the verdict; a /// cheap same-key retry buys most of them back. It cannot mask a real problem, /// because a definite refusal is not retried. -async fn probe_one(ctx: &WorkloadContext, agent: &str, key: &str) -> KeyProbe { +async fn probe_one(ctx: &WorkloadContext, agent: &str, key: &str) -> ProbeOutcome { let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); let idempotency_key = golem_common::model::IdempotencyKey::new(key.to_string()); @@ -171,22 +350,30 @@ async fn probe_one(ctx: &WorkloadContext, agent: &str, key: &str) -> KeyProbe { PROBE_METHOD, data_value!(PROBE_MILLIS), ); + let mut timed_out = false; let outcome = match tokio::time::timeout(PROBE_TIMEOUT, invoke).await { Ok(outcome) => outcome, // Treated exactly as a transport failure: retried once under the // same key, then left inconclusive. An agent that will not answer // tells the driver nothing about what the platform holds. - Err(_) => Err(anyhow::anyhow!("probe timed out after {PROBE_TIMEOUT:?}")), + Err(_) => { + timed_out = true; + Err(anyhow::anyhow!("probe timed out after {PROBE_TIMEOUT:?}")) + } }; match outcome { Ok(value) => { - return KeyProbe { - idempotency_key: key.to_string(), - agent: agent.to_string(), - final_value: workload::as_u32_value(value), - error: None, - error_class: None, + return ProbeOutcome { + probe: KeyProbe { + idempotency_key: key.to_string(), + agent: agent.to_string(), + final_value: workload::as_u32_value(value), + error: None, + error_class: None, + skipped: None, + }, + timed_out: false, }; } Err(e) => { @@ -201,14 +388,113 @@ async fn probe_one(ctx: &WorkloadContext, agent: &str, key: &str) -> KeyProbe { inconclusive rather than reporting a lost result: {e:#}" ); } - return KeyProbe { - idempotency_key: key.to_string(), - agent: agent.to_string(), - final_value: None, - error: Some(format!("{e:#}")), - error_class: Some(class), + return ProbeOutcome { + probe: KeyProbe { + idempotency_key: key.to_string(), + agent: agent.to_string(), + final_value: None, + error: Some(format!("{e:#}")), + error_class: Some(class), + skipped: None, + }, + timed_out, }; } } } } + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn strikes(pairs: &[(&str, u32)]) -> BTreeMap { + pairs + .iter() + .map(|(agent, count)| ((*agent).to_string(), *count)) + .collect() + } + + #[test] + fn an_agent_under_the_strike_limit_is_still_asked() { + let seen = strikes(&[("a-1", AGENT_STRIKE_LIMIT - 1)]); + assert_eq!(next_step(&seen, "a-1", false), Step::Ask); + } + + #[test] + fn an_agent_at_the_strike_limit_is_not_asked_again() { + let seen = strikes(&[("a-1", AGENT_STRIKE_LIMIT)]); + assert_eq!( + next_step(&seen, "a-1", false), + Step::Skip(SkipReason::AgentUnresponsive) + ); + } + + /// The breaker is per agent, which is the whole point of it: one wedged + /// agent must not cost the other two hundred their keys. + #[test] + fn one_agents_strikes_do_not_stop_another_being_asked() { + let seen = strikes(&[("a-1", AGENT_STRIKE_LIMIT + 3)]); + assert_eq!(next_step(&seen, "a-2", false), Step::Ask); + } + + #[test] + fn running_out_of_budget_stops_the_pass_asking_anyone() { + assert_eq!( + next_step(&BTreeMap::new(), "a-1", true), + Step::Skip(SkipReason::BudgetExhausted) + ); + } + + /// Both bounds at once report the budget, because that is the one that ends + /// the pass: an operator chasing a wedged agent when the run simply ran long + /// would be chasing the wrong thing. + #[test] + fn the_budget_is_reported_ahead_of_a_wedged_agent() { + let seen = strikes(&[("a-1", AGENT_STRIKE_LIMIT)]); + assert_eq!( + next_step(&seen, "a-1", true), + Step::Skip(SkipReason::BudgetExhausted) + ); + } + + /// A skipped key has to reach the account as *inconclusive*, never as a + /// refusal — the platform was never asked, so it cannot have said no. + #[test] + fn a_skipped_key_is_inconclusive_rather_than_a_definite_answer() { + let probe = skipped_probe( + "k-1".to_string(), + "a-1".to_string(), + SkipReason::AgentUnresponsive, + ); + assert!(probe.final_value.is_none()); + assert_eq!(probe.skipped, Some(SkipReason::AgentUnresponsive)); + assert!(!probe.error_class.unwrap().is_definite_rejection()); + assert!(probe.error.unwrap().contains("a-1")); + } + + /// A pass that gives up must still hand back one entry per key. Anything it + /// drops instead becomes an unexplained hole in the population the verdict + /// is computed over. + #[test] + fn every_key_the_pass_gives_up_on_still_comes_back() { + let keys = ["k-1", "k-2", "k-3"]; + let probes: Vec = keys + .iter() + .map(|key| { + skipped_probe( + (*key).to_string(), + "a-1".to_string(), + SkipReason::BudgetExhausted, + ) + }) + .collect(); + assert_eq!(probes.len(), keys.len()); + assert!( + probes + .iter() + .all(|p| p.skipped == Some(SkipReason::BudgetExhausted)) + ); + } +} diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs new file mode 100644 index 0000000000..be1765d5e5 --- /dev/null +++ b/integration-tests/src/chaos/reachability.rs @@ -0,0 +1,1233 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a partition between worker-service and one executor cost (GOL-370). +//! +//! Three questions, in the order they have to be answered: +//! +//! 1. **Did the fault land?** The agents on the isolated executor must stop +//! being served. If they did not, nothing else in the report means anything, +//! and it says so — see [`ReachabilityViolation::PartitionNotObserved`]. +//! 2. **What did it cost the agents it was not aimed at?** The other executor +//! is reachable throughout and its agents should be untouched. They share a +//! worker-service with the stalled half, and worker-service keeps one +//! process-wide routing table that every stalled caller invalidates, so +//! "untouched" is a claim worth measuring rather than assuming. +//! 3. **Did the isolated agents come back?** Every one of them, and how long +//! after the link was restored. +//! +//! Throughput, not success rate, is what the first two are measured on, and the +//! difference matters. An emitter holds one operation at a time +//! ([`crate::chaos::steady`]), so an agent whose executor is unreachable does +//! not fail repeatedly — it fails *slowly*, once, and offers nothing else for +//! two minutes. A success rate would read that as one failure out of one +//! attempt and call the group 0% degraded. Confirmed operations per second is +//! the number that collapses, so it is the number the report is built on. +//! +//! ### Reading a non-zero isolated cell +//! +//! Operations are placed in the window they were *submitted* in, which is when +//! the platform was asked to do the work. An isolated operation submitted late +//! in the fault window can still be waiting when the link comes back, and then +//! confirms — so the isolated group's during-fault throughput is small rather +//! than exactly zero. At one operation per agent per second against a +//! two-minute client timeout, that is a handful of confirmations against a +//! baseline of hundreds. A cell anywhere near the ceiling means something else. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{ + FaultWindow, Group, PodSplit, Window, longest_silence_ms, round2, window_end, window_secs, + window_start, +}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. Past this it says how many it dropped +/// rather than growing without bound: 200 agents that all failed to recover is +/// one fact, not 200. +const MAX_FINDINGS: usize = 50; + +/// What a reachability finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReachabilityViolation { + /// The isolated executor kept serving its agents through the fault. The + /// partition did not take hold where the run says it did, and every other + /// number here describes an undisturbed cluster. + PartitionNotObserved, + /// The agents on the *reachable* executor lost throughput while the other + /// executor was cut off. They were never partitioned from anything. + ControlDegraded, + /// An isolated agent produced no confirmed operation at all once the link + /// was restored. + NeverRecovered, +} + +impl ReachabilityViolation { + pub fn as_str(self) -> &'static str { + match self { + ReachabilityViolation::PartitionNotObserved => "partition-not-observed", + ReachabilityViolation::ControlDegraded => "control-degraded", + ReachabilityViolation::NeverRecovered => "never-recovered", + } + } +} + +/// One finding. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityFinding { + pub violation: ReachabilityViolation, + /// The agent it localises to, for the findings that localise to one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent: Option, + pub detail: String, +} + +/// What one group of agents managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThroughputCell { + /// `on-pod` is the group the isolated executor owns; `elsewhere` is the + /// control group. The names are the suite's, shared with every other + /// scenario that divides its agents around one pod — see + /// [`crate::chaos::split`]. + pub group: Group, + pub window: Window, + /// Agents of this group that offered at least one operation in this window. + /// Below the group's size means emitters were stalled across the whole + /// window rather than merely slowed. + pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the group was being served, and it is + /// deliberately not `confirmed`: while an executor is unreachable the two + /// differ by exactly the work that was accepted and answered only once the + /// partition came down. + pub served: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + /// The pending-then-timeout behaviour the scenario exists to make visible. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub served_per_sec: f64, + /// This cell's rate against the same group's own before-fault rate. `None` + /// for the before-fault cell itself, and for a group that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The longest the group was answered nothing at all, anywhere in this + /// window. + /// + /// The number that stops a small non-zero rate being read as residual + /// service. A during-fault cell can show a handful of confirmations that + /// all arrived in the last seconds of the window, once the fault was + /// already coming down. Measured against the window's own edges, so a group + /// that fell silent at the start or stayed silent to the end is caught by + /// it too, and a `quietMs` close to `windowSecs` is a total outage however + /// the rate arithmetic came out. + /// + /// `None` when the window has no fixed bounds to measure against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, + pub latency: LatencyStats, +} + +/// The operations the fault landed in the middle of. +/// +/// The population S3 actually disturbed, and the one number a reader wants +/// first. It cannot be read off the cells: these operations were *submitted* +/// before the cut, so every trace of them — their timeouts, their duration — +/// is attributed to the `before-fault` row, which is the last place anyone +/// looks for the damage. +/// +/// The control group's entry is the comparison that makes it mean something, +/// but read it on **duration**, not on count. How many operations a healthy +/// group has in flight at any instant is its duty cycle — operation time over +/// interval — so a group answering in 45ms on a one-second cadence has about +/// one agent in twenty busy. A stalled group accumulates instead: every emitter +/// ends up holding an operation that will not return, so its count climbs to +/// the size of the group. A real run showed 113 of 113 against 3 of 87, and the +/// gap between 47ms and 182 seconds is the finding, not the gap between 3 and +/// 113. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CaughtInFlight { + pub group: Group, + /// Operations submitted before the fault that were still running when it + /// landed. + pub operations: u64, + /// Distinct agents they belonged to. Equal to the group size when every + /// emitter was mid-operation, which is what one-in-flight-per-agent makes + /// the normal case. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. + /// + /// Load-bearing rather than trivia. An operation that stalled and then + /// answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform: with retries off it would have ended + /// indeterminate. That distinction is invisible in the outcome alone. + pub max_attempts: u32, + /// How many were still unresolved when the fault was reported healed. + pub outlived_the_fault: u64, +} + +/// The reachability account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityReport { + /// The executor the partition was aimed at, as the shard-manager names it. + pub isolated_pod: String, + pub isolated_agents: usize, + pub reachable_agents: usize, + /// The thresholds from the suite YAML, recorded so an archived cell can be + /// read years later against the numbers it was judged by rather than + /// against today's config. + pub isolated_ceiling_percent: f64, + pub control_floor_percent: f64, + pub recovery_budget_ms: u64, + pub cells: Vec, + /// What the fault landed in the middle of, per group. Empty for a run that + /// never learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, + /// Per isolated agent, how long after the link was restored its first + /// confirmed operation landed. + pub recovery: LatencyStats, + pub recovery_over_budget: u64, + /// Isolated agents that never confirmed anything after the heal. + pub agents_never_recovered: Vec, + /// Records whose agent the selection never saw. Zero on a healthy run; + /// non-zero means the split and the workload disagree about who was driven. + pub records_outside_the_split: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], dropped rather than carried. Non-zero + /// means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One group's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, + /// When this group actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, +} + +impl ReachabilityReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// the thresholds are all defined relative to a before-and-during + /// comparison that cannot be made. + pub fn build( + records: &[OperationRecord], + split: &PodSplit, + fault: Option, + isolated_ceiling_percent: f64, + control_floor_percent: f64, + recovery_budget: std::time::Duration, + ) -> Self { + let mut tallies: BTreeMap<(Group, Window), Tally> = BTreeMap::new(); + let mut records_outside_the_split = 0u64; + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + records_outside_the_split += 1; + continue; + }; + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((group, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record.attempts_timed_out(); + + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered while the executor was unreachable and answered once the + // partition came down is not service the fault window delivered, + // and counting it there is how a total outage reads as partial. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((group, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // group's before-fault rate, so a lopsided split cannot make one group + // look better than the other. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((group, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; + let rate = if secs > 0.0 { + served as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*group, rate); + } + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); + cells.push(ThroughputCell { + group: *group, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + served, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + served_per_sec: round2(rate), + share_of_baseline_percent: None, + quiet_ms, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(baseline) = baseline_rate.get(&cell.group).filter(|r| **r > 0.0) { + cell.share_of_baseline_percent = + Some(round2(cell.served_per_sec / baseline * 100.0)); + } + } + cells.sort_by_key(|c| (c.group, c.window)); + + // ── Recovery, per isolated agent ──────────────────────────────────── + let recovered_at = fault.and_then(|w| w.recovered_at); + let mut gaps: Vec = Vec::new(); + let mut over_budget = 0u64; + let mut agents_never_recovered: Vec = Vec::new(); + if let Some(healed) = recovered_at { + for agent in &split.on_pod { + let first = records + .iter() + .filter(|r| { + r.stream == Stream::Durable + && &r.agent == agent + && r.outcome == Outcome::Confirmed + }) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= healed) + .min(); + match first { + Some(at) => { + let gap = (at - healed).num_milliseconds().max(0) as u64; + if gap > recovery_budget.as_millis() as u64 { + over_budget += 1; + } + gaps.push(gap); + } + None => agents_never_recovered.push(agent.clone()), + } + } + } + + // ── What the fault landed in the middle of ────────────────────────── + // + // Computed from the whole history rather than from the cells, because + // the cells cannot answer it: these operations were submitted before + // the cut, so every trace of them sits in the `before-fault` row. + let mut caught_in_flight: Vec = Vec::new(); + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + continue; + }; + let still_running = record + .completed_at + .is_none_or(|at| at >= window.injected_at); + if record.submitted_at < window.injected_at && still_running { + by_group.entry(group).or_default().push(record); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + caught_in_flight.push(CaughtInFlight { + group, + operations: caught.len() as u64, + agents: agents.len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect(), + ), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + // Never finished at all, so it certainly outlived it. + (None, _) => true, + (Some(done), Some(healed)) => done >= healed, + // No heal was ever reported; nothing can be said. + (Some(_), None) => false, + }) + .count() as u64, + }); + } + } + + let mut report = ReachabilityReport { + isolated_pod: split.pod_address.clone(), + isolated_agents: split.on_pod.len(), + reachable_agents: split.elsewhere.len(), + isolated_ceiling_percent, + control_floor_percent, + recovery_budget_ms: recovery_budget.as_millis() as u64, + cells, + caught_in_flight, + recovery: LatencyStats::from_durations(gaps), + recovery_over_budget: over_budget, + agents_never_recovered, + records_outside_the_split, + findings: Vec::new(), + findings_omitted: 0, + }; + report.judge(); + report + } + + /// A cell by group and window, if the run produced one. + pub fn cell(&self, group: Group, window: Window) -> Option<&ThroughputCell> { + self.cells + .iter() + .find(|c| c.group == group && c.window == window) + } + + fn judge(&mut self) { + let mut findings: Vec = Vec::new(); + + // Did the fault land? Asked first, because a "no" makes the rest of the + // report a description of a cluster nothing happened to. + if let Some(share) = self + .cell(Group::OnPod, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share > self.isolated_ceiling_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::PartitionNotObserved, + agent: None, + detail: format!( + "the {} agents on the isolated executor kept {share:.1}% of their baseline \ + throughput during the fault, above the {:.0}% ceiling: the partition did \ + not cut worker-service off from {}. Nothing else in this report describes \ + a disturbed cluster.", + self.isolated_agents, self.isolated_ceiling_percent, self.isolated_pod + ), + }); + } + + // What did it cost the half it was not aimed at? + if let Some(share) = self + .cell(Group::Elsewhere, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share < self.control_floor_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::ControlDegraded, + agent: None, + detail: format!( + "the {} agents on the reachable executor kept only {share:.1}% of their \ + baseline throughput while {} was cut off, below the {:.0}% floor. They \ + were never partitioned from anything, so this is what serving an \ + unreachable executor cost the rest of the cluster.", + self.reachable_agents, self.isolated_pod, self.control_floor_percent + ), + }); + } + + for agent in &self.agents_never_recovered { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::NeverRecovered, + agent: Some(agent.clone()), + detail: format!( + "{agent} confirmed no operation at all after the link to {} was restored", + self.isolated_pod + ), + }); + } + + self.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + self.findings = findings; + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| format!("S3 {}: {}", f.violation.as_str(), f.detail)) + .collect(); + + if self.findings_omitted > 0 { + lines.push(format!( + "S3: {} further reachability finding(s) were dropped from the report", + self.findings_omitted + )); + } + if self.records_outside_the_split > 0 { + lines.push(format!( + "S3: {} operation(s) ran against agents the ownership split never saw, so they \ + are in no group and in no cell — the split and the workload disagree about \ + who was driven", + self.records_outside_the_split + )); + } + // Work the fault caught and the run cannot account for. Not a finding — + // an indeterminate operation is doubt, not damage, and the read-back + // and exactly-once accounts are what resolve it — but the operator has + // to see it next to the clean cells rather than infer it from them. + for caught in &self.caught_in_flight { + let unresolved = caught.indeterminate + caught.rejected; + if unresolved > 0 { + lines.push(format!( + "S3: {unresolved} of the {} operations the cut caught in flight on {} did \ + not confirm ({} indeterminate, {} rejected)", + caught.operations, + caught.group.as_str(), + caught.indeterminate, + caught.rejected + )); + } + } + + // Over budget is not a finding. How long a partition heal may take + // before an agent is served again is a judgement, and the driver is not + // the one to make it. + if self.recovery_over_budget > 0 { + lines.push(format!( + "S3: {} of {} isolated agents took longer than the {}ms recovery budget to \ + confirm anything after the heal (p99 {}ms, worst {}ms)", + self.recovery_over_budget, + self.isolated_agents, + self.recovery_budget_ms, + self.recovery.p99_ms, + self.recovery.max_ms + )); + } + lines + } + + /// The lines a reader needs in order to interpret the run, which are not + /// themselves problems. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S3: {} agents on the isolated executor {}, {} elsewhere", + self.isolated_agents, self.isolated_pod, self.reachable_agents + )]; + + // The caught population first: it is what the fault actually disturbed, + // and nothing in the cells below points at it. + for caught in &self.caught_in_flight { + lines.push(format!( + "S3 {}: {} operations across {} agents were in flight when the cut landed — \ + p50 {}ms / p99 {}ms / worst {}ms, {} attempt(s) timed out, up to {} attempts \ + each, {} still unresolved at the heal", + caught.group.as_str(), + caught.operations, + caught.agents, + caught.duration.p50_ms, + caught.duration.p99_ms, + caught.duration.max_ms, + caught.attempts_timed_out, + caught.max_attempts, + caught.outlived_the_fault, + )); + } + + for group in [Group::OnPod, Group::Elsewhere] { + for window in [Window::BeforeFault, Window::DuringFault, Window::AfterFault] { + if let Some(cell) = self.cell(group, window) { + lines.push(format!( + "S3 {} {}: {:.2} served/s{}, {} offered{}, {} indeterminate, {} \ + attempt(s) timed out, {} of {} agents active", + group.as_str(), + window.as_str(), + cell.served_per_sec, + cell.share_of_baseline_percent + .map(|s| format!(" ({s:.1}% of baseline)")) + .unwrap_or_default(), + cell.submitted, + // Silence is the reading that stops a small rate being + // mistaken for residual service. + cell.quiet_ms + .filter(|_| cell.window_secs > 0.0) + .map(|q| { + format!( + ", answered nothing for {:.1}s of a {:.1}s window", + q as f64 / 1000.0, + cell.window_secs + ) + }) + .unwrap_or_default(), + cell.indeterminate, + cell.attempts_timed_out, + cell.agents_active, + if group == Group::OnPod { + self.isolated_agents + } else { + self.reachable_agents + }, + )); + } + } + } + + if self.recovery.count > 0 { + lines.push(format!( + "S3: isolated agents were served again p50 {}ms / p99 {}ms / worst {}ms after \ + the heal", + self.recovery.p50_ms, self.recovery.p99_ms, self.recovery.max_ms + )); + } + lines + } +} + +/// Operations of one outcome. +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::errors::ErrorClass; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use std::time::Duration; + use test_r::test; + + const ISOLATED: &str = "chaos-s3-durable-0000"; + const CONTROL: &str = "chaos-s3-durable-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ISOLATED.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the partition + /// was injected. Negative offsets are the baseline. + fn op(agent: &str, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream: Stream::Durable, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{agent}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// isolated invocation takes while the link is cut. + fn stalled(agent: &str, offset_secs: i64) -> OperationRecord { + let mut record = op(agent, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// A baseline both groups share, then a fault the isolated group is cut off + /// by and the control group sails through. `during_isolated` is how many + /// operations the isolated group still managed. + fn history(during_isolated: usize, during_control: usize) -> Vec { + let mut records = Vec::new(); + // 300s of baseline, one operation per second per agent. + for second in 1..=300 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, -second, Outcome::Confirmed)); + } + } + // 180s of fault. + for i in 0..during_isolated { + records.push(op(ISOLATED, i as i64, Outcome::Confirmed)); + } + for i in 0..during_control { + records.push(op(CONTROL, i as i64, Outcome::Confirmed)); + } + // 240s of recovery, both groups back to cadence. + for second in 181..=420 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, second, Outcome::Confirmed)); + } + } + records + } + + fn build(records: &[OperationRecord]) -> ReachabilityReport { + ReachabilityReport::build( + records, + &split(), + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ) + } + + /// The healthy shape: the cut half stops, the other half does not, and the + /// report says so without raising anything. + #[test] + fn a_partition_that_lands_and_costs_the_control_group_nothing_has_no_findings() { + let report = build(&history(2, 180)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert!( + isolated.share_of_baseline_percent.unwrap() < 25.0, + "the isolated group should have collapsed, got {isolated:?}" + ); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + assert!( + control.share_of_baseline_percent.unwrap() >= 75.0, + "the control group should have held, got {control:?}" + ); + } + + /// The inconclusive case, and the most important one in this module: a + /// partition that never took hold produces a report full of healthy numbers, + /// and it has to read as "this run tested nothing" rather than as a pass. + #[test] + fn an_isolated_group_that_kept_working_says_the_partition_never_landed() { + let report = build(&history(180, 180)); + + assert_eq!( + report + .findings + .iter() + .map(|f| f.violation) + .collect::>(), + vec![ReachabilityViolation::PartitionNotObserved] + ); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("Nothing else in this report")), + "the operator has to be told the rest of the report is meaningless" + ); + } + + /// The finding S3 exists to hunt: agents that were never partitioned from + /// anything losing throughput because their worker-service was busy waiting + /// on a pod they do not use. + #[test] + fn a_control_group_that_degraded_alongside_the_isolated_one_is_a_finding() { + // Half the control group's baseline rate, well under the 75% floor. + let report = build(&history(2, 90)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::ControlDegraded), + "expected collateral damage to be reported, got {:?}", + report.findings + ); + } + + /// Throughput, not raw counts. The baseline is 300s long and the fault + /// window 180s, so comparing totals would call an untouched group degraded + /// by 40% on window length alone. + #[test] + fn a_group_that_held_its_rate_is_not_penalised_for_a_shorter_fault_window() { + // 180 operations across a 180s window is exactly the baseline rate. + let report = build(&history(2, 180)); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + + assert_eq!(control.submitted, 180); + assert!( + (control.share_of_baseline_percent.unwrap() - 100.0).abs() < 1.0, + "an unchanged rate should read as ~100% of baseline, got {:?}", + control.share_of_baseline_percent + ); + } + + /// The pending-then-timeout behaviour the ticket asks to see in the history. + #[test] + fn attempts_that_hit_the_client_timeout_are_counted_per_cell() { + let mut records = history(0, 180); + records.push(stalled(ISOLATED, 10)); + let report = build(&records); + + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert_eq!(isolated.attempts_timed_out, 2); + assert_eq!(isolated.indeterminate, 1); + assert_eq!(isolated.confirmed, 0); + } + + /// Recovery is measured from the heal, not from when the operation was + /// submitted: an agent whose call was already in flight when the link came + /// back was served promptly, and the number has to say so. + #[test] + fn recovery_is_measured_from_the_heal() { + let mut records = history(0, 180); + // Submitted mid-fault, answered five seconds after the heal. + let mut late = op(ISOLATED, 100, Outcome::Confirmed); + late.completed_at = Some(t0() + TimeDelta::seconds(185)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(late); + + let report = build(&records); + assert_eq!(report.recovery.count, 1); + assert_eq!(report.recovery.max_ms, 5_000); + assert!(report.agents_never_recovered.is_empty()); + } + + /// An isolated agent that never answered again is the strongest finding this + /// report can make: the link is back and nothing else is wrong. + #[test] + fn an_isolated_agent_that_never_came_back_is_a_finding() { + let records: Vec = history(0, 180) + .into_iter() + .filter(|r| !(r.agent == ISOLATED && r.submitted_at >= t0())) + .collect(); + + let report = build(&records); + assert_eq!(report.agents_never_recovered, vec![ISOLATED.to_string()]); + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::NeverRecovered + && f.agent.as_deref() == Some(ISOLATED)) + ); + } + + /// Without the fault window there is no before-and-during to compare, so + /// every threshold in this report is undefined. It must count what it saw + /// and claim nothing — a verdict here would be invented. + #[test] + fn a_run_that_never_learned_when_the_fault_was_reports_counts_but_no_verdict() { + let report = ReachabilityReport::build( + &history(180, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert!(report.findings.is_empty()); + assert!(report.cells.iter().all(|c| c.window == Window::Unknown)); + assert!(report.cells.iter().all(|c| c.served_per_sec == 0.0)); + assert_eq!(report.recovery.count, 0); + // And nothing is silently blamed on the agents themselves. + assert!(report.agents_never_recovered.is_empty()); + } + + /// An agent the selection never saw is reported, not folded into a group. + /// It means the split and the workload disagree about who was driven, and + /// silently counting it as a control would corrupt the one comparison the + /// whole scenario rests on. + #[test] + fn operations_against_an_unknown_agent_are_reported_rather_than_grouped() { + let mut records = history(2, 180); + records.push(op("chaos-s3-durable-9999", -10, Outcome::Confirmed)); + + let report = build(&records); + assert_eq!(report.records_outside_the_split, 1); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("the ownership split never saw")) + ); + } + + /// Two hundred agents that all failed to recover is one fact, not two + /// hundred, and an artifact carrying all of them is unreadable. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let many: Vec = (0..MAX_FINDINGS + 10) + .map(|i| format!("chaos-s3-durable-{i:04}")) + .collect(); + let mut split = split(); + split.on_pod = many.clone(); + + let report = ReachabilityReport::build( + &[], + &split, + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert_eq!(report.agents_never_recovered.len(), MAX_FINDINGS + 10); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + } + + /// Over budget is context, not a finding: how long worker-service's retry + /// loop may take to notice a healed link is a judgement, and the driver is + /// not the one to make it. + #[test] + fn a_slow_recovery_is_an_attention_line_rather_than_a_finding() { + let mut records = history(0, 180); + let mut slow = op(ISOLATED, 100, Outcome::Confirmed); + slow.completed_at = Some(t0() + TimeDelta::seconds(300)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(slow); + + let report = build(&records); + assert_eq!(report.recovery_over_budget, 1); + assert!(report.findings.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("recovery budget")) + ); + } + + /// An operation the cut landed in the middle of: submitted just before it, + /// still running when it landed, answered only once the link returned. + fn caught(agent: &str, timed_out_attempts: u32) -> OperationRecord { + let mut record = op(agent, -1, Outcome::Confirmed); + record.duration_ms = 178_000; + record.completed_at = Some(record.submitted_at + TimeDelta::milliseconds(178_000)); + record.attempts = timed_out_attempts + 1; + record.attempt_log = (1..=timed_out_attempts) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .chain(std::iter::once(AttemptRecord { + attempt: timed_out_attempts + 1, + started_at: record.submitted_at, + duration_ms: 56_000, + returned_value: Some(1), + succeeded: true, + error_class: None, + error: None, + })) + .collect(); + record + } + + /// The population the fault actually disturbed, which no cell can show: + /// these were submitted before the cut, so their timeouts and their + /// duration are attributed to the `before-fault` row. + #[test] + fn the_operations_the_cut_caught_are_reported_apart_from_the_cells() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + records.push(caught(ISOLATED, 1)); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the isolated group had an operation in flight"); + + assert_eq!(caught.operations, 1); + assert_eq!(caught.agents, 1); + assert_eq!(caught.confirmed, 1); + assert_eq!(caught.duration.max_ms, 178_000); + assert_eq!(caught.attempts_timed_out, 1); + // The retry is what landed it. With retries off it would have been + // indeterminate, and the outcome alone cannot say so. + assert_eq!(caught.max_attempts, 2); + // It answered before the heal was stamped, so it did not outlive it. + assert_eq!(caught.outlived_the_fault, 0); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("were in flight when the cut landed")), + "the caught population has to be in front of the reader" + ); + } + + /// An operation still unanswered when the link came back is a different + /// statement from one that resolved inside the window, and the report has + /// to keep them apart. + #[test] + fn an_operation_still_running_at_the_heal_is_counted_as_outliving_it() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut late = caught(ISOLATED, 1); + late.completed_at = Some(t0() + TimeDelta::seconds(200)); + late.duration_ms = 201_000; + records.push(late); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .unwrap(); + assert_eq!(caught.outlived_the_fault, 1); + } + + /// The number that stops a small during-fault rate reading as residual + /// service. A real run showed 1.1% of baseline from operations that all + /// arrived in the last four seconds of a 182-second window, once the fault + /// was already coming down. + #[test] + fn a_group_silent_until_the_heal_reports_how_long_it_answered_nothing() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + // A late tail, exactly as the heal window produces. + for offset in 176..180 { + records.push(op(ISOLATED, offset, Outcome::Confirmed)); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 4); + assert_eq!(cell.quiet_ms, Some(176_020)); + assert!( + cell.quiet_ms.unwrap() as f64 / 1000.0 > cell.window_secs * 0.9, + "silence has to dominate the window, not trail it" + ); + // The same group's baseline answers once a second, so its longest + // silence is one interval. That contrast is the whole reading: 176s of + // silence is not a slower version of this, it is a different state. + let baseline = report.cell(Group::OnPod, Window::BeforeFault).unwrap(); + assert_eq!(baseline.quiet_ms, Some(1_000)); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("answered nothing for 176.0s of a 180.0s window")), + "notes were {:?}", + report.note_lines() + ); + } + + /// The regression the first S16 run turned up, in the module that shares + /// the construction. + /// + /// The workload keeps offering work to an unreachable executor all through + /// the partition, so anything derived from submission times looks busy no + /// matter what the platform is doing. Every one of these is offered during + /// the fault and answered only after the heal: the fault window served the + /// isolated group nothing, and the cell has to say so. + #[test] + fn work_offered_to_an_isolated_group_and_answered_after_the_heal_is_not_service() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + for offset in 0..120 { + let mut record = op(ISOLATED, offset, Outcome::Confirmed); + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - offset) as u64 * 1_000; + records.push(record); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 120, "they were offered during the fault"); + assert_eq!(cell.confirmed, 120, "and they did all eventually confirm"); + assert_eq!(cell.served, 0, "but none of it was served during the fault"); + assert_eq!(cell.served_per_sec, 0.0); + assert_eq!(cell.quiet_ms, Some(180_000), "silent for the whole window"); + } + + /// Work the cut caught and the run cannot account for. Not a finding, but + /// it must sit next to the clean cells rather than be inferred from them. + #[test] + fn caught_work_that_never_confirmed_is_raised_to_the_operator() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut lost = caught(ISOLATED, 2); + lost.outcome = Outcome::Indeterminate; + records.push(lost); + + let report = build(&records); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught in flight") && l.contains("did not confirm")), + "attention was {:?}", + report.attention_lines() + ); + // Still not a finding: doubt is not damage, and the exactly-once and + // read-back accounts are what resolve it. + assert!(report.findings.is_empty()); + } + + /// A run with no fault window cannot say what was in flight when the cut + /// landed, because it does not know when that was. + #[test] + fn a_run_without_a_fault_window_claims_nothing_was_caught() { + let report = ReachabilityReport::build( + &history(2, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + assert!(report.caught_in_flight.is_empty()); + assert!(report.cells.iter().all(|c| c.quiet_ms.is_none())); + } +} diff --git a/integration-tests/src/chaos/relay.rs b/integration-tests/src/chaos/relay.rs new file mode 100644 index 0000000000..65084094fd --- /dev/null +++ b/integration-tests/src/chaos/relay.rs @@ -0,0 +1,1832 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a fault cost the agents calling across executors (GOL-368, GOL-382). +//! +//! Two scenarios share this report and disagree about which way its numbers +//! should point, which is what [`RelayExpectation`] is for. +//! +//! **S2 injects a fault it expects to be inert**, and measures that nothing +//! happened. The claim under test is architectural: an executor never opens a +//! connection to another executor. When an agent invokes an agent its own +//! executor does not own, `DirectWorkerInvocationRpc` hands the call to +//! `worker_proxy`, which is a client of *worker-service*. The reply comes back +//! the same way. So executor A reaches executor B's agents by asking a third +//! party, and cutting the A-to-B link cuts nothing. +//! +//! **S21 injects a fault aimed at that third party**, and measures what it +//! cost. Same populations, same cells, opposite verdict. +//! +//! ### What "the premium" is +//! +//! The one piece of vocabulary these scenarios invent, so it is worth spelling +//! out. The RPC stream has each agent call another agent, and the callee lands +//! on the caller's own executor or on the other one purely by how their agent +//! ids hash. +//! +//! Every call starts driver -> worker-service -> executor. A **co-located** call +//! ends there, because the callee is an agent that executor already owns. A +//! **cross-pod** call cannot: an executor never opens a connection to another +//! executor, so `DirectWorkerInvocationRpc` hands the call to `worker_proxy`, +//! which is a client of worker-service, and the reply comes back the same way. +//! +//! So a cross-pod call crosses worker-service twice and a co-located one crosses +//! it once. Subtract the two median latencies and everything they share +//! cancels: the driver's own round trip, the first crossing, the agent's work. +//! What is left is the one extra crossing only cross-pod calls pay, which is +//! **what it costs an agent to have its callee on another pod**. S2 measured +//! 38ms and 50ms; S21 has seen 22ms at the suite's usual rate and 43-50ms at +//! eight times it. +//! +//! Throughput cannot do this job. The driver sets the cadence, so both +//! populations run at the rate they were asked to whether or not a call leaves +//! the pod: S2's first run measured 9.51/s cross-pod and 10.49/s co-located +//! before, during and after a partition, and those cells look identical on a +//! healthy run *and* on a run whose pairing was broken. +//! +//! The premium's value is that it survives a window where *everything* is +//! slower, because anything hitting both populations equally drops out of a +//! difference. Its limit is the same sentence read backwards: a fault costing +//! both crossings the same leaves it flat while both populations degrade. S21 +//! met both cases — the fault hit only the second crossing at 100 ops/s and both +//! crossings at 800 — so each population's own p50 is reported beside the +//! premium, and no verdict rests on the premium alone. +//! +//! ### Why either scenario needs more oracles than a normal one +//! +//! A scenario that expects damage fails safe: if the fault misses, the damage is +//! absent and the report says the fault was not observed. Neither of these does. +//! S2 fails the other way — a run where the pairs were accidentally co-located, +//! where the workload never started, or where the partition was never injected +//! all produce the same clean report as a run that proved the point. S21 fails +//! the same way one step along: a stress that missed worker-service produces the +//! same undisturbed numbers as a platform that shrugged the load off. +//! +//! So the numbers here exist mostly to stop a run passing for the wrong reason: +//! +//! 1. **Were the pairs actually split?** [`Placement::CrossPod`] must hold at +//! least `cross_pod_floor_percent` of the callers. See +//! [`RelayViolation::PairingTooThin`]. +//! 2. **Are the two populations really two?** A cross-pod call has to cost more +//! than a co-located one on the undisturbed baseline. See +//! [`RelayViolation::CrossPodNotRelayed`]. +//! 3. **Did the fault do what the scenario said it would?** Under `Inert`, a +//! cross-pod drop is the finding. Under `RelayDegraded`, a premium that never +//! widened is. See [`RelayViolation::CrossPodDegraded`] and +//! [`RelayViolation::RelayDidNotRecover`]. +//! 4. **Did it stop when the fault stopped?** Shared, because a bounded fault +//! has to have a bounded effect. See +//! [`RelayViolation::CrossPodDidNotReturn`]. +//! +//! ### The one thing S2 cannot check +//! +//! Whether its partition took hold. Every other partition scenario confirms the +//! fault landed by watching something stop; there nothing is supposed to stop, +//! so there is no in-cluster evidence to read and the run relies entirely on +//! Chaos Mesh reporting `AllInjected`. S21 is not in that position — the premium +//! is its own evidence — and [`RelayReport::partition_evidence`] says which of +//! the two a reader is holding. +//! +//! ### Throughput, not success rate +//! +//! For the same reason as [`crate::chaos::reachability`]: a stalled agent fails +//! slowly and once, which a success rate reads as one failure out of one +//! attempt. Confirmed operations per second is what a success rate cannot say. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::pinned::routing_agent_id_in; +use crate::chaos::split::{ + FaultWindow, Window, longest_silence_ms, round2, window_end, window_secs, window_start, +}; +use crate::chaos::summary::LatencyStats; +use crate::chaos::workload::{COUNTER_AGENT, WorkloadContext, rpc_callee_name}; +use crate::chaos::{RelayConfig, ScenarioCode}; +use chrono::{DateTime, Utc}; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use tracing::info; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// How far apart the two populations' shares of their own baselines have to be +/// before the difference is read as one of them being hurt more. +/// +/// The driver sets the cadence for both, so on an undisturbed run they land +/// within a point of each other: S2's two green runs measured cross-pod at +/// 100.11% against co-located at 99.9%, then 100.23% against 99.91%. Without a +/// margin, ordinary jitter of that size would file +/// [`RelayViolation::CoLocatedDegradedMore`] on any run where the coin landed +/// the other way up. +const PLACEMENT_SHARE_MARGIN_PERCENT: f64 = 5.0; + +/// Where the two halves of an RPC pair ended up. +/// +/// Not a property of the fault, unlike [`crate::chaos::split::Group`]. Both +/// halves are placed by hashing their agent ids onto shards, so this records +/// what the hash gave rather than what the driver chose. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Placement { + /// Caller and callee are owned by different executors, so the call crosses + /// the partitioned link. The population the scenario is about. + CrossPod, + /// Caller and callee are owned by the same executor, so the call never + /// leaves the pod. The run's own control. + CoLocated, +} + +impl Placement { + pub fn as_str(self) -> &'static str { + match self { + Placement::CrossPod => "cross-pod", + Placement::CoLocated => "co-located", + } + } +} + +impl std::fmt::Display for Placement { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What the run's fault is supposed to do to the relay. +/// +/// The two scenarios built on this pairing measure the same things and disagree +/// only about which way the numbers should point, so the split lives here +/// rather than in two copies of the report. +/// +/// It is not a cosmetic label. Under [`Self::Inert`] a cross-pod population that +/// fell behind is the finding; under [`Self::RelayDegraded`] it is the expected +/// result and the finding is the opposite one, a fault that changed nothing. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RelayExpectation { + /// The fault cannot reach the relay path, and the run exists to show that. + /// + /// S2: two executors cut off from each other, when no executor ever opens a + /// connection to another executor. The default, because it is what this + /// module was built for. + #[default] + Inert, + /// The fault is aimed at the relay itself, and the run exists to measure + /// what that costs without losing work. + /// + /// S21: worker-service starved of CPU while both populations depend on it. + RelayDegraded, +} + +impl RelayExpectation { + pub fn as_str(self) -> &'static str { + match self { + RelayExpectation::Inert => "inert", + RelayExpectation::RelayDegraded => "relay-degraded", + } + } +} + +impl std::fmt::Display for RelayExpectation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Something about the run worth an operator's attention. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RelayViolation { + /// Too few pairs straddled the two executors for the partition to have had + /// anything to cut. The run is inconclusive, not clean. + PairingTooThin, + /// Cross-pod calls were served materially less during the partition while + /// co-located calls were not. **This is the finding the scenario exists to + /// make**: it says the two executors were reaching each other directly. + CrossPodDegraded, + /// Both populations dropped together. Something disturbed the cluster, but + /// a fault that hurts calls which never leave a pod is not evidence about + /// the link between pods. + BothDegraded, + /// Cross-pod calls never recovered their baseline rate after the heal, even + /// though they held up during the fault. Late damage, and still worth a + /// look. + CrossPodDidNotReturn, + /// Cross-pod calls cost no more than co-located ones on the undisturbed + /// baseline, so the population labelled cross-pod was not paying for a + /// network hop and this run measured nothing. + /// + /// The guard against the whole scenario being quietly vacuous. Every other + /// check here compares the two populations *through* the fault, and all of + /// them pass trivially if the two populations are really the same thing. + CrossPodNotRelayed, + /// Latency was still above its baseline after the fault healed and a whole + /// recovery window had passed. + /// + /// [`RelayExpectation::RelayDegraded`] only, and the one thing this + /// scenario fails on. It deliberately does *not* fail on the fault having + /// been small: the run exists to see what a saturated relay costs, and a + /// platform that shrugs it off is a result rather than a broken test. A + /// bounded fault whose cost outlives it is not. + RelayDidNotRecover, + /// Co-located calls lost more of their throughput than cross-pod ones did. + /// + /// [`RelayExpectation::RelayDegraded`] only, and backwards for a fault on + /// the shared relay. A cross-pod call crosses worker-service twice and a + /// co-located one crosses it once, so a fault on worker-service cannot hurt + /// the shorter path more. + /// + /// Which makes this the same kind of statement as [`Self::BothDegraded`] is + /// for the control: something other than the fault disturbed the run. It is + /// not "the load hit the executors" — on golem-dev it cannot, because the + /// executors hold a dedicated node pool that worker-service's own node + /// selector excludes. The candidates are an executor that restarted, and a + /// shard reassignment that left the pairing describing agents which have + /// since moved. + CoLocatedDegradedMore, +} + +impl RelayViolation { + pub fn as_str(self) -> &'static str { + match self { + RelayViolation::PairingTooThin => "pairing-too-thin", + RelayViolation::CrossPodDegraded => "cross-pod-degraded", + RelayViolation::BothDegraded => "both-degraded", + RelayViolation::CrossPodDidNotReturn => "cross-pod-did-not-return", + RelayViolation::CrossPodNotRelayed => "cross-pod-not-relayed", + RelayViolation::RelayDidNotRecover => "relay-did-not-recover", + RelayViolation::CoLocatedDegradedMore => "co-located-degraded-more", + } + } +} + +impl std::fmt::Display for RelayViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One finding, with the evidence behind it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelayFinding { + pub violation: RelayViolation, + pub detail: String, +} + +/// How each caller's pair was placed, decided before the fault. +/// +/// Built once and carried, rather than recomputed while reading the history. +/// Ownership can move during a run, and a report that classified the same +/// operation differently depending on when it was read would be worse than one +/// that states the placement it measured against. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelayPairing { + /// The two executors, as the shard-manager names them. + pub pods: Vec, + /// Caller agents whose callee is on the other executor. + pub cross_pod: Vec, + /// Caller agents whose callee is on the same executor. + pub co_located: Vec, + /// Callers whose owner or callee's owner the routing table did not resolve. + /// Counted rather than assigned: a guess here would put an operation in the + /// wrong population and the report is a comparison between populations. + pub unresolved: Vec, +} + +impl RelayPairing { + /// Which population a caller belongs to, or `None` if it was never placed. + pub fn placement_of(&self, caller: &str) -> Option { + if self.cross_pod.iter().any(|a| a == caller) { + Some(Placement::CrossPod) + } else if self.co_located.iter().any(|a| a == caller) { + Some(Placement::CoLocated) + } else { + None + } + } + + /// The share of placed callers that straddle the two executors. + /// + /// Denominator is the callers that were *placed*, not every caller + /// configured. An unresolved caller says nothing either way, and counting + /// it against the share would let a flaky routing-table read block a run + /// whose pairs were split perfectly well. Same reasoning as S9's exclusion + /// of unreadable agents from its forward-leg share. + pub fn cross_pod_percent(&self) -> Option { + let placed = self.cross_pod.len() + self.co_located.len(); + if placed == 0 { + return None; + } + Some(round2(100.0 * self.cross_pod.len() as f64 / placed as f64)) + } +} + +/// One population's throughput in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelayCell { + pub placement: Placement, + pub window: Window, + /// Callers of this population that offered at least one operation here. + pub agents_active: usize, + /// Operations *offered* in this window, attributed by submission time, and + /// how they eventually ended up. + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. This + /// is the one that says whether the population was being served. + pub served: u64, + pub window_secs: f64, + pub served_per_sec: f64, + /// This cell's rate against the same population's own before-fault rate. + /// `None` for the before-fault cell itself. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// The longest stretch of this window in which nothing was answered. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub longest_silence_ms: Option, + /// Round-trip latency of the operations answered in this window. + /// + /// The most sensitive instrument this report has, and on the first run the + /// only one that could tell the two populations apart at all. Throughput is + /// set by the driver's own cadence, so both populations sit at the rate they + /// were asked to run at whether or not a call leaves the pod. Latency is + /// not: a cross-pod call pays executor -> worker-service -> executor, and + /// that hop shows up as a flat premium per call. + /// + /// So this is where the architecture is actually visible. A *change* in the + /// premium across the fault window would say the relay path changed under a + /// partition it is supposed to be indifferent to. + pub latency: LatencyStats, +} + +/// The whole relay verdict. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelayReport { + /// Which scenario produced this. Carried so the report's own prose names it + /// rather than every line hard-coding one of the two scenarios that build + /// this report. + pub scenario: ScenarioCode, + /// Which way the numbers were supposed to point. + pub expectation: RelayExpectation, + /// How the pairs were placed, and against which executors. + pub pairing: RelayPairing, + pub cross_pod_percent: Option, + /// The thresholds from the suite YAML, recorded so an archived result can + /// be read against the numbers it was judged by. + pub cross_pod_floor_percent: f64, + pub cross_pod_floor_throughput_percent: f64, + pub co_located_floor_throughput_percent: f64, + pub cross_pod_premium_floor_ms: u64, + /// How far above its baseline latency may still sit after the heal before + /// the run says the fault outlived itself. Read under + /// [`RelayExpectation::RelayDegraded`] only; 100 would be a p50 that + /// returned exactly. + pub recovery_floor_percent: f64, + /// How much more a cross-pod call cost than a co-located one on the + /// undisturbed baseline, in milliseconds of p50. `None` when either + /// population had no baseline. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cross_pod_premium_ms: Option, + /// The same premium measured inside the fault window. + /// + /// The difference between the two populations is one worker-service hop, so + /// this number is that hop under whatever the fault did to it. Recorded for + /// both expectations: S21 judges it, and S2 gains a second way of saying its + /// partition changed nothing, since a partition between executors has no + /// business moving the cost of a relay through a third party. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cross_pod_premium_during_fault_ms: Option, + /// The fault-window premium as a percentage of the baseline premium, so + /// **100 means it did not move** and 250 means it is two and a half times + /// as wide. Same convention as + /// [`RelayCell::share_of_baseline_percent`]. + /// + /// `None` when either premium is missing, or when the baseline premium is + /// zero and a percentage would divide by it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cross_pod_premium_inflation_percent: Option, + /// Each population's p50 inside the fault window as a percentage of its own + /// p50 outside it, so **100 means it did not move**. + /// + /// The reading the premium cannot give. A premium is a difference between + /// the two populations and stays flat whenever a fault costs both + /// traversals the same, which is exactly what happened when S21 ran at + /// eight times the suite's rate: both p50s rose over a fifth and the + /// premium moved 18 points. These two say whether the fault landed at all; + /// the premium says whether it landed harder on the relay hop. + /// + /// Taken as the *smaller* of the ratios against the baseline and the + /// recovery window. A baseline carries cold starts and a recovery window + /// does not, so either comparison alone can be flattered by which end it + /// was taken from, and the smaller one is the reading no drift inflates. + /// `None` when the fault window is missing, or when nothing outside it is. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cross_pod_latency_inflation_percent: Option, + /// The same reading for the pairs that stayed on one executor. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub co_located_latency_inflation_percent: Option, + pub cells: Vec, + /// What is, and is not, known about the fault having taken hold. Spelled + /// out because this scenario has no in-cluster evidence of its own — see + /// the module docs. + pub partition_evidence: String, + /// Records whose caller the pairing never placed. Non-zero means the + /// pairing and the workload disagree about who was driven. + pub records_outside_the_pairing: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], dropped rather than carried. + pub findings_omitted: u64, +} + +impl RelayReport { + /// Whether the run produced a verdict an operator has to act on. + pub fn has_findings(&self) -> bool { + !self.findings.is_empty() + } + + /// One cell, if the run produced it. + pub fn cell(&self, placement: Placement, window: Window) -> Option<&RelayCell> { + self.cells + .iter() + .find(|c| c.placement == placement && c.window == window) + } + + /// Lines an operator has to read. + pub fn attention_lines(&self) -> Vec { + let code = self.scenario; + let mut lines: Vec = self + .findings + .iter() + .map(|f| format!("{code} {}: {}", f.violation.as_str(), f.detail)) + .collect(); + + if self.findings_omitted > 0 { + lines.push(format!( + "{code}: {} further relay finding(s) were dropped from the report", + self.findings_omitted + )); + } + if self.records_outside_the_pairing > 0 { + lines.push(format!( + "{code}: {} operation(s) ran against callers the pairing never placed, so they \ + are in no population and in no cell — the pairing and the workload disagree \ + about who was driven", + self.records_outside_the_pairing + )); + } + lines + } + + /// Context a reader needs to judge the numbers, findings or not. + /// + /// The evidence line goes here rather than into [`Self::attention_lines`] + /// deliberately. It is a standing property of the scenario, true of every + /// run including the good ones, and an attention item that fires every + /// single time teaches a reader to skip attention items. + /// + /// Under [`RelayExpectation::RelayDegraded`] the degradation itself is a + /// note for the same reason. The run was asked to hurt these calls, so + /// reporting that it did as an attention item would bury the one line that + /// says whether the platform stayed correct while it happened. + pub fn note_lines(&self) -> Vec { + let code = self.scenario; + let mut lines = vec![format!("{code}: {}", self.partition_evidence)]; + + if let Some(percent) = self.cross_pod_percent { + lines.push(format!( + "{code}: {percent}% of placed callers had their callee on the other executor \ + ({} cross-pod, {} co-located, {} unplaced)", + self.pairing.cross_pod.len(), + self.pairing.co_located.len(), + self.pairing.unresolved.len() + )); + } + + if let Some(premium) = self.cross_pod_premium_ms { + lines.push(format!( + "{code}: a cross-pod call cost {premium}ms more than a co-located one at p50 on \ + the undisturbed baseline. That premium is the relay hop through worker-service, \ + and it is the evidence that the two populations really are split" + )); + } + + if let Some(during) = self.cross_pod_premium_during_fault_ms { + let movement = match self.cross_pod_premium_inflation_percent { + Some(percent) => format!("{percent}% of its baseline width"), + None => "an unmeasured share of its baseline width".to_string(), + }; + // The same measurement means opposite things to the two scenarios, + // so the sentence that interprets it has to differ. Reporting one + // reading under both would leave half the runs carrying a line that + // argues against what they set out to show. + let reading = match self.expectation { + RelayExpectation::Inert => { + "A partition between two executors has no business moving the cost of a relay \ + through a third party, so this is a second way of saying the fault was inert" + } + RelayExpectation::RelayDegraded => { + "That premium is one worker-service hop and nothing else, so this is where \ + this run's degradation lives" + } + }; + lines.push(format!( + "{code}: during the fault that same premium was {during}ms, {movement}. {reading}" + )); + } + + if let (Some(cross), Some(co)) = ( + self.cell(Placement::CrossPod, Window::DuringFault), + self.cell(Placement::CoLocated, Window::DuringFault), + ) { + let share = |cell: &RelayCell| { + cell.share_of_baseline_percent + .map(|s| s.to_string()) + .unwrap_or_else(|| "an unmeasured".to_string()) + }; + // Throughput is the blunt instrument here and the line says so + // under the expectation that would otherwise be read wrong. A + // degradation run whose cells both sit near 100% has not failed to + // degrade anything; the driver sets the cadence, so these hold until + // the platform cannot keep up at all, and the premium line above is + // where the degradation actually shows. + let caveat = match self.expectation { + RelayExpectation::Inert => "", + RelayExpectation::RelayDegraded => { + ". The driver sets the cadence, so both hold up until the platform cannot \ + keep up at all — these are the SLO breach when it comes, not the measurement" + } + }; + lines.push(format!( + "{code}: during the fault cross-pod ran at {}% of its own baseline and \ + co-located at {}%{caveat}", + share(cross), + share(co), + )); + } + + lines + } +} + +/// Places every caller's pair by asking the shard-manager who owns each half. +/// +/// Runs before the fault, and its answer is then fixed for the whole run. Shard +/// ownership can move, and a report that re-derived placement while reading the +/// history could file the same caller under both populations. +/// +/// Fails rather than proceeding unplaced. A run whose pairs were never resolved +/// would still produce two throughput cells and a clean-looking verdict, and +/// that verdict would be about nothing. +pub async fn select_pairing( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + callers: &[String], +) -> anyhow::Result { + let table = deps + .shard_manager() + .get_routing_table() + .await + .map_err(|e| anyhow::anyhow!("S2: could not read the routing table: {e:?}"))?; + + let mut cross_pod = Vec::new(); + let mut co_located = Vec::new(); + let mut unresolved = Vec::new(); + let mut pods: BTreeSet = BTreeSet::new(); + + for caller in callers { + let callee = rpc_callee_name(caller); + let caller_pod = table.lookup(&routing_agent_id_in(&ctx.counters, COUNTER_AGENT, caller)); + let callee_pod = table.lookup(&routing_agent_id_in(&ctx.counters, COUNTER_AGENT, &callee)); + + match (caller_pod, callee_pod) { + (Some(from), Some(to)) => { + pods.insert(from.to_string()); + pods.insert(to.to_string()); + if from == to { + co_located.push(caller.clone()); + } else { + cross_pod.push(caller.clone()); + } + } + // One half unresolved says nothing about the pair. Recorded, not + // guessed: see `RelayPairing::cross_pod_percent`. + _ => unresolved.push(caller.clone()), + } + } + + if cross_pod.is_empty() && co_located.is_empty() { + anyhow::bail!( + "S2: the routing table placed none of the {} RPC callers, so the run has no \ + populations to compare", + callers.len() + ); + } + + let pairing = RelayPairing { + pods: pods.into_iter().collect(), + cross_pod, + co_located, + unresolved, + }; + info!( + "S2: {} of {} callers call across executors ({}%), {} stay on one, {} unplaced; \ + executors seen: {}", + pairing.cross_pod.len(), + callers.len(), + pairing + .cross_pod_percent() + .map(|p| p.to_string()) + .unwrap_or_else(|| "no".to_string()), + pairing.co_located.len(), + pairing.unresolved.len(), + pairing.pods.join(", ") + ); + Ok(pairing) +} + +/// One population's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + served_at: Vec>, + /// Durations of the operations *answered* in this window, so the latency + /// stats and the served count describe the same set of operations. + served_ms: Vec, +} + +/// Which window an instant fell in. +fn window_of(at: DateTime, fault: Option) -> Window { + let Some(fault) = fault else { + return Window::Unknown; + }; + if at < fault.injected_at { + Window::BeforeFault + } else if fault.recovered_at.is_some_and(|healed| at >= healed) { + Window::AfterFault + } else { + Window::DuringFault + } +} + +/// Builds the report from the run's history. +/// +/// `records` is the whole history; only [`Stream::Rpc`] entries are read, so a +/// caller does not have to pre-filter and cannot filter differently from the +/// way the cells are built. +pub fn build( + records: &[OperationRecord], + pairing: RelayPairing, + fault: Option, + scenario: ScenarioCode, + config: &RelayConfig, +) -> RelayReport { + let mut tallies: BTreeMap<(Placement, Window), Tally> = BTreeMap::new(); + let mut first_submitted: BTreeMap> = BTreeMap::new(); + let mut last_completed: BTreeMap> = BTreeMap::new(); + let mut records_outside_the_pairing = 0u64; + + for record in records.iter().filter(|r| r.stream == Stream::Rpc) { + let Some(placement) = pairing.placement_of(&record.agent) else { + records_outside_the_pairing += 1; + continue; + }; + + first_submitted + .entry(placement) + .and_modify(|at| *at = (*at).min(record.submitted_at)) + .or_insert(record.submitted_at); + + // Offered work is filed by submission time: that is when the platform + // was asked, which is the question a throughput cell answers. + let offered = tallies + .entry((placement, window_of(record.submitted_at, fault))) + .or_default(); + offered.agents.insert(record.agent.clone()); + offered.submitted += 1; + match record.outcome { + Outcome::Confirmed => offered.confirmed += 1, + Outcome::Rejected => offered.rejected += 1, + Outcome::Indeterminate => offered.indeterminate += 1, + } + + // Answered work is filed by completion time, which is a different + // window whenever an operation was held across an edge. + if let Some(completed_at) = record.completed_at { + last_completed + .entry(placement) + .and_modify(|at| *at = (*at).max(completed_at)) + .or_insert(completed_at); + if record.outcome == Outcome::Confirmed { + let answered = tallies + .entry((placement, window_of(completed_at, fault))) + .or_default(); + answered.served_at.push(completed_at); + answered.served_ms.push(record.duration_ms); + } + } + } + + let mut cells: Vec = Vec::new(); + let mut baseline_rate: BTreeMap = BTreeMap::new(); + + // Before-fault first, so every later cell has a baseline to divide by. + for window in [ + Window::BeforeFault, + Window::DuringFault, + Window::AfterFault, + Window::Unknown, + ] { + for placement in [Placement::CrossPod, Placement::CoLocated] { + let Some(tally) = tallies.get(&(placement, window)) else { + continue; + }; + let first = first_submitted.get(&placement).copied(); + let last = last_completed.get(&placement).copied(); + let secs = window_secs(window, fault, first, last); + let served = tally.served_at.len() as u64; + let served_per_sec = if secs > 0.0 { + round2(served as f64 / secs) + } else { + 0.0 + }; + + if window == Window::BeforeFault && served_per_sec > 0.0 { + baseline_rate.insert(placement, served_per_sec); + } + let share = if window == Window::BeforeFault { + None + } else { + baseline_rate + .get(&placement) + .map(|base| round2(100.0 * served_per_sec / base)) + }; + + cells.push(RelayCell { + placement, + window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + served, + window_secs: round2(secs), + served_per_sec, + share_of_baseline_percent: share, + latency: LatencyStats::from_durations(tally.served_ms.clone()), + longest_silence_ms: longest_silence_ms( + &tally.served_at, + window_start(window, fault, first), + window_end(window, fault, last), + ), + }); + } + } + + // The gap between the two populations in one window, in milliseconds of + // p50. Both populations cross worker-service; only the cross-pod one crosses + // it twice, so whatever is left after the subtraction is one relay hop and + // nothing else. That is what makes this a usable instrument in a window + // where *everything* is slower. + let premium_in = |window: Window| { + let p50 = |placement: Placement| { + cells + .iter() + .find(|c| c.placement == placement && c.window == window) + .map(|c| c.latency.p50_ms as i64) + }; + match (p50(Placement::CrossPod), p50(Placement::CoLocated)) { + (Some(cross), Some(co)) => Some(cross - co), + _ => None, + } + }; + + // The baseline premium is a statement about the workload rather than about + // the fault: it says the two populations really are split. Measuring it + // during the fault instead would confuse "the pairing is wrong" with "the + // fault did something". + let cross_pod_premium_ms = premium_in(Window::BeforeFault); + let cross_pod_premium_during_fault_ms = premium_in(Window::DuringFault); + let cross_pod_premium_inflation_percent = + match (cross_pod_premium_ms, cross_pod_premium_during_fault_ms) { + // A zero baseline has no width to grow by, and a percentage of it would + // be a division by zero dressed up as a measurement. The run is already + // reporting `cross-pod-not-relayed` in that case, which is the more + // useful thing to say. + (Some(baseline), Some(during)) if baseline > 0 => { + Some(round2(100.0 * during as f64 / baseline as f64)) + } + _ => None, + }; + + // The second reading, and the one that does not care which hop the cost + // landed on. Each population against itself, taken as the smaller of the + // ratios either side of the window so that neither a warm-up-heavy baseline + // nor a clean recovery window can be the reason a run passes. + let latency_inflation = |placement: Placement| { + let p50 = |window: Window| { + cells + .iter() + .find(|c| c.placement == placement && c.window == window) + .map(|c| c.latency.p50_ms as f64) + }; + let during = p50(Window::DuringFault)?; + [Window::BeforeFault, Window::AfterFault] + .into_iter() + .filter_map(p50) + .filter(|outside| *outside > 0.0) + .map(|outside| 100.0 * during / outside) + .reduce(f64::min) + .map(round2) + }; + let cross_pod_latency_inflation_percent = latency_inflation(Placement::CrossPod); + let co_located_latency_inflation_percent = latency_inflation(Placement::CoLocated); + + let cross_pod_percent = pairing.cross_pod_percent(); + let mut report = RelayReport { + scenario, + expectation: config.expectation, + pairing, + cross_pod_percent, + cross_pod_floor_percent: config.cross_pod_floor_percent, + cross_pod_floor_throughput_percent: config.cross_pod_floor_throughput_percent, + co_located_floor_throughput_percent: config.co_located_floor_throughput_percent, + cross_pod_premium_floor_ms: config.cross_pod_premium_floor_ms, + recovery_floor_percent: config.recovery_floor_percent, + cross_pod_premium_ms, + cross_pod_premium_during_fault_ms, + cross_pod_premium_inflation_percent, + cross_pod_latency_inflation_percent, + co_located_latency_inflation_percent, + cells, + partition_evidence: fault_evidence(fault, config.expectation), + records_outside_the_pairing, + findings: Vec::new(), + findings_omitted: 0, + }; + report.findings = judge(&report); + if report.findings.len() > MAX_FINDINGS { + report.findings_omitted = (report.findings.len() - MAX_FINDINGS) as u64; + report.findings.truncate(MAX_FINDINGS); + } + report +} + +/// What can honestly be said about the fault having landed. +/// +/// The two expectations are in genuinely different positions here, and the line +/// says which one the reader is holding. An inert fault leaves nothing in the +/// cluster to observe, so the run has only Chaos Mesh's word for it. A fault +/// aimed at the relay leaves a mark in the run's own numbers, and the report +/// points at it rather than asking to be trusted. +fn fault_evidence(fault: Option, expectation: RelayExpectation) -> String { + match fault { + None => "the run never learned when the fault was injected, so every cell here is filed \ + under an unknown window and none of them can be read against it" + .to_string(), + Some(window) if window.recovered_at.is_none() => { + "the run saw the fault injected but never saw it healed, so the during-fault window \ + runs to the last operation rather than to the heal" + .to_string() + } + Some(_) => match expectation { + RelayExpectation::Inert => { + "Chaos Mesh reporting AllInjected is the only evidence that the partition took \ + hold. Unlike every other partition scenario there is nothing in the cluster \ + that is supposed to stop, so a clean result here cannot by itself distinguish \ + an inert fault from an absent one" + .to_string() + } + RelayExpectation::RelayDegraded => { + "the fault is aimed at the relay itself, so unlike the control that shares these \ + populations this run carries its own evidence: the cross-pod premium is one \ + worker-service hop, and a premium that widened during the window says the \ + fault reached it" + .to_string() + } + }, + } +} + +/// Turns the cells into findings. +/// +/// Split by expectation after the checks the two share. Those are the ones about +/// the *measurement* — whether the populations exist and whether they are really +/// two — and they hold whichever way the fault is supposed to point. +fn judge(report: &RelayReport) -> Vec { + let mut findings = pairing_findings(report); + + let cross = report + .cell(Placement::CrossPod, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent); + let co = report + .cell(Placement::CoLocated, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent); + + match report.expectation { + RelayExpectation::Inert => findings.extend(inert_findings(report, cross, co)), + RelayExpectation::RelayDegraded => { + findings.extend(degraded_findings(report, cross, co)); + } + } + + // Shared, and last. Both scenarios require the relay to come back: the + // control because nothing should have moved it, and the degradation + // scenario because a fault that is bounded in time has to be bounded in + // effect too. Only worth saying when the fault window itself was clean + // under `Inert`, since a population that dropped and stayed down is already + // reported there and repeating it would double-count one problem. + let already_reported = report.expectation == RelayExpectation::Inert + && cross.is_some_and(|s| s < report.cross_pod_floor_throughput_percent); + let after = report + .cell(Placement::CrossPod, Window::AfterFault) + .and_then(|c| c.share_of_baseline_percent); + if !already_reported + && after.is_some_and(|after| after < report.cross_pod_floor_throughput_percent) + { + findings.push(RelayFinding { + violation: RelayViolation::CrossPodDidNotReturn, + detail: format!( + "cross-pod was only at {}% of its baseline after the heal, below the {}% floor. \ + The fault was bounded in time, so its effect has to be bounded too", + after.unwrap_or_default(), + report.cross_pod_floor_throughput_percent + ), + }); + } + + findings +} + +/// The checks that are about the measurement rather than about the fault. +/// +/// Run first under both expectations, because everything after them compares +/// two populations and all of it passes for free if there are not two. +fn pairing_findings(report: &RelayReport) -> Vec { + let mut findings = Vec::new(); + + match report.cross_pod_percent { + None => findings.push(RelayFinding { + violation: RelayViolation::PairingTooThin, + detail: "no RPC caller was placed on either executor, so the run drove no pairs the \ + fault could reach" + .to_string(), + }), + Some(percent) if percent < report.cross_pod_floor_percent => { + findings.push(RelayFinding { + violation: RelayViolation::PairingTooThin, + detail: format!( + "only {percent}% of placed callers had their callee on the other executor, \ + below the {}% floor — the fault had almost nothing to reach, so a clean \ + result here would not have been earned", + report.cross_pod_floor_percent + ), + }); + } + Some(_) => {} + } + + match report.cross_pod_premium_ms { + Some(premium) if premium < report.cross_pod_premium_floor_ms as i64 => { + findings.push(RelayFinding { + violation: RelayViolation::CrossPodNotRelayed, + detail: format!( + "on the undisturbed baseline a cross-pod call cost {premium}ms more than a \ + co-located one at p50, under the {}ms floor. A call to an agent this \ + executor does not own goes out through worker-service and back, which is a \ + real network round trip, so the two populations costing the same says they \ + are not actually split — either the pairing is wrong or both halves were \ + served locally. Every comparison below is then between two samples of the \ + same thing", + report.cross_pod_premium_floor_ms + ), + }); + } + None => findings.push(RelayFinding { + violation: RelayViolation::CrossPodNotRelayed, + detail: "one of the two populations produced no baseline latency, so the run cannot \ + show that its cross-pod calls were paying for a network hop" + .to_string(), + }), + Some(_) => {} + } + + findings +} + +/// S2's verdict: the fault should have changed nothing, so any drop is a +/// finding. +fn inert_findings(report: &RelayReport, cross: Option, co: Option) -> Vec { + let mut findings = Vec::new(); + let cross_dropped = cross.is_some_and(|s| s < report.cross_pod_floor_throughput_percent); + let co_dropped = co.is_some_and(|s| s < report.co_located_floor_throughput_percent); + + // Order matters. Both populations dropping is not evidence about the link + // between pods, so it must not be reported as though it were. + if cross_dropped && co_dropped { + findings.push(RelayFinding { + violation: RelayViolation::BothDegraded, + detail: format!( + "cross-pod fell to {}% of its own baseline and co-located to {}%, below their \ + {}% and {}% floors. Calls that never leave a pod cannot be hurt by a partition \ + between pods, so this says the run was disturbed by something other than the \ + link under test", + cross.unwrap_or_default(), + co.unwrap_or_default(), + report.cross_pod_floor_throughput_percent, + report.co_located_floor_throughput_percent + ), + }); + } else if cross_dropped { + findings.push(RelayFinding { + violation: RelayViolation::CrossPodDegraded, + detail: format!( + "cross-pod fell to {}% of its own baseline against a {}% floor while co-located \ + held at {}%. The only difference between the two populations is whether the \ + call crosses the partitioned link, so this says the executors were reaching \ + each other directly rather than through worker-service", + cross.unwrap_or_default(), + report.cross_pod_floor_throughput_percent, + co.map(|s| s.to_string()) + .unwrap_or_else(|| "no measured".to_string()) + ), + }); + } + + findings +} + +/// S21's verdict: the fault was aimed at the relay, so the finding is a fault +/// that missed it. +/// +/// Neither population dropping is *not* a finding on its own here. The driver +/// sets the cadence, so throughput holds until the platform is too slow to keep +/// up at all, and a scenario that waited for that would only ever fire on a +/// worker-service that had already fallen over. +fn degraded_findings( + report: &RelayReport, + cross: Option, + co: Option, +) -> Vec { + let mut findings = Vec::new(); + + // What this scenario does *not* do any more: require the fault to have hurt + // by some minimum amount. + // + // It used to, twice over — a floor on the premium's widening and a floor on + // each population's own p50 — and both were wrong for the same reason. The + // question here is "stress worker-service, drive traffic through it, and see + // what happens", and a floor turns "the platform coped" into a failure of + // the test. Worse, the floors were guesses: the premium floor was set at + // 150% by predicting worker-service contributes about the premium's own + // width to a call, and three runs measured 120%. The numbers are reported + // below and an operator reads them; nothing here gates on their size. + // + // What is still worth failing on is a *bounded* fault leaving an unbounded + // effect, which is what these runs actually found. + let floor = report.recovery_floor_percent; + let elevated = |placement: Placement| { + let baseline = report.cell(placement, Window::BeforeFault)?.latency.p50_ms as f64; + let after = report.cell(placement, Window::AfterFault)?.latency.p50_ms as f64; + (baseline > 0.0).then(|| round2(100.0 * after / baseline)) + }; + if let (Some(cross_after), Some(co_after)) = ( + elevated(Placement::CrossPod), + elevated(Placement::CoLocated), + ) && (cross_after >= floor || co_after >= floor) + { + findings.push(RelayFinding { + violation: RelayViolation::RelayDidNotRecover, + detail: format!( + "after the fault healed, cross-pod p50 was {cross_after}% of its baseline and \ + co-located p50 {co_after}%, against a {floor}% floor. The fault was bounded in \ + time, so its cost should be too. Latency that is still elevated a full recovery \ + window later — and the per-minute breakdown of the run's history is where to \ + confirm it is still climbing rather than settling — is the fault outliving \ + itself" + ), + }); + } + + // Reported as a share of a share: co-located keeping less of its own + // baseline than cross-pod kept of its own. Comparing the two rates directly + // would compare two populations that never ran at the same rate to begin + // with. + if let (Some(cross), Some(co)) = (cross, co) + && cross - co > PLACEMENT_SHARE_MARGIN_PERCENT + { + findings.push(RelayFinding { + violation: RelayViolation::CoLocatedDegradedMore, + detail: format!( + "co-located held {co}% of its own baseline and cross-pod held {cross}%, so the \ + shorter path lost more, by more than the {PLACEMENT_SHARE_MARGIN_PERCENT} \ + points these two normally sit apart. A cross-pod call crosses worker-service \ + twice and a co-located one crosses it once, so a fault on worker-service cannot \ + hurt the shorter path more — something other than the fault disturbed this run. \ + Check the executor restarts and the ownership samples: a shard that moved leaves \ + the pairing describing agents that have since relocated" + ), + }); + } + + findings +} + +#[cfg(test)] +mod tests { + use test_r::test; + + use super::*; + use crate::chaos::history::Phase; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + secs, 0).unwrap() + } + + fn fault() -> Option { + Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }) + } + + /// How long a call takes in the fixtures, by population. + /// + /// Not decoration. The relay premium is what + /// [`RelayViolation::CrossPodNotRelayed`] is built on, so a fixture where + /// both populations cost the same is a fixture of a broken run — which is + /// exactly what one test below wants and what the others must avoid. + const CO_LOCATED_MS: u64 = 100; + const CROSS_POD_MS: u64 = 150; + + fn record(agent: &str, submitted: i64, completed: Option) -> OperationRecord { + let duration_ms = if agent.starts_with("cross") { + CROSS_POD_MS + } else { + CO_LOCATED_MS + }; + record_costing(agent, submitted, completed, duration_ms) + } + + /// The same record with the call's cost named rather than derived. + /// + /// The derived version fixes a population's latency for the whole run, which + /// is right for the control and useless for the scenario that measures the + /// premium *changing*. Kept as two functions so the existing fixtures keep + /// saying what they said. + fn record_costing( + agent: &str, + submitted: i64, + completed: Option, + duration_ms: u64, + ) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Rpc, + phase: Phase::Fault, + agent: agent.to_string(), + method: "increment_through_rpc".to_string(), + idempotency_key: format!("{agent}-{submitted}"), + submitted_at: at(submitted), + completed_at: completed.map(at), + attempts: 1, + outcome: if completed.is_some() { + Outcome::Confirmed + } else { + Outcome::Indeterminate + }, + duration_ms, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + fn pairing(cross: &[&str], co: &[&str]) -> RelayPairing { + RelayPairing { + pods: vec!["exec-a".to_string(), "exec-b".to_string()], + cross_pod: cross.iter().map(|s| s.to_string()).collect(), + co_located: co.iter().map(|s| s.to_string()).collect(), + unresolved: Vec::new(), + } + } + + /// The control's thresholds, matching S2's suite entry closely enough that + /// a test reads against the numbers a real run is judged by. + fn inert_config() -> RelayConfig { + RelayConfig { + cross_pod_floor_percent: 25.0, + cross_pod_floor_throughput_percent: 70.0, + co_located_floor_throughput_percent: 70.0, + cross_pod_premium_floor_ms: 5, + expectation: RelayExpectation::Inert, + recovery_floor_percent: 110.0, + } + } + + /// S21's, with floors the fixtures can straddle: the premium has to be at + /// least half again as wide during the fault, or both populations at least + /// a tenth slower inside the window than outside it. + fn degraded_config() -> RelayConfig { + RelayConfig { + expectation: RelayExpectation::RelayDegraded, + recovery_floor_percent: 110.0, + ..inert_config() + } + } + + /// A steady run where neither population moved is the expected outcome, and + /// it must produce no findings at all. + #[test] + fn a_partition_that_changed_nothing_reports_nothing() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + records.push(record("cross-0", second, Some(second))); + records.push(record("co-0", second, Some(second))); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert!( + !report.has_findings(), + "expected a clean control run, got {:?}", + report.findings + ); + assert_eq!(report.cross_pod_percent, Some(50.0)); + } + + /// The finding the scenario exists to make: the cross-pod half collapsed + /// and the co-located half did not. + #[test] + fn cross_pod_collapsing_alone_says_the_executors_talked_directly() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + records.push(record("co-0", second, Some(second))); + // Cross-pod is served before and after, but not during. + if !(100..200).contains(&second) { + records.push(record("cross-0", second, Some(second))); + } else { + records.push(record("cross-0", second, None)); + } + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + let violations: Vec<_> = report.findings.iter().map(|f| f.violation).collect(); + assert_eq!(violations, vec![RelayViolation::CrossPodDegraded]); + } + + /// Both halves dropping is not evidence about the link, and must not be + /// reported as though it were. + #[test] + fn both_populations_dropping_is_not_blamed_on_the_link() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + let served = !(100..200).contains(&second); + records.push(record("cross-0", second, served.then_some(second))); + records.push(record("co-0", second, served.then_some(second))); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + let violations: Vec<_> = report.findings.iter().map(|f| f.violation).collect(); + assert_eq!(violations, vec![RelayViolation::BothDegraded]); + assert!( + !violations.contains(&RelayViolation::CrossPodDegraded), + "a fault that also hurt calls which never left a pod must not be read as \ + executor-to-executor traffic" + ); + } + + /// A run whose pairs nearly all landed on one executor proved nothing, and + /// has to say so rather than reporting the clean numbers it produced. + #[test] + fn too_few_cross_pod_pairs_makes_the_run_inconclusive() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + records.push(record("cross-0", second, Some(second))); + for co in [ + "co-0", "co-1", "co-2", "co-3", "co-4", "co-5", "co-6", "co-7", "co-8", + ] { + records.push(record(co, second, Some(second))); + } + } + let report = build( + &records, + pairing( + &["cross-0"], + &[ + "co-0", "co-1", "co-2", "co-3", "co-4", "co-5", "co-6", "co-7", "co-8", + ], + ), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert_eq!(report.cross_pod_percent, Some(10.0)); + assert!( + report + .findings + .iter() + .any(|f| f.violation == RelayViolation::PairingTooThin), + "a 10% split is below the 25% floor and must be called out" + ); + } + + /// The guard the first cluster run showed was needed. + /// + /// Run 33789281692 came back perfectly clean on every throughput cell, and + /// throughput could not have said otherwise: the driver sets the cadence, + /// so both populations run at the rate they were asked to whether or not a + /// call leaves the pod. Only latency separated them, at 151ms against + /// 101ms. A run where that premium is absent is a run whose two populations + /// are one population, and every comparison it makes passes for free. + #[test] + fn two_populations_that_cost_the_same_did_not_cross_a_pod_boundary() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + // Both at the co-located cost: nothing here paid a relay hop. + records.push(record("co-0", second, Some(second))); + records.push(record("co-1", second, Some(second))); + } + // `co-1` is *labelled* cross-pod, and the latency says it is not. + let report = build( + &records, + pairing(&["co-1"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert_eq!(report.cross_pod_premium_ms, Some(0)); + assert!( + report + .findings + .iter() + .any(|f| f.violation == RelayViolation::CrossPodNotRelayed), + "a pairing whose halves cost the same has to be called out, got {:?}", + report.findings + ); + } + + /// The premium is the evidence the split is real, so it belongs in the + /// notes on every run rather than only when something is wrong. + #[test] + fn the_relay_premium_is_reported_on_a_clean_run() { + let mut records = Vec::new(); + for second in (0..300).step_by(2) { + records.push(record("cross-0", second, Some(second))); + records.push(record("co-0", second, Some(second))); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert!( + !report.has_findings(), + "expected clean, got {:?}", + report.findings + ); + assert_eq!( + report.cross_pod_premium_ms, + Some((CROSS_POD_MS - CO_LOCATED_MS) as i64) + ); + assert!( + report.note_lines().iter().any(|l| l.contains("relay hop")), + "the premium is the evidence the populations are split; it must reach the reader" + ); + } + + /// An unresolved caller says nothing either way, so it must not drag the + /// cross-pod share down and fail an otherwise well-split run. + #[test] + fn unresolved_callers_do_not_count_against_the_split() { + let pairing = RelayPairing { + pods: vec!["exec-a".to_string(), "exec-b".to_string()], + cross_pod: vec!["cross-0".to_string()], + co_located: vec!["co-0".to_string()], + unresolved: (0..50).map(|i| format!("lost-{i}")).collect(), + }; + assert_eq!(pairing.cross_pod_percent(), Some(50.0)); + } + + /// A record for a caller nobody placed is counted, not silently dropped: + /// it means the pairing and the workload disagree. + #[test] + fn records_for_unplaced_callers_are_counted() { + let records = vec![record("nobody", 10, Some(10))]; + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert_eq!(report.records_outside_the_pairing, 1); + } + + /// A run with no fault window cannot be read against the fault, and the + /// report has to say that rather than presenting cells as if it could. + #[test] + fn a_run_with_no_fault_window_says_so() { + let records = vec![record("cross-0", 10, Some(10))]; + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + None, + ScenarioCode::S2, + &inert_config(), + ); + assert!(report.partition_evidence.contains("unknown window")); + } + + /// A run that spans the fault with a named cost on each side. + /// + /// `during_*` applies to operations that complete inside the window, which + /// is how the cells file them, so a fixture whose fault-window calls cost + /// more is a fixture of a relay under load. + fn records_across_the_fault( + during_cross_ms: u64, + during_co_ms: u64, + during_step: usize, + ) -> Vec { + let mut records = Vec::new(); + for second in (0..100).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + for second in (100..200).step_by(during_step) { + records.push(record_costing( + "cross-0", + second, + Some(second), + during_cross_ms, + )); + records.push(record_costing("co-0", second, Some(second), during_co_ms)); + } + for second in (200..300).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + records + } + + /// S21's clean run. Both populations got slower and the cross-pod one got + /// slower *twice over*, which is what one starved worker-service hop looks + /// like from the outside. + #[test] + fn a_relay_fault_that_widened_the_premium_reports_nothing() { + // Baseline premium 50ms; during the fault 190 - 90 = 100ms, so 200%. + let records = records_across_the_fault(190, 90, 2); + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + assert!( + !report.has_findings(), + "expected a clean degradation run, got {:?}", + report.findings + ); + assert_eq!(report.cross_pod_premium_ms, Some(50)); + assert_eq!(report.cross_pod_premium_during_fault_ms, Some(100)); + assert_eq!(report.cross_pod_premium_inflation_percent, Some(200.0)); + } + + /// The premium's blind spot, and the reason it is not the only reading. + /// + /// Both populations pay a flat 100ms more, so the gap between them never + /// moves and the premium reads exactly 100%. That is not a fault which + /// missed worker-service; it is one that cost both traversals the same, and + /// S21 produced precisely this shape once its workload rose to eight times + /// the suite's rate. Judging on the premium alone called it inconclusive. + #[test] + fn a_fault_that_slowed_both_populations_equally_still_reached_the_relay() { + let records = records_across_the_fault(250, 200, 2); + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + assert_eq!(report.cross_pod_premium_inflation_percent, Some(100.0)); + // 250/150 and 200/100, and each is the smaller of its two ratios + // because the windows either side of the fault are identical. + assert_eq!(report.cross_pod_latency_inflation_percent, Some(166.67)); + assert_eq!(report.co_located_latency_inflation_percent, Some(200.0)); + assert!( + !report.has_findings(), + "both populations slowing is the fault landing, not a run with nothing to show, \ + got {:?}", + report.findings + ); + } + + /// A fault the platform shrugged off is a result, not a failure of the run. + /// + /// This used to be a finding, on the reasoning that a scenario aimed at + /// worker-service ought to prove it reached worker-service. It cost two + /// configured floors and it made "the platform coped" indistinguishable + /// from "the test is broken". The run reports what the fault cost and + /// leaves the reading to whoever asked for it. + #[test] + fn a_fault_the_platform_shrugged_off_is_not_a_finding() { + let records = records_across_the_fault(CROSS_POD_MS, CO_LOCATED_MS, 2); + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + assert_eq!(report.cross_pod_premium_inflation_percent, Some(100.0)); + assert!( + !report.has_findings(), + "a fault that cost nothing is a reading, not a finding, got {:?}", + report.findings + ); + } + + /// The one thing this scenario does fail on: a fault bounded in time whose + /// cost is not bounded with it. Both S21 runs at eight times the suite rate + /// ended a whole recovery window above their baselines and still climbing. + #[test] + fn a_fault_whose_cost_outlived_it_is_a_finding() { + let mut records = Vec::new(); + for second in (0..100).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + for second in (100..200).step_by(2) { + records.push(record_costing("cross-0", second, Some(second), 250)); + records.push(record_costing("co-0", second, Some(second), 200)); + } + // The heal changed nothing: 200 against a 150 baseline and 150 against + // 100 are both well past the floor. + for second in (200..300).step_by(2) { + records.push(record_costing("cross-0", second, Some(second), 200)); + records.push(record_costing("co-0", second, Some(second), 150)); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + let finding = report + .findings + .iter() + .find(|f| f.violation == RelayViolation::RelayDidNotRecover) + .expect("latency still elevated after the heal should be a finding"); + assert!(finding.detail.contains("133.33"), "{}", finding.detail); + assert!(finding.detail.contains("150"), "{}", finding.detail); + } + + /// The same numbers under the control's expectation raise nothing, because + /// a premium that held steady is exactly what S2 wants to see. + #[test] + fn the_control_does_not_ask_its_fault_to_widen_the_premium() { + let records = records_across_the_fault(150, 100, 2); + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S2, + &inert_config(), + ); + assert!( + !report.has_findings(), + "the control should not require its fault to change anything, got {:?}", + report.findings + ); + assert_eq!(report.cross_pod_premium_inflation_percent, Some(100.0)); + } + + /// Cross-pod collapsing is the control's headline finding and the + /// degradation scenario's expected result. It must not be reported as a + /// defect by the one that asked for it. + #[test] + fn a_degradation_run_does_not_report_its_own_fault_as_a_defect() { + // Cross-pod is served a tenth as often inside the window, and pays a + // widened premium while it happens. + let records = records_across_the_fault(190, 90, 20); + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + let violations: Vec<_> = report.findings.iter().map(|f| f.violation).collect(); + assert!( + !violations.contains(&RelayViolation::CrossPodDegraded) + && !violations.contains(&RelayViolation::BothDegraded), + "the scenario asked for this degradation, so it is context and not a finding: {:?}", + report.findings + ); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains("cross-pod ran at")), + "the degradation still has to be reported somewhere: {:?}", + report.note_lines() + ); + } + + /// The load hit the executors instead of the relay. A cross-pod call crosses + /// worker-service twice and a co-located one crosses it once, so the shorter + /// path cannot be the one that suffers more. + #[test] + fn the_shorter_path_losing_more_says_the_load_missed_the_relay() { + let mut records = Vec::new(); + for second in (0..100).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + // The premium still widens, so this is not caught by the vacuity guard. + // Co-located is served far less often than cross-pod all the same. + for second in (100..200).step_by(4) { + records.push(record_costing("cross-0", second, Some(second), 190)); + } + for second in (100..200).step_by(50) { + records.push(record_costing("co-0", second, Some(second), 90)); + } + for second in (200..300).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + assert!( + report + .findings + .iter() + .any(|f| f.violation == RelayViolation::CoLocatedDegradedMore), + "the shorter path losing more should be a finding, got {:?}", + report.findings + ); + } + + /// The bound that survives the expectation split: a fault that is over has + /// to stop costing anything, whichever way it was supposed to point. + #[test] + fn a_relay_that_never_came_back_is_a_finding_under_either_expectation() { + let mut records = Vec::new(); + for second in (0..100).step_by(2) { + records.push(record("cross-0", second, Some(second))); + records.push(record("co-0", second, Some(second))); + } + for second in (100..200).step_by(2) { + records.push(record_costing("cross-0", second, Some(second), 190)); + records.push(record_costing("co-0", second, Some(second), 90)); + } + // Cross-pod stays down long after the heal. + for second in (200..300).step_by(50) { + records.push(record("cross-0", second, Some(second))); + } + for second in (200..300).step_by(2) { + records.push(record("co-0", second, Some(second))); + } + for (code, config) in [ + (ScenarioCode::S2, inert_config()), + (ScenarioCode::S21, degraded_config()), + ] { + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + code, + &config, + ); + assert!( + report + .findings + .iter() + .any(|f| f.violation == RelayViolation::CrossPodDidNotReturn), + "{code} should require the relay to recover, got {:?}", + report.findings + ); + } + } + + /// The two populations sitting a fraction of a point apart is what an + /// undisturbed run looks like, and it must not be read as one of them being + /// hurt. + #[test] + fn ordinary_jitter_between_the_populations_is_not_an_inversion() { + let mut records = Vec::new(); + for second in (0..100).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + // Cross-pod is served on every even second of the window; co-located + // misses one, which is the sub-point difference S2's green runs showed. + for second in (100..200).step_by(2) { + records.push(record_costing("cross-0", second, Some(second), 190)); + if second != 150 { + records.push(record_costing("co-0", second, Some(second), 90)); + } + } + for second in (200..300).step_by(2) { + records.push(record_costing( + "cross-0", + second, + Some(second), + CROSS_POD_MS, + )); + records.push(record_costing("co-0", second, Some(second), CO_LOCATED_MS)); + } + let report = build( + &records, + pairing(&["cross-0"], &["co-0"]), + fault(), + ScenarioCode::S21, + °raded_config(), + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == RelayViolation::CoLocatedDegradedMore), + "a fraction of a point apart is not the shorter path being hurt: {:?}", + report.findings + ); + } +} diff --git a/integration-tests/src/chaos/resolution.rs b/integration-tests/src/chaos/resolution.rs new file mode 100644 index 0000000000..19735b0dbb --- /dev/null +++ b/integration-tests/src/chaos/resolution.rs @@ -0,0 +1,670 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The name-resolution account (GOL-373). +//! +//! S4 poisons `shard-manager.golem-release.svc.cluster.local` on one executor +//! and drives quota work on both. This module turns the operation history into +//! the comparison that answers the ticket. +//! +//! ### Why the expected answer is "nothing", and what that costs to report +//! +//! The executor holds one connection to the shard manager, built by +//! `GrpcClient::new`, whose idle TTL is `Duration::MAX` +//! (`golem-service-base/src/grpc/client.rs`). DNS is consulted when that +//! connection is established and never again while it lives. A name that stops +//! resolving therefore reaches nothing: no re-resolution is attempted, so the +//! SERVFAIL is never asked for. +//! +//! That makes S4 the second scenario in the suite — after S19 — where a clean +//! report and a report of nothing are the same document. The suite answers that +//! in two places and neither is here: the workflow proves the mechanism on a +//! throwaway pod before the run, and [`crate::chaos::split`] refuses a run whose +//! quota population landed entirely on one executor. What this module owes the +//! reader is the third part: it states the comparison it made on every run, +//! including the ones with no findings, so a result that says nothing happened +//! also says what it looked at. +//! +//! ### Why the comparison is across executors, not across time +//! +//! S19 compares the faulted pod's post-fault latency against its own baseline, +//! because a clock skew's cost arrives and leaves with the fault. A DNS failure +//! that reaches anything would not: the executor would be trying and failing to +//! rebuild a connection, so the cost would be concurrent with the fault and +//! nothing else in the run would move. +//! +//! So the headline is the target group against the **control group in the same +//! window**. Both halves are running the same workload against the same shard +//! manager at the same instant, so a difference between them is the fault and +//! not the hour. The against-its-own-baseline reading is kept as well, because +//! a fault that cost something and then failed to give it back is a different +//! result from one that cost nothing. +//! +//! ### Why the headline is p99 and the recovery reading is p50 +//! +//! Not a hedge: the two questions have different shapes, and MF2 run 1 proved +//! it the expensive way. +//! +//! What this fault does when it bites is **park a minority of reservations for +//! tens of seconds** behind a lease the executor cannot renew. Most operations +//! are untouched, so the median barely moves. That run measured the target +//! executor at p50 62ms against the control's 51ms — a 122% ratio that read as +//! "no effect" — while the same cells carried a p90 of **37 seconds** and a p99 +//! of **100 seconds** against the control's 63ms and 81ms. Comparing medians +//! for a tail phenomenon is measuring the wrong thing, and it reported +//! `fault-did-not-bite` for a fault that bit as hard as anything in the suite. +//! +//! "Did it come back", though, *is* a location question. The tail after the +//! heal is the parked work draining, which is the expected shape of a recovery +//! rather than a failure to recover; the median returning to baseline is what +//! says the steady state is back. So the recovery reading stays on p50, and the +//! post-fault p99 is reported as context rather than judged. +//! +//! [`crate::chaos::skew`] compares p50 for both, and that is not an +//! inconsistency: a clock skew shifts *every* renewal's timing, which is a +//! location shift. The statistic follows the mechanism. +//! +//! Both are recorded rather than failed. What fails an S4 run lives elsewhere: +//! the exactly-once oracle, and a shard assignment that moved. +//! +//! ### The same table, read two opposite ways +//! +//! MF2 (GOL-537) is S4 with the cached connection taken away: it restarts the +//! shard manager inside the DNS window, so the executor has to rebuild the +//! channel and the rebuild is the first moment resolution matters. Both +//! executors lose the connection; only one of them can resolve the name to get +//! it back. That is why the comparison is the same one S4 makes, and why the +//! verdict is inverted rather than duplicated. +//! +//! Under [`ResolutionExpectation::Survives`] a target group that fell behind +//! its control is the finding. Under [`ResolutionExpectation::Degrades`] it is +//! the expected result, and the finding is the opposite one — a composition +//! that changed nothing, which means the second fault failed to force a +//! re-resolution and the run measured S4 again under an MF code. + +use crate::chaos::history::{OperationRecord, Stream}; +use crate::chaos::split::{ + self, FaultWindow, Group, PodSplit, StreamCell, Window, recovery_percent, round2, +}; +use crate::chaos::summary::LatencyStats; +use serde::{Deserialize, Serialize}; + +/// What the run's fault is supposed to do to name resolution. +/// +/// The two scenarios built on this account measure the same things and disagree +/// only about which way the numbers should point, so the split lives here +/// rather than in two copies of the report. Same shape as +/// [`crate::chaos::relay::RelayExpectation`], for the same reason. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResolutionExpectation { + /// The fault cannot reach anything, and the run exists to show that. + /// + /// S4: one executor cannot resolve the shard manager, but its connection to + /// the shard manager is already up and never expires. The default, because + /// it is what this module was built for. + #[default] + Survives, + /// The fault is expected to bite, because something took the cached + /// connection away and the executor has to resolve the name to rebuild it. + /// + /// MF2: the shard manager is restarted inside the DNS window. + Degrades, +} + +impl ResolutionExpectation { + pub fn as_str(self) -> &'static str { + match self { + ResolutionExpectation::Survives => "survives", + ResolutionExpectation::Degrades => "degrades", + } + } +} + +impl std::fmt::Display for ResolutionExpectation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What the account can say went wrong, both recorded rather than failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResolutionViolation { + /// The target executor's quota stream was slower than the control + /// executor's while the name was poisoned. + QuotaDegraded, + /// It was still slower than its own baseline after the name came back. + QuotaDidNotRecover, + /// Expected to bite and did not. Only reachable under + /// [`ResolutionExpectation::Degrades`]: the second fault was supposed to + /// take the cached connection away and force a re-resolution, and the two + /// executors came out indistinguishable anyway — so the run measured S4 + /// under an MF code and says nothing new. + FaultDidNotBite, +} + +impl ResolutionViolation { + pub fn as_str(self) -> &'static str { + match self { + ResolutionViolation::QuotaDegraded => "quota-degraded", + ResolutionViolation::QuotaDidNotRecover => "quota-did-not-recover", + ResolutionViolation::FaultDidNotBite => "fault-did-not-bite", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResolutionFinding { + pub violation: ResolutionViolation, + pub detail: String, +} + +/// The name-resolution account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResolutionReport { + /// Which scenario produced this, so the lines a reader sees name the run + /// rather than the module. S4 and MF2 share every number below. + pub scenario: String, + /// Which way the numbers are supposed to point. See + /// [`ResolutionExpectation`]; it decides what counts as a finding, not just + /// how the report is worded. + pub expectation: ResolutionExpectation, + /// The name the fault was pointed at, mirrored from the suite so an + /// archived result says what was poisoned without needing the manifest. + pub poisoned_name: String, + /// How far above the control group the target group's during-fault p50 may + /// sit before [`ResolutionViolation::QuotaDegraded`] is recorded. + /// + /// Under [`ResolutionExpectation::Degrades`] the same number is read the + /// other way: staying *under* it is the finding. + pub degradation_ceiling_percent: f64, + /// How far above its own baseline the target group's post-fault p50 may sit + /// before [`ResolutionViolation::QuotaDidNotRecover`] is recorded. + pub recovery_floor_percent: f64, + pub cells: Vec, + /// Target group's during-fault **p99** as a percentage of the control + /// group's, in the same window. The headline, and the number the ceiling is + /// read against. Reported whether or not it breaches. + #[serde(skip_serializing_if = "Option::is_none")] + pub during_fault_percent: Option, + /// The same comparison at p50, reported as context and judged against + /// nothing. + /// + /// Carried because the gap between the two is the shape of the fault: a + /// median that held while the tail blew out is work parking, which is what + /// this fault does when it bites. + #[serde(skip_serializing_if = "Option::is_none")] + pub during_fault_median_percent: Option, + /// Target group's post-fault p50 as a percentage of its own baseline. + /// + /// p50 on purpose, unlike the headline. See the module doc: the tail after + /// the heal is parked work draining, and judging it would fail a recovery + /// for recovering. + #[serde(skip_serializing_if = "Option::is_none")] + pub quota_recovery_percent: Option, + /// Post-fault p99, reported as context so the drain is visible. + #[serde(skip_serializing_if = "Option::is_none")] + pub after_fault_tail_ms: Option, + pub findings: Vec, +} + +impl ResolutionReport { + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() + } + + pub fn cell(&self, group: Group, window: Window) -> Option<&StreamCell> { + self.cells + .iter() + .find(|c| c.group == group && c.window == window) + } + + /// What a reader has to act on. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| format!("{} {}: {}", self.scenario, f.violation.as_str(), f.detail)) + .collect() + } + + /// Context a reader needs to judge the numbers, findings or not. + /// + /// The comparison goes here on every run, including the clean ones. It is + /// the only line that says what the run looked at, and both scenarios built + /// on this account are ones where a clean report and a report of nothing + /// read identically. + pub fn note_lines(&self) -> Vec { + let scenario = &self.scenario; + let mut lines = Vec::new(); + + match self.during_fault_percent { + Some(percent) => { + let expected = match self.expectation { + ResolutionExpectation::Survives => { + "The executor's shard-manager channel never expires, so the expected \ + reading is about 100" + } + ResolutionExpectation::Degrades => { + "The shard manager was restarted inside this window, so the channel had \ + to be rebuilt and the expected reading is well above 100" + } + }; + lines.push(format!( + "{scenario}: with {} unresolvable on the target executor, its quota p50 ran \ + at {percent}% of the control executor's over the same window (ceiling {}%). \ + {expected}", + self.poisoned_name, self.degradation_ceiling_percent + )); + } + None => lines.push(format!( + "{scenario}: no during-fault quota comparison could be made — one of the two \ + executor groups confirmed nothing in that window, so the run says nothing about \ + whether {} mattered", + self.poisoned_name + )), + } + + if let Some(percent) = self.quota_recovery_percent { + lines.push(format!( + "{scenario}: the target executor's quota p50 settled at {percent}% of its own \ + baseline after the name came back (floor {}%)", + self.recovery_floor_percent + )); + } + + lines + } +} + +/// Everything the caller has to decide, kept out of the suite YAML's way. +#[derive(Debug, Clone)] +pub struct ResolutionInputs<'a> { + pub scenario: &'a str, + pub expectation: ResolutionExpectation, + pub split: &'a PodSplit, + pub fault: Option, + pub poisoned_name: String, + pub degradation_ceiling_percent: f64, + pub recovery_floor_percent: f64, +} + +/// Builds the account. +pub fn build(records: &[OperationRecord], inputs: ResolutionInputs<'_>) -> ResolutionReport { + let cells = split::stream_cells(records, Stream::Quota, inputs.split, inputs.fault); + + let mut report = ResolutionReport { + scenario: inputs.scenario.to_string(), + expectation: inputs.expectation, + poisoned_name: inputs.poisoned_name, + degradation_ceiling_percent: inputs.degradation_ceiling_percent, + recovery_floor_percent: inputs.recovery_floor_percent, + during_fault_percent: None, + during_fault_median_percent: None, + quota_recovery_percent: recovery_percent(&cells), + after_fault_tail_ms: None, + cells, + findings: Vec::new(), + }; + + report.during_fault_percent = during_fault_ratio(&report, |l| l.p99_ms); + report.during_fault_median_percent = during_fault_ratio(&report, |l| l.p50_ms); + report.after_fault_tail_ms = report + .cell(Group::OnPod, Window::AfterFault) + .map(|c| c.latency.p99_ms); + report.findings = findings(&report); + report +} + +/// Target group's during-fault latency against the control group's, same +/// window, at whichever percentile the caller asks for. +/// +/// `None` when either side confirmed nothing in the window, or when the control +/// side's reading was zero. A percentage of nothing is not a comparison, and one +/// reported anyway would be read as evidence. +fn during_fault_ratio( + report: &ResolutionReport, + stat: impl Fn(&LatencyStats) -> u64, +) -> Option { + let control = stat(&report.cell(Group::Elsewhere, Window::DuringFault)?.latency) as f64; + let target = stat(&report.cell(Group::OnPod, Window::DuringFault)?.latency) as f64; + (control > 0.0).then(|| round2(100.0 * target / control)) +} + +fn findings(report: &ResolutionReport) -> Vec { + let mut findings = Vec::new(); + + let target_p99 = |window| { + report + .cell(Group::OnPod, window) + .map(|c| c.latency.p99_ms) + .unwrap_or_default() + }; + let control_p99 = |window| { + report + .cell(Group::Elsewhere, window) + .map(|c| c.latency.p99_ms) + .unwrap_or_default() + }; + + if let Some(percent) = report.during_fault_percent { + match report.expectation { + // The fault should reach nothing. A gap between the two executors + // means something re-resolved the name, which is what MF2 sets out + // to force on purpose. + ResolutionExpectation::Survives if percent > report.degradation_ceiling_percent => { + findings.push(ResolutionFinding { + violation: ResolutionViolation::QuotaDegraded, + detail: format!( + "the executor that could not resolve {} ran its quota work at {percent}% \ + of the executor that could ({}ms against {}ms at p99), over a ceiling of \ + {}%. The shard-manager channel is built once with an infinite idle TTL, \ + so a cost here means something rebuilt it", + report.poisoned_name, + target_p99(Window::DuringFault), + control_p99(Window::DuringFault), + report.degradation_ceiling_percent, + ), + }); + } + // The fault should bite, because the second one took the cached + // connection away. Two executors that came out alike mean it did + // not, and every number below describes S4 rather than this run. + ResolutionExpectation::Degrades if percent <= report.degradation_ceiling_percent => { + findings.push(ResolutionFinding { + violation: ResolutionViolation::FaultDidNotBite, + detail: format!( + "the executor that could not resolve {} ran its quota work at {percent}% \ + of the executor that could ({}ms against {}ms at p99), under a ceiling \ + of {}%. Both lost their shard-manager connection to the restart and only \ + one could resolve the name to rebuild it, so they should not match. \ + Check that the restart landed inside the DNS window and that the \ + executor actually reconnected — a run reading like this measured S4", + report.poisoned_name, + target_p99(Window::DuringFault), + control_p99(Window::DuringFault), + report.degradation_ceiling_percent, + ), + }); + } + _ => {} + } + } + + // Read the same way under both expectations. Losing the lease is a + // legitimate response to either fault; never getting it back is not. + if let Some(percent) = report.quota_recovery_percent + && percent > report.recovery_floor_percent + { + findings.push(ResolutionFinding { + violation: ResolutionViolation::QuotaDidNotRecover, + detail: format!( + "the target executor's quota p50 is still at {percent}% of its own baseline after \ + {} resolved again, against a floor of {}%", + report.poisoned_name, report.recovery_floor_percent + ), + }); + } + + findings +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{Outcome, Phase}; + use chrono::{DateTime, Utc}; + use std::collections::BTreeMap; + use test_r::test; + + const CEILING: f64 = 300.0; + const FLOOR: f64 = 150.0; + const NAME: &str = "shard-manager.golem-release.svc.cluster.local"; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + } + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["target-a".to_string()], + elsewhere: vec!["control-a".to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn op(agent: &str, submitted_secs: i64, duration_ms: u64) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Quota, + phase: Phase::Fault, + agent: agent.to_string(), + method: "reserve_and_increment".to_string(), + idempotency_key: format!("{agent}-{submitted_secs}"), + submitted_at: at(submitted_secs), + completed_at: Some(at(submitted_secs)), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + fn build_as( + records: &[OperationRecord], + expectation: ResolutionExpectation, + ) -> ResolutionReport { + build( + records, + ResolutionInputs { + scenario: match expectation { + ResolutionExpectation::Survives => "S4", + ResolutionExpectation::Degrades => "MF2", + }, + expectation, + split: &split(), + fault: Some(fault()), + poisoned_name: NAME.to_string(), + degradation_ceiling_percent: CEILING, + recovery_floor_percent: FLOOR, + }, + ) + } + + fn build_with(records: &[OperationRecord]) -> ResolutionReport { + build_as(records, ResolutionExpectation::Survives) + } + + /// The pair of agents both scenarios drive: a matched population on each + /// executor, inside the fault window. + fn matched(target_ms: u64, control_ms: u64) -> Vec { + (0..10) + .flat_map(|i| { + [ + op("target-a", 120 + i, target_ms), + op("control-a", 120 + i, control_ms), + ] + }) + .collect() + } + + /// MF2's expected result: the executor that could not re-resolve fell + /// behind the one that could, so there is nothing to report. + #[test] + fn under_degrades_a_slower_target_is_the_expected_result() { + let report = build_as(&matched(200, 20), ResolutionExpectation::Degrades); + assert_eq!(report.during_fault_percent, Some(1000.0)); + assert!(!report.has_violations(), "{:?}", report.findings); + } + + /// The shape MF2 run 2 actually produced, and the reason the headline is a + /// p99. + /// + /// A minority of reservations park for tens of seconds behind a lease the + /// executor cannot renew; the rest are untouched. The median barely moves + /// and the tail blows out. An account comparing medians called that + /// `fault-did-not-bite` for a fault that bit as hard as anything in the + /// suite. + #[test] + fn a_held_median_with_a_blown_tail_is_the_fault_biting() { + // Ninety fast operations and ten parked ones, on the target only. + let mut records: Vec = (0..90) + .flat_map(|i| [op("target-a", 120 + i, 60), op("control-a", 120 + i, 60)]) + .collect(); + // Inside the window: the fault runs at(100)..at(200), and latency is + // filed by completion, so a stamp outside it lands in another cell. + records.extend((0..10).map(|i| op("target-a", 150 + i, 40_000))); + + let report = build_as(&records, ResolutionExpectation::Degrades); + assert_eq!( + report.during_fault_median_percent, + Some(100.0), + "the median is supposed to hold — that is the trap" + ); + assert!( + report.during_fault_percent.unwrap() > 300.0, + "the tail is where the fault shows: {:?}", + report.during_fault_percent + ); + assert!( + !report.has_violations(), + "a blown tail under `degrades` is the expected result, not a finding: {:?}", + report.findings + ); + } + + /// MF2's actual failure mode, and the one it exists to catch: the second + /// fault did not force a re-resolution, so the run measured S4. + #[test] + fn under_degrades_two_alike_executors_mean_the_fault_never_bit() { + let report = build_as(&matched(20, 20), ResolutionExpectation::Degrades); + assert_eq!(report.during_fault_percent, Some(100.0)); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(ResolutionViolation::FaultDidNotBite) + ); + } + + /// The same numbers, read the opposite way by the other scenario. This is + /// the whole reason `expectation` exists rather than two copies of the + /// account. + #[test] + fn the_same_cells_produce_opposite_verdicts_under_the_two_expectations() { + let alike = matched(20, 20); + let slower = matched(200, 20); + assert!(!build_as(&alike, ResolutionExpectation::Survives).has_violations()); + assert!(build_as(&alike, ResolutionExpectation::Degrades).has_violations()); + assert!(build_as(&slower, ResolutionExpectation::Survives).has_violations()); + assert!(!build_as(&slower, ResolutionExpectation::Degrades).has_violations()); + } + + /// The expected result, and the one that has to stay quiet. + #[test] + fn two_executors_that_perform_alike_under_the_fault_produce_no_finding() { + let records: Vec = (0..10) + .flat_map(|i| { + [ + op("target-a", 120 + i, 20), + op("control-a", 120 + i, 20), + op("target-a", 10 + i, 20), + op("target-a", 220 + i, 20), + ] + }) + .collect(); + let report = build_with(&records); + assert_eq!(report.during_fault_percent, Some(100.0)); + assert!(!report.has_violations(), "{:?}", report.findings); + } + + /// The result that would make S4 interesting, and the one MF2 is built to + /// produce on purpose. + #[test] + fn a_target_executor_slower_than_the_control_is_a_finding() { + let records: Vec = (0..10) + .flat_map(|i| [op("target-a", 120 + i, 200), op("control-a", 120 + i, 20)]) + .collect(); + let report = build_with(&records); + assert_eq!(report.during_fault_percent, Some(1000.0)); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(ResolutionViolation::QuotaDegraded) + ); + } + + /// A comparison that cannot be made must not be reported as one that was. + #[test] + fn a_control_group_that_did_nothing_yields_no_comparison_and_no_finding() { + let records: Vec = (0..10).map(|i| op("target-a", 120 + i, 200)).collect(); + let report = build_with(&records); + assert_eq!(report.during_fault_percent, None); + assert!(!report.has_violations(), "{:?}", report.findings); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains("no during-fault quota comparison")), + "the run must say it could not compare, rather than say nothing" + ); + } + + /// Latency that stays high after the name comes back is a second finding, + /// separate from the during-fault one. + #[test] + fn latency_that_never_returns_to_baseline_is_recorded_on_its_own() { + let mut records: Vec = + (0..10).map(|i| op("target-a", 10 + i, 20)).collect(); + records.extend((0..10).map(|i| op("target-a", 220 + i, 400))); + let report = build_with(&records); + assert_eq!(report.quota_recovery_percent, Some(2000.0)); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(ResolutionViolation::QuotaDidNotRecover) + ); + } + + /// The comparison is stated on a clean run too, because a clean report and + /// a report of nothing are otherwise the same document. + #[test] + fn a_clean_run_still_says_what_it_compared() { + let records: Vec = (0..4) + .flat_map(|i| [op("target-a", 120 + i, 20), op("control-a", 120 + i, 20)]) + .collect(); + let report = build_with(&records); + assert!(report.attention_lines().is_empty()); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains(NAME) && line.contains("100%")), + "notes were {:?}", + report.note_lines() + ); + } +} diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 0937b0d0f2..d71d30ca16 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -25,8 +25,13 @@ //! same scenario ran anywhere else. use crate::chaos::pinned::PinnedSelection; +use crate::chaos::scheduled::ScheduledSelection; +use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; -use crate::chaos::{FaultConfig, PinnedConfig, RetryPolicy, WorkloadConfig}; +use crate::chaos::{ + ComposedConfig, DeleteConfig, FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, + RetryPolicy, RevertConfig, RollbackConfig, ScheduledConfig, StorageConfig, WorkloadConfig, +}; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; use serde::{Deserialize, Serialize}; @@ -34,7 +39,14 @@ use std::path::Path; /// Bumped when the on-disk shape changes incompatibly. Archived results outlive /// the tooling that reads them, so the shape has to say which shape it is. -pub const RESULT_SCHEMA_VERSION: u32 = 2; +/// +/// 3: the storage scenarios stopped sharing one verdict. `storage` carries an +/// `expect` block where it used to carry `outageQuietFloorPercent`, and the +/// storage-fault account carries the same block plus +/// `leastServingStreamPercent`. A version 2 result does not deserialise into +/// the version 3 types, which is what the bump is for; the report generator +/// reads both, because the runs already in the bucket are worth rendering. +pub const RESULT_SCHEMA_VERSION: u32 = 3; /// A phase's wall-clock extent. These are the numbers the workflow pins Grafana /// time ranges to, so they are recorded in UTC with no ambiguity. @@ -130,6 +142,60 @@ pub struct ChaosResult { /// to own. Present only for scenarios that pin the target. #[serde(default, skip_serializing_if = "Option::is_none")] pub pinned_selection: Option, + /// The scheduled-registration workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled: Option, + /// How the schedule targets divided around the executor the fault was aimed + /// at. Present only for S10, and load-bearing for reading its percentiles: + /// without it there is no way to tell the affected population from the + /// control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled_selection: Option, + /// The suspended-waiter workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise: Option, + /// How the promise waiters divided around the executor the fault was aimed + /// at. Present only for S11, and load-bearing for the same reason as + /// `scheduledSelection`: without it there is no way to tell the affected + /// population from the control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_selection: Option, + /// The reachability workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation: Option, + /// How the agents divided around the executor the fault was aimed at. + /// Present for S3, which cuts that executor off from worker-service, and + /// for S19, which moves its clock. Load-bearing for the same reason as + /// `promiseSelection`, and for one more: both verdicts are comparisons + /// between the two groups, so a report without this cannot be re-checked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation_selection: Option, + /// The revert workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert: Option, + /// How the revert agents divided around the executor the kill was aimed at. + /// Present only for S7. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert_selection: Option, + /// The deletion workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete: Option, + /// How the agent slots divided around the executor the kill was aimed at. + /// Present only for S6. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_selection: Option, + /// The component rollback the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, + /// The storage the run was configured to take away, and the thresholds its + /// account was judged by. Present only for S16. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + /// The second fault the run was configured to compose with the first, and + /// where in its window it was asked to land. Present only for the `MF` + /// codes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub composed: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -185,6 +251,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -193,10 +260,24 @@ mod tests { scheduled_agents: 20, promise_agents: 20, quota_agents: 20, + rpc_agents: 0, rate_per_sec: 10, }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, + composed: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -209,6 +290,542 @@ mod tests { } } + /// The S11 shape. Same contract as the S10 test below, for the same reason: + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud reads these by + /// name, and the two repositories cannot be changed atomically. + #[test] + fn an_s11_result_carries_the_promise_wakeup_fields_the_investigation_report_reads() { + use crate::chaos::history::{WaiterWakeupLog, WakeupRecord}; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::wakeups::WakeupReport; + + let now = Utc::now(); + let waiter = "chaos-s11-promise-waiter-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![waiter.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S11".to_string(); + result.promise = Some(crate::chaos::PromiseConfig { + waiters: 200, + dwell_millis: 5000, + wakeup_budget_secs: 60, + }); + result.promise_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_promise_wakeups(WakeupReport::build( + &[], + &[WaiterWakeupLog { + agent: waiter.clone(), + wakes: Some(1), + wakeups: vec![WakeupRecord { + token: format!("{waiter}-00000001"), + armed_at: now, + woken_at: now + chrono::Duration::seconds(6), + }], + error: None, + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(60), + 0, + )); + + let json = serde_json::to_value(&result).unwrap(); + let wakeups = &json["summary"]["promiseWakeups"]; + for key in [ + "wakeupBudgetMs", + "dwellMs", + "completionsConfirmed", + "completionsIndeterminate", + "completionsRejected", + "wakeupsRecorded", + "wokeOnce", + "indeterminateThatWoke", + "inconclusive", + "unverifiable", + "unknownTokens", + "waitersUnreadable", + "waitersTruncated", + "waitersStoodDown", + "waitersWedged", + "delay", + "findings", + "findingsOmitted", + ] { + assert!( + !wakeups[key].is_null(), + "summary.promiseWakeups.{key} is what the investigation report reads" + ); + } + assert_eq!(json["promise"]["wakeupBudgetSecs"], 60); + assert_eq!(json["promiseSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S11 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S11"); + assert!(parsed.summary.promise_wakeups.is_some()); + assert!(parsed.promise_selection.is_some()); + } + + /// The S9 shape. Same contract as the others: the investigation report in + /// golem-cloud reads these fields by name. + #[test] + fn an_s9_result_carries_the_rollback_fields_the_investigation_report_reads() { + use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; + + let now = Utc::now(); + let mut forward = std::collections::BTreeMap::new(); + forward.insert("chaos-s9-durable-0000".to_string(), Some(2u32)); + let mut back = std::collections::BTreeMap::new(); + back.insert("chaos-s9-durable-0000".to_string(), Some(1u32)); + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S9".to_string(); + result.rollback = Some(crate::chaos::RollbackConfig { + settle_secs: 90, + rolled_forward_floor_percent: 90.0, + control_retries: 2, + control_retry_delay_secs: 5, + kill_delay_secs: 2, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)).with_rollback( + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: VersionCensus::build("before-rollback", 2, &forward), + rolled_back: Some(VersionCensus::build("after-recovery", 1, &back)), + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 198, + accepted_after_retry: 2, + refused: 0, + max_retries: 2, + }, + rolled_forward_floor_percent: 90.0, + }, + ); + + let json = serde_json::to_value(&result).unwrap(); + let rollback = &json["summary"]["rollback"]; + for key in [ + "forwardRevision", + "rollbackRevision", + "forwardVersion", + "rollbackVersion", + "rolledForward", + "rolledBack", + "control", + "rolledForwardFloorPercent", + ] { + assert!( + !rollback[key].is_null(), + "summary.rollback.{key} is what the investigation report reads" + ); + } + for key in [ + "requested", + "acceptedFirstTry", + "acceptedAfterRetry", + "refused", + ] { + assert!(!rollback["control"][key].is_null(), "control.{key}"); + } + assert_eq!(json["rollback"]["rolledForwardFloorPercent"], 90.0); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S9"); + assert!(parsed.summary.rollback.is_some()); + } + + /// The S6 shape. Same contract as the S3, S7, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s6_result_carries_the_resurrection_fields_the_investigation_report_reads() { + use crate::chaos::deletions::DeleteRound; + use crate::chaos::resurrection::ResurrectionReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let agent = "chaos-s6-delete-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S6".to_string(); + result.delete = Some(crate::chaos::DeleteConfig { + agents: 200, + increments_per_round: 3, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.delete_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_resurrection(ResurrectionReport::build( + &[DeleteRound { + agent, + round: 0, + before_delete: Some(3), + outcome: crate::chaos::history::Outcome::Confirmed, + rejected_as_not_found: false, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(0), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 3, + )); + + let json = serde_json::to_value(&result).unwrap(); + let resurrection = &json["summary"]["resurrection"]; + for key in [ + "incrementsPerRound", + "roundsRecorded", + "deletesConfirmed", + "deletesIndeterminate", + "deletesRejected", + "deletedExactly", + "indeterminateThatDeleted", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !resurrection[key].is_null(), + "summary.resurrection.{key} is what the investigation report reads" + ); + } + assert_eq!(json["delete"]["incrementsPerRound"], 3); + assert_eq!(json["deleteSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S6"); + assert!(parsed.summary.resurrection.is_some()); + assert!(parsed.delete_selection.is_some()); + } + + /// The S7 shape. Same contract as the S3, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s7_result_carries_the_truncation_fields_the_investigation_report_reads() { + use crate::chaos::reverts::RevertRound; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::truncation::TruncationReport; + + let now = Utc::now(); + let agent = "chaos-s7-revert-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S7".to_string(); + result.revert = Some(crate::chaos::RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.revert_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_truncation(TruncationReport::build( + &[RevertRound { + agent, + round: 0, + before_revert: Some(10), + asked_to_revert: 2, + outcome: crate::chaos::history::Outcome::Confirmed, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(8), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 4, + 2, + )); + + let json = serde_json::to_value(&result).unwrap(); + let truncation = &json["summary"]["truncation"]; + for key in [ + "incrementsPerRound", + "revertInvocations", + "roundsRecorded", + "revertsConfirmed", + "revertsIndeterminate", + "revertsRejected", + "appliedExactly", + "indeterminateThatApplied", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !truncation[key].is_null(), + "summary.truncation.{key} is what the investigation report reads" + ); + } + assert_eq!(json["revert"]["revertInvocations"], 2); + assert_eq!(json["revertSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S7"); + assert!(parsed.summary.truncation.is_some()); + assert!(parsed.revert_selection.is_some()); + } + + /// The S3 shape. Same contract as the S10 and S11 tests, for the same + /// reason: `ci-scripts/chaos-investigation-report.py` in golem-cloud reads + /// these by name, and the two repositories cannot be changed atomically. + #[test] + fn an_s3_result_carries_the_reachability_fields_the_investigation_report_reads() { + use crate::chaos::reachability::ReachabilityReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["chaos-s3-durable-0000".to_string()], + elsewhere: vec!["chaos-s3-durable-0001".to_string()], + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S3".to_string(); + result.isolation = Some(crate::chaos::IsolationConfig { + agents: 200, + interval_millis: 1000, + isolated_ceiling_percent: 25.0, + control_floor_percent: 75.0, + recovery_budget_secs: 60, + }); + result.isolation_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_reachability(ReachabilityReport::build( + &[], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: Some(now + chrono::Duration::seconds(180)), + }), + 25.0, + 75.0, + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let reachability = &json["summary"]["reachability"]; + for key in [ + "isolatedPod", + "isolatedAgents", + "reachableAgents", + "isolatedCeilingPercent", + "controlFloorPercent", + "recoveryBudgetMs", + "cells", + "recovery", + "recoveryOverBudget", + "agentsNeverRecovered", + "recordsOutsideTheSplit", + "findings", + "findingsOmitted", + ] { + assert!( + !reachability[key].is_null(), + "summary.reachability.{key} is what the investigation report reads" + ); + } + assert_eq!(json["isolation"]["controlFloorPercent"], 75.0); + assert_eq!(json["isolationSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S3 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S3"); + assert!(parsed.summary.reachability.is_some()); + assert!(parsed.isolation_selection.is_some()); + } + + /// The S10 shape, whose report is read by a script in another repository. + /// + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud renders these + /// fields by name. The two repositories cannot be changed atomically, so a + /// rename here would silently empty a section of the investigation report + /// rather than fail anything. Naming the keys in a test is what makes that + /// break loudly and locally. + #[test] + fn an_s10_result_carries_the_schedule_fire_fields_the_investigation_report_reads() { + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; + + let now = Utc::now(); + let due = now + chrono::Duration::seconds(10); + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S10".to_string(); + result.scheduled = Some(crate::chaos::ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_schedule_fires(ScheduleFireReport::build( + &[], + &[TargetFireLog { + agent: "chaos-s10-scheduled-target-0000".to_string(), + polls: Some(1), + fires: vec![FireRecord { + token: "chaos-s10-scheduled-target-0000-00000001".to_string(), + scheduled_at: due, + observed_at: due + chrono::Duration::seconds(4), + }], + error: None, + }], + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + &std::collections::BTreeSet::from(["chaos-s10-scheduled-target-0000".to_string()]), + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let fires = &json["summary"]["scheduleFires"]; + for key in [ + "leaseBudgetMs", + "registrationsConfirmed", + "registrationsIndeterminate", + "firesRecorded", + "firedOnce", + "indeterminateThatFired", + "inconclusive", + "unverifiable", + "unknownTokens", + "targetsUnreadable", + "targetsTruncated", + "delay", + "findings", + "findingsOmitted", + "overdueOnArrival", + "overdueDelay", + ] { + assert!( + !fires[key].is_null(), + "summary.scheduleFires.{key} is what the investigation report reads" + ); + } + let cell = &fires["delay"][0]; + assert_eq!(cell["group"], "on-killed-executor"); + assert_eq!(cell["window"], "during-fault"); + assert_eq!(cell["delay"]["p99Ms"], 4000); + assert_eq!(cell["overBudget"], 0); + assert_eq!(cell["minDelayMs"], 4000); + assert_eq!(json["scheduled"]["leaseBudgetSecs"], 60); + + // And it still round-trips, so an archived S10 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S10"); + assert!(parsed.summary.schedule_fires.is_some()); + assert!(!Stream::Scheduled.to_string().is_empty()); + } + + /// The CI annotation branches on `summary.attention` being non-empty, so + /// the two lists have to stay two lists across the repo boundary. Folding + /// context back into `attention` would make the annotation fire on every + /// healthy run, which is how it came to mean nothing the first time. + #[test] + fn a_result_separates_findings_from_context_for_the_ci_annotation() { + use crate::chaos::summary::Note; + + let mut result = sample_result(TerminationReason::Completed); + result.summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("4 scheduled targets filled their fire log and dropped entries"), + ]); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["summary"]["attention"].as_array().unwrap().len(), + 1, + "summary.attention is what --attention-count counts" + ); + assert_eq!( + json["summary"]["notes"].as_array().unwrap().len(), + 1, + "summary.notes is what the report renders as run context" + ); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.summary.attention.len(), 1); + assert_eq!(parsed.summary.notes.len(), 1); + } + + /// A tag can be moved; a digest identifies the build a run actually tested. + /// The workflow emits both, and the runbook tells a reader to match them + /// against the deployment manifests, so both key names are load-bearing. + #[test] + fn run_metadata_carries_the_image_digest_beside_the_tag() { + use golem_test_framework::benchmark::RunMetadata; + + let mut result = sample_result(TerminationReason::Completed); + result.run_metadata = Some(RunMetadata { + worker_executor_image_tag: Some("v1.5.10-dev.2".to_string()), + worker_executor_image_digest: Some("sha256:60eac87a".to_string()), + ..Default::default() + }); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["runMetadata"]["workerExecutorImageTag"], + "v1.5.10-dev.2" + ); + assert_eq!( + json["runMetadata"]["workerExecutorImageDigest"], + "sha256:60eac87a" + ); + } + #[test] fn result_round_trips_through_json() { let result = sample_result(TerminationReason::Completed); @@ -388,6 +1005,7 @@ mod sample_artifact { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -396,10 +1014,24 @@ mod sample_artifact { scheduled_agents: 20, promise_agents: 20, quota_agents: 20, + rpc_agents: 0, rate_per_sec: 10, }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, + composed: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -418,4 +1050,224 @@ mod sample_artifact { result.save(&path).unwrap(); println!("wrote sample result to {path}"); } + + /// The same, for S10, whose report has a section of its own. + /// + /// A separate artifact rather than a field added to the one above: the two + /// scenarios do not share a workload shape, and a sample that carried both + /// a mixed workload and a scheduled one would not resemble anything the + /// driver ever writes. + #[test] + fn write_sample_s10_result_artifact() { + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, TargetFireLog}; + use crate::chaos::scheduled::ScheduledSelection; + + let Ok(path) = std::env::var("CHAOS_SAMPLE_RESULT_S10") else { + return; + }; + let at = |s: i64| Utc.timestamp_opt(1_800_000_000 + s, 0).unwrap(); + let killed = "chaos-s10-scheduled-target-0000"; + let survivor = "chaos-s10-scheduled-target-0001"; + + // Four registrations per target. On the killed executor one of them + // never fires, which is the finding the section exists to render. + let mut records = Vec::new(); + let mut killed_fires = Vec::new(); + let mut survivor_fires = Vec::new(); + for i in 0..8u64 { + let target = if i % 2 == 0 { killed } else { survivor }; + // Spread so some actions fall due before the kill and some during it, + // which is what gives the report both a control row and the row it is + // actually about. + let submitted = at(285 + i as i64 * 4); + let token = format!("{target}-{:08}", i / 2); + records.push(OperationRecord { + op_id: i, + stream: Stream::Scheduled, + phase: if submitted < at(300) { + Phase::Baseline + } else { + Phase::Fault + }, + agent: target.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.clone(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 8 + i, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + }); + + let due = submitted + chrono::Duration::seconds(10); + // One registration stalls on the client's attempt timeout and only + // lands long after its action was due — the shape the first real run + // produced 26 times over. + if i == 1 { + let stalled = records.last_mut().unwrap(); + stalled.completed_at = Some(submitted + chrono::Duration::seconds(125)); + stalled.duration_ms = 125_000; + stalled.attempts = 2; + } + if target == killed { + // The last one is the action the kill swallowed. + if i < 6 { + killed_fires.push(FireRecord { + token, + scheduled_at: due, + // Late by a shard reassignment. + observed_at: due + chrono::Duration::milliseconds(41_500), + }); + } + } else { + // The stalled registration's action fires the moment it lands, + // which the raw arithmetic calls 115s late. + let late = if i == 1 { 115_200 } else { 120 }; + survivor_fires.push(FireRecord { + token, + scheduled_at: due, + observed_at: due + chrono::Duration::milliseconds(late), + }); + } + } + + let logs = vec![ + TargetFireLog { + agent: killed.to_string(), + polls: Some(killed_fires.len() as u64), + fires: killed_fires, + error: None, + }, + TargetFireLog { + agent: survivor.to_string(), + polls: Some(survivor_fires.len() as u64), + fires: survivor_fires, + error: None, + }, + ]; + + let on_pod = std::collections::BTreeSet::from([killed.to_string()]); + let report = ScheduleFireReport::build( + &records, + &logs, + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: at(300), + recovered_at: Some(at(420)), + }), + &on_pod, + std::time::Duration::from_secs(60), + ); + + let readback: Vec = logs + .iter() + .map(|log| { + let scoped: Vec<&OperationRecord> = + records.iter().filter(|r| r.agent == log.agent).collect(); + AgentReadback::evaluate( + Stream::Scheduled, + &log.agent, + &scoped, + Ok(log.polls.unwrap_or(0)), + ) + }) + .collect(); + + let result = ChaosResult { + schema_version: RESULT_SCHEMA_VERSION, + scenario_code: "S10".to_string(), + scenario_name: "executor-crash-during-scheduled-fire".to_string(), + completed: false, + termination_reason: TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }, + started_at: at(0), + ended_at: Some(at(900)), + phases: Phases { + baseline: Some({ + let mut w = PhaseWindow::started(at(0)); + w.end(at(300)); + w + }), + fault: Some({ + let mut w = PhaseWindow::started(at(300)); + w.end(at(420)); + w + }), + recovery: Some({ + let mut w = PhaseWindow::started(at(420)); + w.end(at(720)); + w + }), + }, + fault_injected_at: Some(at(300)), + fault_recovered_at: Some(at(420)), + fault_id: Some("chaos-s10-12345".to_string()), + fault_target_observed: Some("worker-executor-abc123".to_string()), + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + manifest: None, + duration_secs: 60, + }, + workload: None, + pinned: None, + pinned_selection: None, + scheduled: Some(crate::chaos::ScheduledConfig { + targets: 2, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }), + scheduled_selection: Some(ScheduledSelection { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![killed.to_string()], + elsewhere: vec![survivor.to_string()], + targets_per_pod: [ + ("10.0.1.1:9000".to_string(), 1), + ("10.0.1.2:9000".to_string(), 1), + ] + .into_iter() + .collect(), + number_of_shards: 1024, + }), + promise: None, + promise_selection: None, + isolation: None, + isolation_selection: None, + revert: None, + revert_selection: None, + delete: None, + delete_selection: None, + rollback: None, + storage: None, + composed: None, + retry_policy: RetryPolicy::default(), + scope: RunScope { + environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), + component_ids: vec!["0192f000-0000-7000-8000-000000000002".to_string()], + agent_id_prefix: "chaos-s10".to_string(), + idempotency_key_prefix: "chaos-s10-".to_string(), + }, + summary: ChaosSummary::build(&records, readback, Vec::new(), Some(at(300))) + .with_schedule_fires(report), + run_metadata: None, + }; + result.save(&path).unwrap(); + println!("wrote sample S10 result to {path}"); + } } diff --git a/integration-tests/src/chaos/resurrection.rs b/integration-tests/src/chaos/resurrection.rs new file mode 100644 index 0000000000..f5209448fc --- /dev/null +++ b/integration-tests/src/chaos/resurrection.rs @@ -0,0 +1,828 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every deleted agent stay deleted (GOL-372)? +//! +//! The same two-value oracle [`crate::chaos::truncation`] uses, one step +//! further. There a round asked the platform to forget some of an agent's work; +//! here it asks it to forget the agent. Invoking a deleted id creates a **new** +//! agent, so the counter a deletion leaves behind has exactly two legal values: +//! +//! * `0` — the deletion took, and the id is a fresh agent counting from nothing +//! * `V` — the deletion did not take, and the old agent is still there +//! +//! Anything else means an agent came back carrying part of a state it was +//! supposed to have lost. +//! +//! ### Which of the two is the finding +//! +//! Neither, on its own. Both are legitimate outcomes of a delete the driver +//! never heard back about — a lost response leaves the question genuinely open. +//! What makes one a defect is the platform's own answer next to it: +//! +//! * confirmed, and the agent is still worth `V` → **resurrection**. The +//! platform said the agent was gone and it is not. +//! * refused, and the agent is gone → the opposite, and just as wrong. +//! +//! ### Why this is worth a scenario at all +//! +//! Because the happy path is already defended and the crash path is not +//! obviously so. `Worker::start_deleting` in the executor exists specifically to +//! stop a background status flush from "resurrecting the cached status" after +//! the durable removal — its own comment. Deleting is four steps (interrupt, +//! mark, remove from the worker service, remove from the active set) and only +//! the third is durable, so a pod that dies between the mark and the removal +//! leaves an agent marked for deletion that was never removed. Whoever picks up +//! its shard next decides what that means. + +use crate::chaos::deletions::{COUNTER_OF_A_NEW_AGENT, DeleteRound}; +use crate::chaos::history::Outcome; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one deletion. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResurrectionViolation { + /// The platform confirmed the deletion and the agent came back carrying the + /// value it had before. The failure this scenario is named for. + ResurrectedWithState, + /// The agent came back worth neither nothing nor what it had been: part of + /// a state it was supposed to have lost survived. + PartialState, + /// The platform refused the deletion and the agent is gone anyway, for a + /// reason other than not finding it. + /// + /// A refusal that says the agent was not there is **not** this: see + /// [`ResurrectionReport::deleted_despite_not_found`]. + RefusedButDeleted, +} + +impl ResurrectionViolation { + pub fn as_str(self) -> &'static str { + match self { + ResurrectionViolation::ResurrectedWithState => "resurrected-with-state", + ResurrectionViolation::PartialState => "partial-state", + ResurrectionViolation::RefusedButDeleted => "refused-but-deleted", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionFinding { + pub violation: ResurrectionViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// What the agent was worth before the delete, and what the slot reported + /// afterwards. Carried so a finding reads without the history. + pub before: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Deletions that took: the slot came back as a new agent. + pub deleted: u64, + /// Deletions that did not. Legitimate only when the platform never + /// confirmed them. + pub survived: u64, + pub violations: u64, + pub unjudgeable: u64, + pub unprobed: u64, +} + +/// Deletions the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeletesCaught { + pub group: Group, + /// Deletes submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about. + pub deletes: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The resurrection account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionReport { + /// What each round was configured to do, so an archived finding reads + /// without the suite YAML. + pub increments_per_round: u32, + pub rounds_recorded: u64, + pub deletes_confirmed: u64, + pub deletes_indeterminate: u64, + pub deletes_rejected: u64, + /// Confirmed deletes whose slot came back as a new agent. + pub deleted_exactly: u64, + /// Deletes the driver never heard back about that took anyway. + pub indeterminate_that_deleted: u64, + /// Deletes the driver never heard back about that did not. + pub indeterminate_that_did_not: u64, + /// Deletions the platform reported as `AGENT_NOT_FOUND` whose agent was + /// nonetheless gone afterwards. + /// + /// Not a violation, and the distinction matters. Deleting is not + /// idempotent — `delete_worker_internal` opens with a metadata lookup and + /// returns not-found when there is nothing there — and worker-service's + /// routing layer retries a call whose executor became unreachable. So a + /// delete that succeeded, on an executor that then died, is retried against + /// the new owner and comes back as not-found. The work happened; the answer + /// is misleading. That is worth an operator's attention and is not the + /// platform resurrecting anything. + pub deleted_despite_not_found: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Deleted, + /// Gone, but reported to the caller as "no such agent". + DeletedDespiteNotFound, + Survived, + Violation(ResurrectionViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &DeleteRound) -> Verdict { + let Some(before) = round.before_delete else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + + if observed == COUNTER_OF_A_NEW_AGENT { + // Gone. Legitimate unless the platform said it refused — and even then, + // one refusal means the opposite of what it looks like. See below. + if round.outcome == Outcome::Rejected { + if round.rejected_as_not_found { + return Verdict::DeletedDespiteNotFound; + } + return Verdict::Violation(ResurrectionViolation::RefusedButDeleted); + } + return Verdict::Deleted; + } + if observed == before { + // Still there. Legitimate unless the platform said it was gone. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(ResurrectionViolation::ResurrectedWithState); + } + return Verdict::Survived; + } + Verdict::Violation(ResurrectionViolation::PartialState) +} + +impl ResurrectionReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), ResurrectionCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = ResurrectionReport { + increments_per_round, + rounds_recorded: rounds.len() as u64, + deletes_confirmed: 0, + deletes_indeterminate: 0, + deletes_rejected: 0, + deleted_exactly: 0, + indeterminate_that_deleted: 0, + indeterminate_that_did_not: 0, + deleted_despite_not_found: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| ResurrectionCell { + group, + window, + rounds: 0, + deleted: 0, + survived: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.deletes_confirmed += 1, + Outcome::Indeterminate => report.deletes_indeterminate += 1, + Outcome::Rejected => report.deletes_rejected += 1, + } + + match judge(round) { + Verdict::Deleted => { + cell.deleted += 1; + if round.outcome == Outcome::Confirmed { + report.deleted_exactly += 1; + } else { + report.indeterminate_that_deleted += 1; + } + } + Verdict::DeletedDespiteNotFound => { + cell.deleted += 1; + report.deleted_despite_not_found += 1; + } + Verdict::Survived => { + cell.survived += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_delete.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + findings.push(ResurrectionFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + observed, + detail: detail_for(violation, before, observed), + }); + } + } + } + + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(DeletesCaught { + group, + deletes: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S6 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S6: {} further resurrection finding(s) were dropped from the report", + self.findings_omitted + )); + } + + if self.deleted_despite_not_found > 0 { + lines.push(format!( + "S6: {} deletion(s) came back as AGENT_NOT_FOUND and had in fact taken effect. \ + Deleting is not idempotent and worker-service retries a call whose executor \ + became unreachable, so a delete that succeeded on a dying pod is reported to \ + the caller as though the agent had never existed. The work happened; the \ + answer did not say so.", + self.deleted_despite_not_found + )); + } + + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.deletes) + .sum(); + if caught == 0 { + lines.push( + "S6: the kill caught no delete in flight on the targeted executor, so this run \ + says nothing about crashing during a deletion. Every verdict below describes \ + deletes that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S6: {} rounds of {} increments then a delete; {} confirmed, {} in doubt, {} refused", + self.rounds_recorded, + self.increments_per_round, + self.deletes_confirmed, + self.deletes_indeterminate, + self.deletes_rejected + )]; + lines.push(format!( + "S6: {} confirmed deletes left a slot that came back as a new agent; {} in doubt \ + deleted anyway, {} in doubt did not", + self.deleted_exactly, self.indeterminate_that_deleted, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S6: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S6 {}: {} deletes across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.deletes, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S6 {} {}: {} rounds, {} deleted, {} survived, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.deleted, + cell.survived, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&DeleteRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for(violation: ResurrectionViolation, before: u64, observed: u64) -> String { + match violation { + ResurrectionViolation::ResurrectedWithState => format!( + "the platform confirmed the deletion of an agent worth {before}, and the slot came \ + back worth {observed} — the old agent is still there with its state intact" + ), + ResurrectionViolation::PartialState => format!( + "the agent was worth {before} when it was deleted and the slot came back worth \ + {observed}, which is neither a new agent ({COUNTER_OF_A_NEW_AGENT}) nor the old \ + one ({before}): part of a state it was supposed to have lost survived" + ), + ResurrectionViolation::RefusedButDeleted => format!( + "the platform refused to delete an agent worth {before} and it is gone anyway — the \ + slot came back worth {observed}" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use test_r::test; + + const ON_POD: &str = "chaos-s6-delete-0000"; + const CONTROL: &str = "chaos-s6-delete-0001"; + /// What a round builds an agent up to before deleting it. + const BEFORE: u64 = 3; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> DeleteRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + DeleteRound { + agent: agent.to_string(), + round: 0, + before_delete: before, + outcome, + rejected_as_not_found: false, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(90)), + observed_after: observed, + } + } + + fn build(rounds: &[DeleteRound]) -> ResurrectionReport { + ResurrectionReport::build(rounds, &split(), Some(fault()), 3) + } + + fn violations(report: &ResurrectionReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said the agent was gone, and the slot + /// came back counting from nothing. + #[test] + fn a_deletion_that_took_is_not_a_finding() { + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(0), + )]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_exactly, 1); + assert!(!report.has_violations()); + } + + /// The failure this scenario is named for. + #[test] + fn a_confirmed_deletion_whose_agent_came_back_is_a_resurrection() { + // Worth 3, deleted, and the slot reported 4 — the old agent, incremented. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(BEFORE), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::ResurrectedWithState] + ); + assert!(report.has_violations(), "this must be able to fail the run"); + let finding = &report.findings[0]; + assert_eq!(finding.before, BEFORE); + assert_eq!(finding.observed, BEFORE); + } + + /// Its opposite: refused, and gone regardless. + #[test] + fn a_refused_deletion_that_happened_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Rejected, Some(0))]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + } + + /// Neither a new agent nor the old one: some of a state that was supposed + /// to be gone survived. + #[test] + fn a_slot_that_came_back_part_way_is_a_finding() { + // Worth 3, so the delete was allowed to leave 0 or 3. It left 2. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(2), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::PartialState] + ); + assert!( + report.findings[0].detail.contains("neither a new agent"), + "{}", + report.findings[0].detail + ); + } + + /// A delete the driver never heard back about may land either way, and + /// neither answer is a defect. + #[test] + fn a_deletion_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Indeterminate, Some(0)), + round( + CONTROL, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE), + ), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_deleted, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// Why `incrementsPerRound` must be at least two, stated exactly. + /// + /// The two legal answers never collide: a fresh agent reports 1 and a + /// survivor reports `before + 1`. What one increment removes is the *gap* + /// between them, and the gap is where a partial state would show up. At + /// `before = 1` the answers are 1 and 2 with nothing in between, so + /// `PartialState` cannot fire whatever the platform does. + #[test] + fn one_increment_leaves_no_room_for_a_partial_state_to_be_seen() { + // At before = 3 there is room, and a slot landing in it is caught. + let seen = build(&[round(ON_POD, -10, Some(3), Outcome::Confirmed, Some(2))]); + assert_eq!(violations(&seen), vec![ResurrectionViolation::PartialState]); + + // At before = 1 every value is one of the two legal answers, so no + // observation can ever produce this finding. `require_delete` refuses + // the configuration rather than shipping a blind third of the oracle. + for observed in [0, 1] { + let report = build(&[round( + ON_POD, + -10, + Some(1), + Outcome::Indeterminate, + Some(observed), + )]); + assert!( + !violations(&report).contains(&ResurrectionViolation::PartialState), + "observed {observed} should be legal at before=1" + ); + } + } + + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(0))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught nothing proves nothing, and the run + /// has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_delete_says_the_run_proved_nothing() { + let report = build(&[round( + ON_POD, + -100, + Some(BEFORE), + Outcome::Confirmed, + Some(0), + )]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A delete still unresolved when the pod died is the population the whole + /// scenario is about. + #[test] + fn deletes_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(BEFORE), Outcome::Indeterminate, Some(0)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a delete"); + assert_eq!(entry.deletes, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")) + ); + } + + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(ON_POD, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(CONTROL, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().deleted, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + /// The bug the first S6 run reported 125 times, pinned. + /// + /// A slot's last round has no increment after it and is closed by a plain + /// read instead. A read reports the counter directly; an increment reports + /// the counter it just raised. `observed_after` means the counter the + /// deletion left behind, so the *workload* subtracts one from the increment + /// and the final read stores its value as-is. Get that backwards and every + /// slot's last round reads as a partial state — 125 findings out of 138,275 + /// rounds, all in `after-fault`, all with the same shape. + #[test] + fn a_round_closed_by_the_final_read_is_judged_on_the_same_scale() { + // A deleted agent reads 0. That is the fresh value, not a partial one. + let closed_by_read = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(COUNTER_OF_A_NEW_AGENT), + )]); + assert!( + violations(&closed_by_read).is_empty(), + "a final read of a deleted agent must not read as a partial state: {:?}", + closed_by_read.findings + ); + assert_eq!(closed_by_read.deleted_exactly, 1); + + // And a survivor read reports what it was worth, not one more. + let survivor = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE), + )]); + assert!(violations(&survivor).is_empty(), "{:?}", survivor.findings); + assert_eq!(survivor.indeterminate_that_did_not, 1); + } + + /// The other thing the first run found, and the reason it is not a finding. + /// + /// Deleting is not idempotent, and worker-service retries a call whose + /// executor became unreachable. So a delete that succeeded on a pod that + /// then died is retried against the new owner and comes back + /// `AGENT_NOT_FOUND`. The agent really is gone; only the answer is wrong. + #[test] + fn a_not_found_refusal_whose_agent_is_gone_is_reported_not_failed() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = true; + let report = build(&[r]); + + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_despite_not_found, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("AGENT_NOT_FOUND") && l.contains("had in fact taken effect")), + "the operator still has to be told: {:?}", + report.attention_lines() + ); + } + + /// Any *other* refusal with the agent gone is still a finding. The + /// not-found case is an exception with a mechanism behind it, not a blanket + /// excuse for refusals. + #[test] + fn a_refusal_that_is_not_about_finding_the_agent_still_fails_the_run() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = false; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + assert!(report.has_violations()); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 4) + .map(|_| round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(BEFORE))) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 4); + assert!(report.has_violations()); + } +} diff --git a/integration-tests/src/chaos/reverts.rs b/integration-tests/src/chaos/reverts.rs new file mode 100644 index 0000000000..e21e15c351 --- /dev/null +++ b/integration-tests/src/chaos/reverts.rs @@ -0,0 +1,334 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that build state up and then ask the platform to take it back +//! (GOL-371). +//! +//! One emitter per agent, running rounds. A round is `increments_per_round` +//! increments followed by one `revert` of the last `revert_invocations` of +//! them. Both numbers are exact, which is what gives this scenario an oracle +//! with no band of doubt in it: the last increment of a round *returns* the +//! counter's value, so the driver knows exactly what the agent was worth before +//! the revert, and afterwards there are exactly two legitimate answers. +//! +//! ### The revert must never be retried +//! +//! Every other operation in this suite retries once, under the original +//! idempotency key, and that retry is load-bearing: it is what exposes +//! duplicate execution. A revert has no such key and is not idempotent. +//! Reverting "the last two invocations" twice takes back four. So a revert +//! whose response was lost but which actually landed would, on retry, revert +//! again — and the result would be indistinguishable from the platform tearing +//! a truncation, which is precisely the finding this scenario exists to make. +//! +//! The retry is therefore switched off for the revert call alone, and left on +//! for the increments around it. This is the only place in the suite that does +//! that, and it is not an oversight anywhere else. +//! +//! ### How a round is judged +//! +//! Not by reading the counter: a read is an invocation, and it would land in +//! the oplog between the increments and the next revert, shifting what "the +//! last N invocations" means. The **next round's first increment** is the probe +//! instead. It returns the new value, so the value the revert left behind is +//! that minus one, and it costs nothing extra. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{RetryPolicy, RevertConfig}; +use chrono::{DateTime, Utc}; +use golem_common::model::worker::{RevertLastInvocations, RevertWorkerTarget}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertRound { + pub agent: String, + pub round: u32, + /// The counter value the round's last increment returned, so the value the + /// agent was worth immediately before the revert. `None` when an increment + /// in the round did not answer, which leaves the round unjudgeable rather + /// than failed. + pub before_revert: Option, + /// How many invocations this round's revert asked to take back. + pub asked_to_revert: u32, + /// What the revert call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// The counter value observed after the revert, taken from the next round's + /// first increment. `None` for the last round of the run, which the final + /// read-back answers instead. + pub observed_after: Option, +} + +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// Same conversion as [`crate::chaos::deletions::counter_left_by`] and named for +/// the same reason: the account judges `observed_after` as a counter, and the +/// increment probe reports one more than that. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + +/// A running revert workload. +pub struct RevertHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl RevertHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Rounds recorded so far, in no particular order. + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + /// Signals every emitter to stop and waits for operations in flight to + /// record themselves. + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos revert workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agents a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Revert, index)) + .collect() +} + +/// Starts one emitter per agent. +pub fn start(ctx: WorkloadContext, config: &RevertConfig) -> RevertHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos revert workload starting: {} emitters, {} increments then a revert of {} per \ + round, {:?} between rounds", + config.agents, + config.increments_per_round, + config.revert_invocations, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Revert, index); + let mut round: u32 = 0; + // Index into `rounds` of the round still waiting to be judged by + // the next increment this agent runs. + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + // ── Increments ────────────────────────────────────────────── + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = increment(&ctx, &agent, seq).await; + + // The first increment of a round is also the probe that + // says what the previous round's revert left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(counter_left_by(observed)); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + // ── Revert ────────────────────────────────────────────────── + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let outcome = revert_once(&ctx, &agent, round, &config).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(RevertRound { + agent: agent.clone(), + round, + before_revert: all_answered.then_some(last_value).flatten(), + asked_to_revert: config.revert_invocations, + outcome, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + RevertHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// One increment, returning the counter value it reported. +async fn increment(ctx: &WorkloadContext, agent: &str, seq: u64) -> Option { + let key = ctx.idempotency_key(agent, seq); + workload::increment_counter(ctx, Stream::Revert, agent, key) + .await + .value + .map(u64::from) +} + +/// One revert, with retries switched off. See the module docs. +async fn revert_once( + ctx: &WorkloadContext, + agent: &str, + round: u32, + config: &RevertConfig, +) -> Outcome { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-revert-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let number_of_invocations = config.revert_invocations; + let ctx2 = once.clone(); + + workload::run_operation( + &once, + Stream::Revert, + agent.to_string(), + "revert", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user + .revert( + &agent_id, + RevertWorkerTarget::RevertLastInvocations(RevertLastInvocations { + number_of_invocations: number_of_invocations as u64, + }), + ) + .await?; + Ok(None) + } + }, + ) + .await + .outcome +} + +/// The value a completed run of `rounds` rounds should leave on an agent whose +/// every round landed. +pub fn expected_after(config: &RevertConfig, rounds: u32) -> u64 { + config.net_per_round() as u64 * rounds as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn config() -> RevertConfig { + RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + } + } + + /// The arithmetic the whole oracle rests on: a round that lands is worth + /// exactly its increments less what the revert took back. + #[test] + fn a_completed_round_is_worth_its_increments_less_the_revert() { + assert_eq!(config().net_per_round(), 2); + assert_eq!(expected_after(&config(), 0), 0); + assert_eq!(expected_after(&config(), 10), 20); + } + + /// The increment probe and the final read have to land on the same scale. + /// See [`crate::chaos::deletions::counter_left_by`] for the run that proved + /// what happens when they do not. + #[test] + fn the_two_probes_agree_on_what_a_revert_left_behind() { + assert_eq!(counter_left_by(9), 8); + assert_eq!(counter_left_by(1), 0); + assert_eq!(counter_left_by(0), 0); + } + + /// A revert that takes back everything it added is legal and leaves the + /// agent where it started. Nothing in the arithmetic may go negative. + #[test] + fn a_round_that_reverts_everything_it_added_is_worth_nothing() { + let config = RevertConfig { + increments_per_round: 3, + revert_invocations: 3, + ..config() + }; + assert_eq!(config.net_per_round(), 0); + assert_eq!(expected_after(&config, 100), 0); + } +} diff --git a/integration-tests/src/chaos/rollback.rs b/integration-tests/src/chaos/rollback.rs new file mode 100644 index 0000000000..3c253d70af --- /dev/null +++ b/integration-tests/src/chaos/rollback.rs @@ -0,0 +1,450 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every agent come back to the build it was rolled back to (GOL-369)? +//! +//! S5 asks whether agents reach a *new* build when an executor dies mid-update. +//! S9 asks the return question, and the return leg is the one that matters +//! operationally: a rollback is what you reach for when the new build is +//! already going wrong, so it happening under a dying executor is exactly the +//! situation you would be in. +//! +//! ### Why the evidence is the running code, not the metadata +//! +//! `Counter::component_version` is compiled into each build — `1` in +//! `agent-counters`, `2` in `agent-counters-v2`, and nothing else differs +//! between them. Component metadata says which revision the platform *believes* +//! an agent is on; invoking `component_version` says what the code actually +//! executing reports. Only the second can distinguish a rollback that landed +//! from one the platform merely recorded. +//! +//! ### Why the forward leg is verified before the backward one +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. The same instinct as S6's smoke round and S10's +//! how-much-did-the-kill-catch line: a clean report from a scenario that never +//! happened is the worst artifact this suite can produce. +//! +//! ### Why the control-plane retries are counted apart +//! +//! The workload's retry is one attempt, transport-only, under the original +//! idempotency key, and it exists to *expose* duplicate execution. The +//! rollback's per-agent update requests are a different thing entirely: they are +//! control-plane calls aimed at agents whose executor is about to be killed, and +//! a request refused because its owner just died says nothing about the +//! platform's correctness. Counting them together would let control-plane noise +//! read as workload trouble. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// What the agents were running when asked. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionCensus { + /// When the census was taken, for the report to label it. + pub at: String, + /// The version every agent was expected to report. + pub expected: u32, + pub agents: usize, + pub on_expected: usize, + /// Agents on some other build, by the version they reported. Non-empty is + /// the finding; the key says which build they are stuck on. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub on_other: BTreeMap, + /// Agents that answered nothing. Neither passed nor failed: an agent that + /// cannot be read says nothing either way about which build it is on. + pub unreadable: usize, +} + +impl VersionCensus { + /// Builds a census from what each agent reported. + pub fn build(at: &str, expected: u32, observed: &BTreeMap>) -> Self { + let mut on_expected = 0; + let mut unreadable = 0; + let mut on_other: BTreeMap = BTreeMap::new(); + for version in observed.values() { + match version { + Some(v) if *v == expected => on_expected += 1, + Some(v) => *on_other.entry(*v).or_default() += 1, + None => unreadable += 1, + } + } + VersionCensus { + at: at.to_string(), + expected, + agents: observed.len(), + on_expected, + on_other, + unreadable, + } + } + + /// The share of agents on the expected build, out of those that answered. + /// + /// Unreadable agents are excluded from both halves rather than counted as + /// failures: they are reported separately, and treating silence as a wrong + /// answer would let a flaky read block a rollback that was fine. + pub fn share_of_answered_percent(&self) -> Option { + let answered = self.agents - self.unreadable; + (answered > 0).then(|| self.on_expected as f64 * 100.0 / answered as f64) + } +} + +/// Rollback requests, counted apart from the workload's own retries. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ControlPlaneAttempts { + pub requested: u64, + pub accepted_first_try: u64, + pub accepted_after_retry: u64, + /// Requests that never got through, even after the configured retries. Each + /// one is an agent nobody asked to come back, so it explains a stale agent + /// without excusing one. + pub refused: u64, + pub max_retries: u32, +} + +impl ControlPlaneAttempts { + pub fn accepted(&self) -> u64 { + self.accepted_first_try + self.accepted_after_retry + } +} + +/// The rollback account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackReport { + /// The revision the agents were moved forward to, and the one they were + /// rolled back to. The second carries the original build's code. + pub forward_revision: u64, + pub rollback_revision: u64, + /// What the running code reports on each of those builds, which is not the + /// revision carrying it. A revision number says where the platform filed a + /// build; these say which build answers. Mixing them silently disables + /// [`Self::stuck_on_the_new_build`], because it looks a build version up in + /// a census keyed by build version. + pub forward_version: u32, + pub rollback_version: u32, + /// The forward leg, measured before the rollback was attempted. + pub rolled_forward: VersionCensus, + /// After recovery. `None` if the run aborted before it got there. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rolled_back: Option, + pub control: ControlPlaneAttempts, + /// The floor the forward leg had to clear for the rollback to be worth + /// attempting, from the suite YAML. + pub rolled_forward_floor_percent: f64, +} + +impl RollbackReport { + /// Whether enough agents reached the new build for a rollback to mean + /// anything. + pub fn forward_leg_landed(&self) -> bool { + self.rolled_forward + .share_of_answered_percent() + .is_some_and(|share| share >= self.rolled_forward_floor_percent) + } + + /// Agents still on the build they were supposed to leave, after recovery. + pub fn stuck_on_the_new_build(&self) -> usize { + self.rolled_back + .as_ref() + .and_then(|census| census.on_other.get(&self.forward_version).copied()) + .unwrap_or(0) + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + + // The whole scenario rests on the two builds answering differently. If + // they do not, every check downstream passes vacuously, because an + // agent that never moved reads exactly like one that came back. Cheap + // to assert and it catches both a misconfigured constant and a field + // handed the wrong quantity, which is how this went unnoticed once: + // `forward_version` was fed a revision number and + // `stuck_on_the_new_build` silently stopped being able to find anyone. + if self.forward_version == self.rollback_version { + lines.push(format!( + "S9: both builds report component version {}, so this run cannot tell a \ + rollback from an agent that never moved", + self.forward_version + )); + } + + if !self.forward_leg_landed() { + lines.push(format!( + "S9: only {} of {} agents reached revision {} before the rollback ({}% of those \ + that answered, against a {:.0}% floor). Rolling agents back to a build they \ + never left proves nothing, so this run does not test rollback.", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_revision, + self.rolled_forward + .share_of_answered_percent() + .map(|s| format!("{s:.1}")) + .unwrap_or_else(|| "n/a".to_string()), + self.rolled_forward_floor_percent + )); + } + + let stuck = self.stuck_on_the_new_build(); + if stuck > 0 { + lines.push(format!( + "S9: {stuck} agent(s) still report component version {} after recovery, not the \ + {} they were rolled back to", + self.forward_version, self.rollback_version + )); + } + + if self.control.refused > 0 { + lines.push(format!( + "S9: {} of {} rollback requests were refused even after {} control-plane \ + retries. Those agents were never asked to come back, which explains a stale \ + agent without excusing one.", + self.control.refused, self.control.requested, self.control.max_retries + )); + } + + if let Some(census) = &self.rolled_back + && census.unreadable > 0 + { + lines.push(format!( + "S9: {} agent(s) could not be read after recovery, so the run cannot say which \ + build they are on", + census.unreadable + )); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S9: revision {} carries build v{}, revision {} carries build v{} again", + self.forward_revision, + self.forward_version, + self.rollback_revision, + self.rollback_version + )]; + lines.push(format!( + "S9 forward leg: {} of {} agents on revision {} before the rollback ({} \ + unreadable), read from component metadata rather than by asking the build", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_revision, + self.rolled_forward.unreadable + )); + lines.push(format!( + "S9 rollback requests: {} asked, {} accepted first try, {} after a retry, {} refused \ + (up to {} control-plane retries)", + self.control.requested, + self.control.accepted_first_try, + self.control.accepted_after_retry, + self.control.refused, + self.control.max_retries + )); + if let Some(census) = &self.rolled_back { + let elsewhere = if census.on_other.is_empty() { + "none elsewhere".to_string() + } else { + format!("elsewhere {:?}", census.on_other) + }; + lines.push(format!( + "S9 return leg: {} of {} agents report build v{} after recovery ({} unreadable, \ + {elsewhere}), read by asking the running code", + census.on_expected, census.agents, census.expected, census.unreadable, + )); + } + lines + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn census( + expected: u32, + on_expected: usize, + on_other: &[(u32, usize)], + unreadable: usize, + ) -> VersionCensus { + let mut observed: BTreeMap> = BTreeMap::new(); + let mut n = 0; + for _ in 0..on_expected { + observed.insert(format!("agent-{n:04}"), Some(expected)); + n += 1; + } + for (version, count) in on_other { + for _ in 0..*count { + observed.insert(format!("agent-{n:04}"), Some(*version)); + n += 1; + } + } + for _ in 0..unreadable { + observed.insert(format!("agent-{n:04}"), None); + n += 1; + } + VersionCensus::build("test", expected, &observed) + } + + fn report(forward: VersionCensus, back: Option) -> RollbackReport { + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: forward, + rolled_back: back, + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 200, + max_retries: 2, + ..Default::default() + }, + rolled_forward_floor_percent: 90.0, + } + } + + /// An unreadable agent is not a wrong answer. Counting it as one would let a + /// flaky read block a rollback that was perfectly fine. + #[test] + fn silence_is_excluded_from_the_share_rather_than_counted_against_it() { + // 90 on the expected build, 10 silent, none actually wrong. + let c = census(2, 90, &[], 10); + assert_eq!(c.agents, 100); + assert_eq!(c.unreadable, 10); + assert_eq!(c.share_of_answered_percent(), Some(100.0)); + assert!(report(c, None).forward_leg_landed()); + } + + /// The gate the whole scenario rests on: rolling agents back to a build + /// they never left would pass every check downstream. + #[test] + fn a_forward_leg_that_did_not_land_refuses_the_rollback() { + // 50 of 100 answered agents made it, against a 90% floor. + let r = report(census(2, 50, &[(1, 50)], 0), None); + assert!(!r.forward_leg_landed()); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("does not test rollback")), + "the operator has to be told the run proved nothing: {:?}", + r.attention_lines() + ); + } + + /// A census with nothing readable cannot clear the gate, rather than + /// clearing it vacuously on an empty average. + #[test] + fn a_census_nobody_answered_does_not_clear_the_gate() { + let c = census(2, 0, &[], 40); + assert_eq!(c.share_of_answered_percent(), None); + assert!(!report(c, None).forward_leg_landed()); + } + + /// The finding: agents still on the build they were rolled back from. + #[test] + fn agents_still_on_the_old_build_after_recovery_are_raised() { + let r = report(census(2, 200, &[], 0), Some(census(1, 197, &[(2, 3)], 0))); + assert_eq!(r.stuck_on_the_new_build(), 3); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("still report component version 2")), + "{:?}", + r.attention_lines() + ); + } + + /// A clean return raises nothing. + #[test] + fn a_rollback_that_landed_everywhere_raises_nothing() { + let r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + assert_eq!(r.stuck_on_the_new_build(), 0); + assert!(r.attention_lines().is_empty(), "{:?}", r.attention_lines()); + } + + /// A refused control-plane request explains a stale agent without excusing + /// one, so it is raised even when the return leg otherwise looks clean. + #[test] + fn refused_rollback_requests_are_raised_even_on_a_clean_return() { + let mut r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + r.control = ControlPlaneAttempts { + requested: 200, + accepted_first_try: 190, + accepted_after_retry: 6, + refused: 4, + max_retries: 2, + }; + assert_eq!(r.control.accepted(), 196); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("were refused even after")), + "{:?}", + r.attention_lines() + ); + } + + /// Two builds that answer with the same version make every downstream check + /// vacuous, so the report has to say so rather than read as a clean pass. + /// Run 33774261615 did exactly this: `forward_version` was assigned the + /// revision number instead of the build's version, both sides read 1, and + /// the notes claimed each revision carried build v1. + #[test] + fn two_builds_reporting_the_same_version_cannot_test_a_rollback() { + let mut r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + r.forward_version = 1; + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("cannot tell a rollback from an agent that never moved")), + "{:?}", + r.attention_lines() + ); + } + + /// The note has to name the two builds apart, because a reader uses it to + /// confirm the run had something to roll back from. + #[test] + fn the_notes_name_the_forward_and_rollback_builds_distinctly() { + let r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + let first = r.note_lines().first().cloned().unwrap_or_default(); + assert!( + first.contains("revision 2 carries build v2") && first.contains("build v1 again"), + "{first}" + ); + } + + /// Retries are counted apart from first-try acceptances, because the two + /// say different things about how the control plane behaved under a kill. + #[test] + fn first_try_and_retried_acceptances_are_counted_apart() { + let control = ControlPlaneAttempts { + requested: 10, + accepted_first_try: 7, + accepted_after_retry: 2, + refused: 1, + max_retries: 2, + }; + assert_eq!(control.accepted(), 9); + assert_eq!(control.accepted() + control.refused, control.requested); + } +} diff --git a/integration-tests/src/chaos/scenarios/dns_fault.rs b/integration-tests/src/chaos/scenarios/dns_fault.rs new file mode 100644 index 0000000000..6f63c0b3aa --- /dev/null +++ b/integration-tests/src/chaos/scenarios/dns_fault.rs @@ -0,0 +1,607 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The shard manager's name stops resolving on one executor: S4 (GOL-373) and +//! MF2 (GOL-537). +//! +//! [`crate::chaos::resolution`] carries the argument: why the comparison is +//! across executors rather than across time, and why the same table is read two +//! opposite ways. This module is the choreography that produces it. +//! +//! ### Why the two are one module +//! +//! MF2 is S4 with the thing that makes S4 null taken away. Same fault, same +//! population, same instrument; it adds a **shard-manager restart inside the +//! DNS window**, which drops the executor's cached connection and forces it to +//! resolve a name that no longer resolves. +//! +//! That also makes the control group sharper than it is anywhere else in the +//! suite. Both executors lose the connection to the same restart — it is a +//! shared shock, not a targeted one — and the only thing that differs between +//! them is whether they can resolve the name to rebuild it. So the gap between +//! the two groups is the DNS failure with everything else held constant. +//! +//! ### What the fault can reach, which is less than the ticket assumed +//! +//! Two pieces of executor-to-shard-manager traffic exist. `register`, once at +//! startup (`golem-worker-executor/src/grpc/mod.rs`), and **quota lease renewal +//! every ten seconds** (`services/quota.rs`). That is all of it. +//! +//! The executor never calls `get_routing_table`; only worker-service does +//! (`service/worker/routing_logic.rs`). Shard ownership is pushed shard-manager +//! to executor and the health checks run in that same direction +//! (`golem-shard-manager/src/sharding/worker_executor.rs`), so the executor's +//! own resolver cannot affect either. The ticket's "without ownership loss" is +//! right, but not because ownership survives a DNS problem — it is because a +//! DNS problem here cannot get near it. +//! +//! So the quota stream is the instrument, and it is the only stream the +//! workload weights. Its doc in [`crate::chaos::history`] already says it is the +//! only stream whose traffic crosses this link. +//! +//! ### Why the control group is load-bearing here in a way it is not elsewhere +//! +//! Every scenario that aims at one executor keeps the other as a control. In S4 +//! the control is not a comfort, it is the measurement: the headline number is +//! the target group's quota latency *as a percentage of the control group's, in +//! the same window*. A run whose quota agents all landed on one executor has no +//! comparison to make, so it is refused before the window rather than reported +//! after it. +//! +//! ### The choreography +//! +//! 1. **Warm up** the quota and durable agents, so the fault lands on a live +//! population. +//! 2. **Select** the executor owning the largest share of the quota agents, and +//! name it for the workflow. +//! 3. **Baseline** — mixed workload, long enough for several renewal cycles to +//! have gone round undisturbed. +//! 4. **Fault** — keep driving, and sample the assignment late in the window. A +//! DNS failure on an executor has no business moving shards, and a run where +//! it did is a different experiment. MF2 additionally waits for the second +//! fault and samples the assignment around it, because a shard-manager +//! restart is the one part of this that plausibly *could* move shards. +//! 5. **Heal**, then keep driving long enough for the quota stream to settle. +//! 6. **Read back and probe** — the same completion and exactly-once oracles +//! the rest of the suite ends with. + +use crate::chaos::composed::ComposedFaultReport; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::resolution::ResolutionInputs; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_back_agents, read_counters, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultInjected, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, probe, resolution, workload}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// Where in the fault window the assignment is sampled, as a fraction of it. +/// +/// Late, and for S3's reason: this sample is checking that nothing moved, and a +/// table that still looks untouched three quarters of the way in says far more +/// than one that looks untouched immediately. +const OWNERSHIP_SAMPLE_FRACTION: f64 = 0.8; + +/// How long past the enclosing fault window a composed run keeps waiting for +/// the second fault before giving up on it. +/// +/// Generous, and the reason is that a wait which runs out produces a *weaker* +/// report than one that catches a late signal: +/// `secondary-outside-primary` tells a reader the composition missed, where +/// `secondary-never-injected` only says nothing arrived. +const SECONDARY_WAIT_MARGIN: Duration = Duration::from_secs(120); + +/// How long to let the cluster react to the shard-manager restart before +/// sampling the assignment again. +/// +/// The restart is the one fault here that could plausibly move shards, and a +/// sample taken the instant it lands would describe the cluster before it had a +/// chance to. Capped against what remains of the window by the caller, so it +/// cannot fall past the heal and describe a cluster whose DNS was already back. +const RESTART_SETTLE: Duration = Duration::from_secs(60); + +/// Runs S4 or MF2 end to end. +pub async fn run( + code: ScenarioCode, + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let resolution_config = config.require_resolution()?; + // Only MF2 has one. `require_composed` is the loud version, used once the + // presence of the block has already said the run intends a composition. + let composed_config = match config.composed { + Some(_) => Some(config.require_composed()?), + None => None, + }; + let history = OperationHistory::new(code.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(code); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let quota_agents: Vec = (0..workload_config.quota_agents) + .map(|index| ctx.agent_name(Stream::Quota, index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + let mut secondary: Option = None; + let mut composed_report: Option = None; + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr, $resolution:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + if let Some(report) = $resolution { + summary = summary.with_resolution(report); + } + if let Some(report) = composed_report.clone() { + summary = summary.with_composed_fault(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: selection.clone(), + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Reads, not increments, for the reason every scenario here warms up the + // same way: an increment would be invisible to the operation history and + // would leave every read-back off by one. A cold start also looks exactly + // like a stall from outside, and this scenario's whole signal is a latency + // comparison between two populations. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("{code}: warming {} quota agents", quota_agents.len()); + let warm: Vec<(Stream, String, ReadKind)> = quota_agents + .iter() + .map(|agent| (Stream::Quota, agent.clone(), ReadKind::QuotaCounter)) + .collect(); + let _ = read_back_agents(&ctx, &[], warm).await; + info!("{code}: warmed, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + // + // On the quota agents, because a lease renewal is the only executor traffic + // that crosses the poisoned name. The durable agents are along for the + // exactly-once oracle and do not decide where the fault lands. + let subject = split::quota_subject(&ctx); + let split = match split::select(subject, deps, "a_agents).await { + Ok(split) => split, + Err(e) => { + warn!("{code}: cannot aim the DNS failure: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + }; + selection = Some(split.clone()); + + // Refused rather than reported, because the headline number is a ratio + // between the two groups and there is no second group to divide by. S19 + // refuses the same shape for a different reason — there, a lone skewed pod + // rescues its own lease and the fault is inert. Here the fault would be + // just as real and the run would still have nothing to say about it. + if split.elsewhere.is_empty() { + warn!("{code}: every quota agent landed on one executor, so there is no control group"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!( + "all {} quota agents are owned by {}, and the measurement is the target \ + executor's quota latency as a percentage of the other executor's over the \ + same window, so a one-sided split leaves nothing to compare against", + quota_agents.len(), + split.pod_address + ), + }, + &records, + Vec::new(), + None, + None + ); + } + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "{code}: baseline phase, mixed workload for {:?}", + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let mixed = workload::start(ctx.clone(), workload_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("{code}: baseline produced no confirmed operations, aborting before injection"); + mixed.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None + ); + } + + // A rebalance between selection and injection would leave the run naming + // the control group as the affected one and vice versa — a report that is + // not merely wrong but confidently inverted. + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("{code}: ownership drifted between selection and injection: {e:#}"); + mixed.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + + info!( + "{code}: baseline complete ({baseline_operations} confirmed ops), naming {} and \ + signalling readiness", + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: code.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("{code}: no fault-injected signal arrived: {e}"); + mixed.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "{code}: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // ── The second fault, for MF2 ─────────────────────────────────────────── + // + // The workflow applies it and reports it; the driver only learns when. What + // it does with that is sample the assignment on both sides of it, because a + // shard-manager restart is the one part of this composition that could + // plausibly move shards — and if it did, the two quota populations below + // are no longer the ones the run was aimed with. + // + // A wait that runs out is not an abort. The run still has an enclosing + // fault, a workload and every account after this; what it does not have is + // the composition, and the report says exactly that. + if composed_config.is_some() { + let deadline = config.phases.fault() + SECONDARY_WAIT_MARGIN; + match signals.await_secondary_fault(deadline).await { + Ok(signal) => { + info!( + "{code}: second fault {} ({} on {}) reported active at {}", + signal.fault_id, signal.kind, signal.target, signal.injected_at + ); + ownership + .push(sample_ownership(deps, "after-restart", ownership.last(), false).await); + routing_snapshots.push(snapshot_routing(deps, "after-restart").await); + secondary = Some(signal); + + // Then again once the cluster has had time to react, capped + // against what is left of the window so the sample cannot land + // after the heal and describe an executor whose DNS was back. + if let Some(composed) = composed_config { + let remaining = config + .phases + .fault() + .mul_f64((1.0 - composed.after_fraction).max(0.0)); + let settle = RESTART_SETTLE.min(remaining / 2); + info!("{code}: sampling assignment again in {settle:?}"); + tokio::time::sleep(settle).await; + ownership.push( + sample_ownership(deps, "after-restart-settled", ownership.last(), false) + .await, + ); + } + } + Err(e) => { + warn!("{code}: no secondary-fault signal arrived within {deadline:?}: {e}"); + } + } + } + + // One stop, late in the window. Unlike S19 there is nothing to probe here: + // the driver cannot see the executor's resolver, and what it would want to + // measure — whether a resolution was attempted — is not visible from any + // agent. The workflow's DNS capability preflight is where that claim is + // established instead. + // + // Timed from here rather than from injection, because on MF2 the block + // above has already spent part of the window: sleeping the full fraction + // again would run past the heal and take the "during-fault" sample after + // the fault. + let elapsed = Utc::now() + .signed_duration_since(injected.injected_at) + .to_std() + .unwrap_or(Duration::ZERO); + let sample_at = config.phases.fault().mul_f64(OWNERSHIP_SAMPLE_FRACTION); + if let Some(wait) = sample_at.checked_sub(elapsed) { + tokio::time::sleep(wait).await; + } + info!("{code}: sampling assignment late in the fault window"); + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("{code}: no fault-recovered signal arrived: {e}"); + mixed.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "{code}: name resolving again at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // Built here rather than at the end, so an abort during recovery or + // read-back still says whether the two faults ever met. Every account after + // this point describes a cluster that was under both, and a reader who + // cannot tell that apart from one under a single fault has been given the + // wrong document. + if let Some(composed) = composed_config { + let report = ComposedFaultReport::build( + &injected, + fault_recovered_at, + secondary.as_ref(), + composed.min_overlap(), + ); + for finding in &report.findings { + warn!("{code}: {}: {}", finding.violation, finding.detail); + } + composed_report = Some(report); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "{code}: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + mixed.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Account ───────────────────────────────────────────────────────────── + let records = history.snapshot(); + + let fault = fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }); + + let resolution_report = resolution::build( + &records, + ResolutionInputs { + scenario: code.as_str(), + expectation: resolution_config.expectation, + split: &split, + fault, + poisoned_name: resolution_config.poisoned_name.clone(), + degradation_ceiling_percent: resolution_config.degradation_ceiling_percent, + recovery_floor_percent: resolution_config.recovery_floor_percent, + }, + ); + for line in resolution_report.note_lines() { + info!("{line}"); + } + + let readback = read_back(&ctx, &records, "a_agents, workload_config).await; + let before_probe = read_counters(&ctx, &records).await; + let key_probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + + let exactly_once = ExactlyOnceReport::build( + &records, + &key_probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "{code}: exactly-once account — {} keys checked, {} with a final result, {} \ + recovered by the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reason = termination(&exactly_once, &records); + finish!( + reason, + &records, + readback, + Some(exactly_once), + Some(resolution_report) + ); +} + +/// The verdict, in the order a reader would want it. +/// +/// Shorter than every other scenario's, and deliberately so. Duplicate +/// execution first, because it is the only harm visible from outside the +/// cluster and the ticket's headline guarantee. Then a run where nothing +/// succeeded at all, which is a broken run rather than a finding. +/// +/// Neither [`resolution::ResolutionViolation`] appears. Both are comparisons +/// between two executors, and a comparison is a number to read rather than a +/// contract to break: a quota stream that ran slower on the pod that could not +/// resolve the shard manager is the most interesting result either scenario can +/// produce — the point of S4 under one expectation, the point of MF2 under the +/// other — and failing the run on it would bury that under a red cross instead +/// of putting it in front of someone. The assignment check is not here either — +/// [`sample_ownership`] files a movement into the summary's attention list on +/// its own. +fn termination(exactly_once: &ExactlyOnceReport, records: &[OperationRecord]) -> TerminationReason { + if let Some(reason) = exactly_once_termination(exactly_once) { + return reason; + } + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + return TerminationReason::StreamNeverSucceeded { + stream: Stream::Quota.to_string(), + }; + } + TerminationReason::Completed +} + +/// Durable state for every stream that keeps a count. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + quota_agents: &[String], + workload_config: &crate::chaos::WorkloadConfig, +) -> Vec { + let mut agents: Vec<(Stream, String, ReadKind)> = quota_agents + .iter() + .map(|agent| (Stream::Quota, agent.clone(), ReadKind::QuotaCounter)) + .collect(); + agents.extend( + (0..workload_config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .map(|agent| (Stream::Durable, agent, ReadKind::Counter)), + ); + read_back_agents(ctx, records, agents).await +} diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 4ced61c0fe..bf1464ea82 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -14,9 +14,13 @@ //! Chaos scenario implementations. //! -//! One module per scenario code. Each one owns its phase choreography — which -//! is the part that differs, and the part worth reading — while everything -//! around it lives here: where artifacts go, how a signal failure becomes a +//! One module per scenario code, except where several codes are the same +//! choreography under different settings: `storage_fault` runs S14, S15, S16, +//! S17, S18, S22 and S23, the S15A/S15B/S15C eliminations and the composed MF1, +//! which differ in which store the fault is aimed at, what the fault does to +//! it, for how long, and whether a second fault is injected inside it. Each +//! module owns its phase choreography — which is the part that differs, and the +//! part worth reading — while everything around it lives here: where artifacts go, how a signal failure becomes a //! termination reason, how a routing table is sampled, and how a result is //! assembled. //! @@ -25,20 +29,31 @@ //! abort path, or that invented a phase window it never reached, would still //! produce a plausible-looking report from a wasted maintenance window. +pub mod dns_fault; +pub mod relay_fault; pub mod s1; +pub mod s10; +pub mod s11; pub mod s12; pub mod s13; +pub mod s19; +pub mod s3; pub mod s5; +pub mod s6; +pub mod s7; pub mod s8; +pub mod s9; +pub mod storage_fault; use crate::chaos::ScenarioConfig; use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::pinned::PinnedSelection; use crate::chaos::result::{ChaosResult, Phases, RESULT_SCHEMA_VERSION, RunScope}; +use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::signal::SignalError; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, RoutingSnapshot, TerminationReason, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, RoutingSnapshot, TerminationReason, }; use crate::chaos::workload::{self, WorkloadContext}; use chrono::{DateTime, Utc}; @@ -76,6 +91,21 @@ pub struct ScenarioOutcome { pub termination_reason: TerminationReason, /// Present only for scenarios that pin the fault to one executor. pub pinned_selection: Option, + /// Present only for S10, which divides its targets around the executor the + /// fault was aimed at rather than driving only the ones it owns. + pub scheduled_selection: Option, + /// Present only for S11, which divides its waiters around the executor the + /// fault was aimed at the same way S10 divides its targets. + pub promise_selection: Option, + /// Present only for S3, which divides its agents around the executor the + /// partition cuts off rather than around one that dies. + pub isolation_selection: Option, + /// Present only for S7, which divides the agents whose state is being + /// reverted around the executor the kill is aimed at. + pub revert_selection: Option, + /// Present only for S6, which divides the agent slots being deleted around + /// the executor the kill is aimed at. + pub delete_selection: Option, } /// Assembles the archived result. @@ -99,6 +129,19 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR workload: config.workload.clone(), pinned: config.pinned.clone(), pinned_selection: outcome.pinned_selection, + scheduled: config.scheduled.clone(), + scheduled_selection: outcome.scheduled_selection, + promise: config.promise.clone(), + promise_selection: outcome.promise_selection, + isolation: config.isolation.clone(), + isolation_selection: outcome.isolation_selection, + revert: config.revert.clone(), + revert_selection: outcome.revert_selection, + delete: config.delete.clone(), + delete_selection: outcome.delete_selection, + rollback: config.rollback.clone(), + storage: config.storage.clone(), + composed: config.composed.clone(), retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, @@ -201,11 +244,14 @@ const ROUTING_POLL_SECS: u64 = 3; /// Blocks until the routing table covers every shard, or the timeout lapses. /// /// Returns the line to record, so the result says which of the two happened -/// rather than leaving a reader to infer it from timings. +/// rather than leaving a reader to infer it from timings. A settled table is +/// context — it is what every healthy run reports. An unsettled one is a +/// finding, because the baseline then measures convergence rather than the +/// platform. pub async fn wait_for_settled_routing( deps: &BenchmarkTestDependencies, snapshots: &mut Vec, -) -> String { +) -> Note { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(ROUTING_SETTLE_TIMEOUT_SECS); // Assigned on every path through the loop below before it is read. @@ -224,7 +270,7 @@ pub async fn wait_for_settled_routing( if assigned == total && executors > 0 { snapshots.push(snapshot_routing(deps, "settled-before-start").await); info!("Chaos: {last} — settled"); - return format!("{last} (settled before measuring)"); + return Note::context(format!("{last} (settled before measuring)")); } info!("Chaos: {last} — waiting for the table to cover every shard"); } @@ -237,11 +283,11 @@ pub async fn wait_for_settled_routing( if std::time::Instant::now() >= deadline { snapshots.push(snapshot_routing(deps, "unsettled-before-start").await); warn!("Chaos: {last} — proceeding anyway after {ROUTING_SETTLE_TIMEOUT_SECS}s"); - return format!( + return Note::attention(format!( "WARNING: measured against an unsettled cluster — {last}. \ Baseline numbers may reflect routing convergence rather than the \ platform." - ); + )); } tokio::time::sleep(std::time::Duration::from_secs(ROUTING_POLL_SECS)).await; } @@ -394,6 +440,18 @@ pub enum ReadKind { Polls, /// `QuotaCounter.count`, paired with `QuotaCounter.refused`. QuotaCounter, + /// `Counter.count` on the *callee* of an RPC pair (GOL-368). + /// + /// The only kind that reads a different agent from the one it is filed + /// under. The history records the caller, because that is the agent the + /// driver invoked and the one whose executor decides whether the call + /// crosses a pod; the count lives on the callee, because that is the agent + /// `increment_through_rpc` actually advances. + /// + /// Recording the callee instead would make read-back a one-liner and lose + /// the thing S2 exists to measure: which side of the partition each + /// operation started on. + RpcInner, } /// How many agents are read back at once. @@ -438,6 +496,10 @@ pub async fn read_back_agents( ReadKind::Counter => workload::read_counter(&ctx, &agent).await, ReadKind::Polls => workload::read_polls(&ctx, &agent).await, ReadKind::QuotaCounter => workload::read_quota_counter(&ctx, &agent).await, + ReadKind::RpcInner => { + let callee = workload::rpc_callee_name(&agent); + workload::read_counter(&ctx, &callee).await + } }; // Only the quota stream has a second number, and it is the one // that says what losing a lease actually cost. diff --git a/integration-tests/src/chaos/scenarios/relay_fault.rs b/integration-tests/src/chaos/scenarios/relay_fault.rs new file mode 100644 index 0000000000..5d90da2db7 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/relay_fault.rs @@ -0,0 +1,466 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The two scenarios built on the cross-pod RPC split (GOL-368, GOL-382). +//! +//! Both drive agent-to-agent calls whose two halves are known to live on +//! different executors, and both read the same instrument. They disagree only +//! about what the fault is supposed to do to it. +//! +//! **S2, the control.** Cut the two executors off from each other and assert +//! that nothing moves. The claim under test is architectural: +//! `WorkerExecutorClient` appears nowhere in the executor, so when an agent +//! invokes an agent its own executor does not own, +//! `DirectWorkerInvocationRpc::invoke_and_await` finds +//! `shard_service().check_worker()` says no and hands the call to +//! `worker_proxy`, a client of **worker-service**. Executor A reaches executor +//! B's agents through a third party, and a partition between A and B cuts a +//! link carrying no traffic. +//! +//! **S21, the load.** Leave the link alone and starve that third party of CPU +//! instead. Every call in the workload crosses worker-service once and a +//! cross-pod call crosses it twice, so the gap between the two populations is +//! exactly one worker-service hop. S2 discovered that gap while looking for +//! evidence its pairing was real; S21 exists because it is also the only clean +//! way to say a fault reached the relay and not the executors. +//! +//! ### The choreography, and where it differs from S12 +//! +//! 1. **Pair** — before anything runs, ask the shard-manager who owns each +//! caller and each callee, and split the callers into the ones whose call +//! crosses executors and the ones whose call does not. Neither the driver nor +//! the platform chooses this; both halves are placed by hashing their agent +//! ids, so the split is whatever the hash gives. +//! 2. **Baseline** — run the mixed workload, RPC stream included. +//! 3. **Gate** — refuse to spend the fault window unless enough callers came out +//! cross-pod. A run whose pairs all landed together would report exactly the +//! numbers a good run reports, and mean nothing. Same instinct as S9's +//! forward-leg gate. +//! 4. **Re-pair** — check the split again immediately before injection, because +//! an ownership change between the gate and the fault would leave the report +//! comparing populations that no longer exist. +//! 5. **Fault, recovery, read-back** — as in S12. +//! +//! The gate is the step that makes either scenario worth running. Without it a +//! green run is indistinguishable from a broken one. +//! +//! ### Why one module rather than two +//! +//! The pairing, the gate, the re-pair and the read-back indirection are the +//! whole of the driver-side work, and they are identical. What differs is which +//! fault the workflow injects and, in [`crate::chaos::relay`], which direction +//! the numbers are supposed to point. Splitting the file would duplicate the +//! parts that are easy to get subtly wrong in order to separate the parts that +//! are already separated. + +use crate::chaos::history::{OperationHistory, OperationRecord, Phase, Stream}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::relay; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, build_result, read_back_agents, sample_ownership, + signal_termination, snapshot_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::split::FaultWindow; +use crate::chaos::summary::{AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +/// Same reasoning as S12: an in-flight RPC still has to land on its callee, and +/// reading early would report a mismatch that says nothing about the platform. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// Runs S2 or S21 end to end. +pub async fn run( + code: ScenarioCode, + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let relay_config = config.require_relay()?; + let history = OperationHistory::new(code.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(code); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let callers: Vec = (0..workload_config.rpc_agents) + .map(|index| ctx.agent_name(Stream::Rpc, index)) + .collect(); + + // Paired before anything runs, so a run that cannot be paired costs no + // cluster time. Not optional: every `finish!` below attaches the pairing, + // including an abort, because on an abort the pairing is usually the reason. + info!( + "{code}: placing {} RPC pairs against the routing table", + callers.len() + ); + let mut pairing = relay::select_pairing(&ctx, deps, &callers).await?; + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership_samples = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + // Attached whenever the pairing exists, including on an abort. A run + // that stopped at the gate is exactly the run whose pairing a reader + // needs to see, since the pairing is why it stopped. + if !ownership_samples.is_empty() { + summary = summary.with_ownership(ownership_samples.clone()); + } + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + { + summary = summary.with_relay(relay::build( + $records, + pairing.clone(), + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + code, + relay_config, + )); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "{code}: baseline phase, running mixed workload for {:?}", + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = workload::start(ctx.clone(), workload_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership_samples.push(sample_ownership(deps, "before-fault", None, true).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + // ── Gate ──────────────────────────────────────────────────────────────── + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("{code}: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None + ); + } + + // Re-read the split rather than trusting the one taken before the workload. + // Ownership can move between the two, and the report is a comparison + // between populations — one built against a routing table that has since + // changed would compare the wrong agents. + let confirmed = match relay::select_pairing(&ctx, deps, &callers).await { + Ok(confirmed) => confirmed, + Err(e) => { + warn!("{code}: could not re-read the pairing before injection: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("the routing table could not be re-read before the fault: {e}"), + }, + &records, + Vec::new(), + None + ); + } + }; + let cross_pod_percent = confirmed.cross_pod_percent().unwrap_or(0.0); + pairing = confirmed; + + if cross_pod_percent < relay_config.cross_pod_floor_percent { + // The fault would have nothing to cut. Stop before spending the window: + // the clean numbers this run would produce are indistinguishable from + // the clean numbers a good run produces, which makes them worse than no + // numbers at all. + warn!( + "{code}: only {cross_pod_percent}% of callers are cross-pod, below the {}% floor", + relay_config.cross_pod_floor_percent + ); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!( + "only {cross_pod_percent}% of RPC callers had their callee on the other \ + executor, below the {}% floor — the two populations this run compares \ + would have been the same population", + relay_config.cross_pod_floor_percent + ), + }, + &records, + Vec::new(), + None + ); + } + + info!( + "{code}: baseline complete ({baseline_operations} confirmed ops, {cross_pod_percent}% \ + cross-pod), signalling readiness" + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: code.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Neither scenario pins a pod. S2 partitions every executor from every + // other one and S21 loads every worker-service replica, both selected by + // label, so there is nothing for the driver to name — unlike S3, where + // naming one side is the whole basis of the comparison. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("{code}: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "{code}: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("{code}: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "{code}: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "{code}: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership_samples + .push(sample_ownership(deps, "after-recovery", ownership_samples.first(), true).await); + + // Shards must not have moved. Nothing in this fault touches the + // shard-manager, so a reassignment means the run disturbed something it did + // not intend to, and the two populations are no longer the ones that were + // paired. The samples are attached to the summary rather than judged here, + // which is how every other scenario reports ownership. + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("{code}: letting the platform settle for {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + let readback = read_back(&ctx, &records, workload_config).await; + + // ── Exactly-once ──────────────────────────────────────────────────────── + // + // Accounted on the RPC stream rather than the durable one, which is the + // only interesting choice here. The durable stream is the same population + // every other scenario probes and this fault cannot reach it; the RPC + // stream is the one whose work crosses the partitioned link. + // + // It matters most in exactly the run that finds something. A control that + // stays green has almost no indeterminate operations to resolve, so the + // probe finds nothing. A run where cross-pod calls stalled would be full of + // them, and the probe is what says whether a stalled call executed anyway. + // The read-back above is a weaker form of the same question: it compares + // sums per agent, where this attributes to a key. + let before_probe = read_callee_counters(code, &ctx, workload_config).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Rpc).await; + let after_probe = read_callee_counters(code, &ctx, workload_config).await; + + let exactly_once = + ExactlyOnceReport::build(&records, &probes, Stream::Rpc, &before_probe, &after_probe); + info!( + "{code}: exactly-once account — {} keys checked, {} recovered by the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + finish!( + TerminationReason::Completed, + &records, + readback, + Some(exactly_once) + ); +} + +/// The current counter of every RPC callee, keyed by the **callee**. +/// +/// Keyed by the agent whose number it is rather than by the caller the probe +/// addresses, because the only thing this map feeds is +/// `ExactlyOnceReport::probe_executed_per_agent` — a list of who moved during +/// the probe pass. Naming the caller there would point an investigation at an +/// agent whose own counter never changes. +async fn read_callee_counters( + code: ScenarioCode, + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap { + let mut values = std::collections::BTreeMap::new(); + for index in 0..config.rpc_agents { + let callee = workload::rpc_callee_name(&ctx.agent_name(Stream::Rpc, index)); + match workload::read_counter(ctx, &callee).await { + Ok(value) => { + values.insert(callee, value); + } + // Recorded as absent rather than as zero: a callee that could not be + // read says nothing about whether the probe executed against it, and + // a zero here would be read as "it moved backwards". + Err(e) => warn!("{code}: could not read RPC callee {callee}: {e}"), + } + } + values +} + +/// Reads durable state back for every stream that keeps a count. +/// +/// The RPC stream is the one with an indirection: the operation is filed under +/// the caller, but the counter it advanced belongs to the callee. See +/// [`ReadKind::RpcInner`]. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, +) -> Vec { + let mut agents = Vec::new(); + for index in 0..config.rpc_agents { + agents.push(( + Stream::Rpc, + ctx.agent_name(Stream::Rpc, index), + ReadKind::RpcInner, + )); + } + for index in 0..config.durable_agents { + agents.push(( + Stream::Durable, + ctx.agent_name(Stream::Durable, index), + ReadKind::Counter, + )); + } + for index in 0..config.scheduled_agents { + agents.push(( + Stream::Scheduled, + ctx.schedule_target_name(index), + ReadKind::Polls, + )); + } + read_back_agents(ctx, records, agents).await +} diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 1b055901a3..7a4bd855af 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -76,7 +76,9 @@ use crate::chaos::scenarios::{ wait_for_settled_routing, warm_up, write_outputs, }; use crate::chaos::signal::{BaselineReady, FaultSignals, ScaleEvent}; -use crate::chaos::summary::{AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; use chrono::Utc; @@ -162,7 +164,7 @@ pub async fn run( let mut fault_id = None; let mut fault_target_observed = None; let mut inconclusive: Option = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run, alongside the // labelled phase-boundary samples. See ASSIGNMENT_SAMPLE_INTERVAL. @@ -216,7 +218,7 @@ pub async fn run( if let Some(detail) = inconclusive.clone() { summary.attention.push(detail); } - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -233,6 +235,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -557,17 +564,17 @@ pub async fn run( holds {smallest} against a balanced {balanced}" ); if executors < expected { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "executors were scaled back to {expected} during the fault, but only \ {executors} hold shards after settling — the cluster did not take the \ restored executor back" - )); + ))); } else if smallest * 2 < balanced { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "after settling the least-loaded executor holds {smallest} shards against a \ balanced {balanced}: the cluster took the executor back but has not \ rebalanced onto it" - )); + ))); } } @@ -666,6 +673,7 @@ mod tests { final_value, error: final_value.is_none().then(|| "refused".to_string()), error_class: final_value.is_none().then_some(ErrorClass::Response), + skipped: None, } } @@ -797,6 +805,7 @@ mod tests { // What `errors::classify` yields for a timeout: unreadable, so the // band of doubt widens rather than a refusal being invented. error_class: Some(ErrorClass::Transport), + skipped: None, }; let report = ExactlyOnceReport::build( diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs new file mode 100644 index 0000000000..0b8265f702 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -0,0 +1,672 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S10 — executor crash during scheduled-action fire (GOL-378). +//! +//! Every other scenario in this suite kills an executor while the driver is +//! holding a connection to it. S10 kills one while it is holding a *promise*: +//! work the platform accepted, acknowledged, and undertook to run later. Nobody +//! is waiting on the other end of a socket for it, which is exactly why it is +//! worth testing separately. A dropped invocation shows up as a failed request. +//! A dropped scheduled action shows up as nothing at all. +//! +//! ## The two windows, and which one a kill reliably lands in +//! +//! An executor's scheduler claims the actions that are due for the shards it +//! owns, leases each one, enqueues the invocation, and acknowledges. That +//! claim-to-acknowledge window is milliseconds wide: the action itself is a +//! near-no-op, and once the invocation is enqueued it is durable in the target +//! agent's oplog and covered by ordinary worker recovery rather than by the +//! lease. Killing an executor inside it is possible, not aimable, and its +//! signature is a fire delayed by roughly the lease TTL. The run reports when +//! that happens instead of claiming to have arranged it. +//! +//! The window a kill does land in, every time, is the wide one. At any instant +//! several hundred actions are registered and not yet due, and the shards they +//! belong to are owned by an executor that is about to stop existing. Nothing +//! claims them until the shards move, so the delay this scenario measures is +//! dominated by shard reassignment rather than by lease expiry. Both end in the +//! same question: does an accepted action still run, exactly once, once the +//! cluster has finished rearranging itself. +//! +//! ## Why the target agent records tokens rather than counting +//! +//! S12 already counts scheduled fires and compares the total against what the +//! driver registered. That is enough to notice that *something* went wrong +//! across a whole population, and useless for saying what. Here every +//! registration carries its own idempotency key into the scheduled action, and +//! the target agent records it when the action runs. Pairing tokens turns both +//! failures into statements about one named action: this registration, accepted +//! at this time, due at this time, never ran. See [`crate::chaos::fires`]. +//! +//! ## Why the kill is aimed, and why every target is still driven +//! +//! The driver picks the executor owning the largest share of targets and names +//! it in the readiness signal, the same way S8 does. Without that, `mode: one` +//! would pick a pod at random and the run could not say which actions were +//! supposed to be disturbed. +//! +//! Unlike S8, though, the targets it does *not* own keep running too. They are +//! the control group: on a two-executor cluster roughly half the population is +//! never touched, and reporting one percentile over both would let a lease +//! recovery that took its full TTL hide behind the half that was never +//! disturbed. +//! +//! ## What the run measures the kill against +//! +//! The driver cannot see a claim, and does not pretend to. Every target +//! re-registers on a fixed cadence at a fixed lead, so actions arrive into +//! every part of the cycle continuously and there is always a population +//! registered but not yet due. What the run then records is how large that +//! population actually was at the instant the pod died, and how much of it was +//! on the pod — measured from the history, not assumed from the cadence, +//! because a platform that had slowed down would have registered fewer than the +//! arithmetic says. A kill that caught none of it is reported as a warning: a +//! clean account of a mechanism that was never disturbed is not evidence. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`ScheduleFireReport`]. Fire delay +//! is reported against the configured lease budget as SLO evidence, not +//! asserted: how much a lease recovery may cost is a judgement, and the number +//! that matters is in the result either way. +//! +//! ## Reading a fire delay: registration first, scheduler second +//! +//! A large fire delay does not necessarily mean the scheduler was slow. The +//! registering invocation mints its action's due time *before* the call goes +//! out, so a registration that stalls in the client describes an action that was +//! already overdue when the platform first heard about it. Those fire instantly +//! and correctly, and would still arrive in the percentiles as minutes late. +//! +//! [`crate::chaos::fires`] holds them out of the scheduler-delay cells and +//! reports them as `overdue_on_arrival` instead. Two things follow for anyone +//! reading a result: +//! +//! * Check the registration latencies before concluding anything about the +//! scheduler. A client-side stall and a lease recovery look identical in a +//! delay percentile and have nothing to do with each other. +//! * Read the `overdue_on_arrival` count together with its worst case, never +//! alone. The classification catches any registration slower than the +//! configured lead, so a stall shrinking from minutes to seconds moves entries +//! *into* the bucket rather than out of it: the count can rise while the +//! platform gets strictly better. +//! +//! Run-by-run findings live in the S10 runbook in golem-cloud, not here. + +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{ + OperationHistory, OperationRecord, Outcome, Phase, Stream, TargetFireLog, +}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::scheduled::{self, ScheduledSelection}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::summary::{AgentReadback, ChaosSummary, Note, TerminationReason}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::{DateTime, TimeDelta, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let scheduled_config = config.require_scheduled()?; + let history = OperationHistory::new(ScenarioCode::S10.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S10); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: selection.clone(), + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S10: warming {} emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("S10: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend + // a maintenance window proving it. + let chosen = match scheduled::select(&ctx, deps, &targets).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S10: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S10: baseline phase, {} targets registering every {:?} at a {:?} lead, for {:?}", + targets.len(), + scheduled_config.interval(), + scheduled_config.lead(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S10: baseline registered nothing, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no scheduled registration succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None + ); + } + + // Registering is not firing. A platform that accepted every registration and + // scheduled none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never ran. + let sampled = scheduled::sample_fire_count(&ctx, &targets).await; + if sampled == 0 { + warn!("S10: {baseline_operations} registrations accepted and no action has fired"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None + ); + } + info!( + "S10: baseline complete ({baseline_operations} registrations, {sampled} fires across a \ + sample of {} targets)", + scheduled::FIRE_PROOF_SAMPLE.min(targets.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = scheduled::verify_ownership(&ctx, deps, &chosen).await { + warn!("S10: target ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + + info!( + "S10: signalling readiness with fault target {} ({} of {} targets on it)", + chosen.pod_address, + chosen.on_pod.len(), + targets.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S10.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S10: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // How much work was actually in the window the scenario is about. Measured + // from the history rather than assumed from the cadence, because a platform + // that had slowed down would have registered fewer than the arithmetic says. + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + let pending = pending_at_injection( + &history.snapshot(), + injected.injected_at, + scheduled_config.lead(), + &on_pod, + ); + attention_extra.push(pending.note()); + info!("S10: {}", pending.describe()); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S10: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S10: recovery phase, registering for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let skipped = handle.skipped(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "S10 skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase \ + counts understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = scheduled::settle_before_readback(scheduled_config); + info!("S10: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report. The + // first S10 run needed a correction to its delay percentiles that could not + // be applied afterwards, because only the reduced numbers had been kept. + history.record_fire_logs(logs.clone()); + + let report = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + &on_pod, + scheduled_config.lease_budget(), + ); + info!( + "S10: scheduled-fire account — {} registrations accepted, {} fired once, {} never \ + fired, {} inconclusive, {} unverifiable, {} findings", + report.registrations_confirmed, + report.fired_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::fires::FireViolation::NeverFired) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S10: fire delay p99 during the fault, on the killed executor's targets: {p99}ms \ + against a {}ms lease budget", + report.lease_budget_ms + ); + } + + // The count-based read-back as well, on the same read. It cannot localise + // anything the token pairing does not, but it is the view every other + // scenario reports and a disagreement between the two would itself be worth + // knowing about. + let readback = readback_from_polls(&records, &logs); + + let reason = if report.has_violations() { + TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.fired_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback, Some(report)); +} + +/// Actions that were registered but not yet due when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PendingAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl PendingAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing pending proves nothing however clean the rest + /// of its numbers look, which is the one thing here a human has to act on. + pub fn needs_attention(&self) -> bool { + self.total == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + + /// The line an operator needs in order to know whether the kill landed + /// anywhere near the mechanism under test. + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no scheduled action was between registration and its due time \ + when the executor died, so this run says nothing about lease recovery" + .to_string(); + } + format!( + "S10 killed the executor with {} scheduled actions registered and not yet due, {} \ + of them on targets it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the actions in the claim window at the moment of injection. +/// +/// Registered before the kill, due after it, and not definitively refused. A +/// refused registration is not work the platform owes anything, so counting it +/// here would overstate what the kill was aimed at. +pub fn pending_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + lead: Duration, + on_killed_executor: &BTreeSet, +) -> PendingAtInjection { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let mut pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at && r.submitted_at + lead >= injected_at) + { + pending.total += 1; + if on_killed_executor.contains(&record.agent) { + pending.on_killed_executor += 1; + } + } + pending +} + +/// Per-target read-back from `polls`, the view every other scenario reports. +fn readback_from_polls(records: &[OperationRecord], logs: &[TargetFireLog]) -> Vec { + logs.iter() + .filter_map(|log| { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback_for(Stream::Scheduled, &log.agent, scoped, observed) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::AttemptRecord; + use crate::chaos::summary::NoteLevel; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + fn registration(agent: &str, submitted_at: DateTime, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: format!("{agent}-key"), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 10, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 10, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + const LEAD: Duration = Duration::from_secs(10); + + /// The population the scenario is about: registered before the kill, due + /// after it. + #[test] + fn only_actions_still_inside_their_lead_count_as_pending() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + // Due at 105, after the kill at 100. + registration("target-0", at(95), Outcome::Confirmed), + // Due at 99, so it had already fired. + registration("target-0", at(89), Outcome::Confirmed), + // Registered after the kill. + registration("target-0", at(101), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 1); + assert_eq!(pending.on_killed_executor, 1); + } + + /// The split is what makes the percentile readable, so it has to be counted + /// here too rather than inferred later. + #[test] + fn pending_actions_are_split_by_the_executor_that_owned_them() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + registration("target-0", at(95), Outcome::Confirmed), + registration("target-1", at(95), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 2); + assert_eq!(pending.on_killed_executor, 1); + } + + /// A refusal is not work the platform owes anything, so counting it would + /// overstate what the kill was aimed at. + #[test] + fn a_refused_registration_is_not_pending_work() { + let records = vec![registration("target-0", at(95), Outcome::Rejected)]; + let pending = pending_at_injection(&records, at(100), LEAD, &BTreeSet::new()); + assert_eq!(pending.total, 0); + } + + /// The loudest thing this scenario can say: the kill missed the mechanism + /// entirely, so nothing about lease recovery can be read from the run. + #[test] + fn a_kill_that_caught_no_pending_action_says_so_rather_than_reporting_a_clean_run() { + let pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(pending.describe().starts_with("WARNING")); + assert!(pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Attention); + } + + /// The same sentence on a run that landed properly is context. It is the + /// first thing a reader wants and it is true of every healthy run, so + /// putting it in `attention` would fire CI's annotation every time. + #[test] + fn a_kill_that_landed_reports_its_count_as_context() { + let pending = PendingAtInjection { + total: 353, + on_killed_executor: 226, + }; + assert!(!pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Context); + assert!(pending.note().message.contains("353")); + } +} diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs new file mode 100644 index 0000000000..f8ccbd8d1e --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -0,0 +1,661 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S11 — executor crash during promise completion (GOL-377). +//! +//! S8 kills an executor with invocations running on it. S10 kills one holding +//! work it promised to do later. S11 kills one holding agents that are *asleep*: +//! each waiter is suspended inside an invocation, parked on a promise, occupying +//! no thread and running no code, and something outside is about to resolve that +//! promise and expect the agent to carry on. +//! +//! ## Why this is its own scenario +//! +//! A suspended waiter is the one durable-execution state with no in-memory +//! representation to lose and no queue entry to drain. Nothing about it looks +//! like work in flight. Resuming it depends on a chain that a pod kill can break +//! in the middle: the completion is written to storage under the promise's key, +//! then the worker that owns the promise is activated so it notices. Those are +//! two steps, and only the first of them is durable. +//! +//! So the failure this scenario exists to catch is specific and quiet. A +//! completion is accepted — the caller gets a success — the write lands, and the +//! activation goes to an executor that is already dying. The promise is resolved +//! forever after, and the agent waiting on it is never told. Nothing errors, +//! nothing retries, and no count anywhere goes down. +//! +//! ## What the run measures the kill against +//! +//! Each of `waiters` agents holds exactly one promise at a time, so the number of +//! agents standing suspended when the pod dies is a known constant rather than a +//! sample. The driver records how many of them were actually parked across the +//! injection, and on which executor, measured from the history rather than +//! assumed from the cadence — a platform that had slowed down would have armed +//! fewer than the arithmetic says. A kill that caught nothing parked is reported +//! as a warning: a clean account of a mechanism that was never disturbed is not +//! evidence. +//! +//! Like S8 and S10 the kill is aimed, and like S10 every waiter keeps running. +//! The ones on other executors are the control group, which is what stops a +//! recovery that took its whole budget from hiding behind the half of the +//! population that was never touched. +//! +//! ## Two independent answers, and why both are kept +//! +//! Every round is observed twice. The driver holds the `wait` invocation open, so +//! it sees the wakeup as a returning call; the waiter writes the wakeup into its +//! own durable log, which the run reads afterwards. +//! +//! The two disagree exactly when it matters. Killing the executor takes the +//! driver's connection with it, so for the rounds this scenario is about, the +//! client's view is a broken pipe and nothing more. The agent's log is what +//! answers, and [`crate::chaos::wakeups`] is built around it. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`WakeupReport`]. Wakeup delay is +//! reported against the configured budget as SLO evidence, not asserted: how much +//! a shard reassignment plus a worker recovery may cost is a judgement, and the +//! number is in the result either way. +//! +//! ## The waiter that answers nothing +//! +//! An unreadable agent normally means the run cannot say. For a suspended waiter +//! it can mean the opposite, because the read queues behind the invocation the +//! waiter is parked in. A waiter that stopped producing rounds during the run and +//! then answered no read is a worker still parked on a promise resolved minutes +//! ago — the defect itself, observed from two directions. The report separates +//! that from an agent that merely timed out. See [`crate::chaos::wakeups`]. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::waiters; +use crate::chaos::wakeups::WakeupReport; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{PromiseConfig, ScenarioCode, ScenarioConfig}; +use chrono::{DateTime, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the workload stops, before the wakeup logs are read. +/// +/// The rest of the settle is derived from the configuration: the last round's +/// completion goes out one dwell after the workload stops accepting new rounds, +/// and if its waiter's executor died holding it, the resume costs up to one +/// wakeup budget on top. Reading before that elapsed would report wakeups as +/// lost that were merely late, which is the one mistake this scenario cannot +/// afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many waiters to sample after the baseline to prove wakeups happen at all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// completion path is broken, every waiter is equally broken, and the point is to +/// fail before spending the fault window on a run that would report a clean +/// account of nothing. +const WAKE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let promise_config = config.require_promise()?; + let history = OperationHistory::new(ScenarioCode::S11.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S11); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + // The promise component, not the counters one: S11's agents live there, + // and a reader narrowing traces by component needs the one that was + // actually driven. + component_ids: vec![manifest.promise_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let waiter_names: Vec = (0..promise_config.waiters) + .map(|index| ctx.waiter_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $wakeups:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $wakeups { + summary = summary.with_promise_wakeups(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: selection.clone(), + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S11: warming {} waiters before the baseline", + waiter_names.len() + ); + let warmed = waiters::warm(&ctx, &waiter_names).await; + info!("S11: warmed {warmed} waiters, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Prove one whole round works ───────────────────────────────────────── + // Before aiming, before the baseline, before anything that costs the window. + // Arming and completing can both succeed while the parking in between is + // refused, and that combination looks entirely healthy in the operation + // totals — so the totals are not what gets asked. + if let Err(e) = waiters::smoke_test(&ctx, promise_config.dwell()).await { + warn!("S11: a single promise round does not work against this cluster: {e}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("promise round smoke test failed: {e}"), + }, + &records, + None + ); + } + info!("S11: smoke round armed, parked, completed and woke"); + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend a + // maintenance window proving it. + let chosen = match split::select(split::waiter_subject(&ctx), deps, &waiter_names).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S11: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S11: baseline phase, {} waiters parking for {:?} a round, for {:?}", + waiter_names.len(), + promise_config.dwell(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = waiters::start(ctx.clone(), &waiter_names, promise_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S11: baseline completed no promise round, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no promise round succeeded during the baseline phase".to_string(), + }, + &records, + None + ); + } + + // Completing is not waking. A platform that accepted every completion and + // resumed nobody would otherwise reach read-back and report a flawless + // account of a mechanism that never ran. + let sampled = sample_wakes(&ctx, &waiter_names).await; + if sampled == 0 { + warn!("S11: {baseline_operations} operations accepted and no waiter has woken"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + }, + &records, + None + ); + } + info!( + "S11: baseline complete ({baseline_operations} operations, {sampled} wakeups across a \ + sample of {} waiters)", + WAKE_PROOF_SAMPLE.min(waiter_names.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = split::verify_ownership(split::waiter_subject(&ctx), deps, &chosen).await { + warn!("S11: waiter ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + info!( + "S11: signalling readiness with fault target {} ({} of {} waiters on it)", + chosen.pod_address, + chosen.on_pod.len(), + waiter_names.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S11.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S11: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S11: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S11: recovery phase, running rounds for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let stood_down = handle.stalled(); + let rounds = handle.rounds(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = settle_before_readback(promise_config); + info!("S11: letting the last completions land, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + + // Only now, with every `wait` record landed, can the parked population be + // counted — see [`parked_at_injection`]. + if let Some(injected_at) = fault_injected_at { + let parked = parked_at_injection(&records, injected_at, &on_pod); + attention_extra.push(parked.note()); + info!("S11: {}", parked.describe()); + } + + let logs = waiters::read_logs(&ctx, &waiter_names).await; + // Archived alongside the operations, not just reduced into the report: the + // reduced numbers cannot be recomputed later, and a correction to how a + // delay is classified has to be applicable to a run that has already + // happened. + history.record_wakeup_logs(logs.clone()); + + let report = WakeupReport::build( + &records, + &logs, + &chosen, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + promise_config.dwell(), + promise_config.wakeup_budget(), + stood_down, + ); + info!( + "S11: promise-wakeup account — {rounds} rounds started, {} completions accepted, {} \ + woke once, {} never woke, {} inconclusive, {} unverifiable, {} findings", + report.completions_confirmed, + report.woke_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::wakeups::WakeupViolation::NeverWoke) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S11: wakeup delay p99 during the fault, on the killed executor's waiters: {p99}ms \ + against a {}ms budget", + report.wakeup_budget_ms + ); + } + + let reason = if report.has_violations() { + TerminationReason::PromiseWakeupViolated { + findings: report.violations(), + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.woke_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(report)); +} + +/// How long to wait after the workload stops before reading the wakeup logs. +fn settle_before_readback(config: &PromiseConfig) -> Duration { + config.dwell() + config.wakeup_budget() + SETTLE_MARGIN +} + +/// Waiters that were suspended on a promise when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParkedAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl ParkedAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing parked proves nothing however clean the rest of + /// its numbers look, which is the one thing here a human has to act on. So + /// does a run that caught waiters but none of them *on the pod it killed*: + /// the affected group would be empty and every number in the report would + /// describe an undisturbed cluster. + pub fn needs_attention(&self) -> bool { + self.total == 0 || self.on_killed_executor == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no waiter was suspended on a promise when the executor died, so \ + this run says nothing about promise-completion recovery" + .to_string(); + } + if self.on_killed_executor == 0 { + return format!( + "WARNING: {} waiters were suspended when the executor died but none of them \ + were on it, so the affected group is empty and this run says nothing about \ + promise-completion recovery", + self.total + ); + } + format!( + "S11 killed the executor with {} waiters suspended on promises, {} of them on \ + waiters it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the waiters parked across the moment of injection. +/// +/// A `wait` invocation that had been submitted before the kill and had not +/// returned by then. Not derived from the round arithmetic, because a platform +/// that had slowed down would have armed fewer rounds than the cadence says, and +/// the whole point of this number is to say whether the kill landed in anything. +/// +/// **This must be computed from the finished history, not from a snapshot taken +/// at injection time.** [`crate::chaos::workload::run_operation`] appends a +/// record only once the operation has completed, so a snapshot taken at the +/// moment of the kill contains none of the invocations that were open across it +/// — which is precisely the population being counted. The first S11 run reported +/// 5 waiters parked when 200 were, and read as a run that had tested nothing. +pub fn parked_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + on_killed_executor: &BTreeSet, +) -> ParkedAtInjection { + let mut parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == "wait") + .filter(|r| r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at) + .filter(|r| r.completed_at.is_none_or(|done| done > injected_at)) + { + parked.total += 1; + if on_killed_executor.contains(&record.agent) { + parked.on_killed_executor += 1; + } + } + + parked +} + +/// Total wakeups across a small sample of waiters. +async fn sample_wakes(ctx: &WorkloadContext, waiters_list: &[String]) -> u64 { + let sample: Vec = waiters_list + .iter() + .take(WAKE_PROOF_SAMPLE) + .cloned() + .collect(); + waiters::read_logs(ctx, &sample) + .await + .iter() + .filter_map(|log| log.wakes) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{Outcome, Phase, Stream}; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn wait_record(agent: &str, submitted: i64, completed: Option) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "wait".to_string(), + idempotency_key: format!("{agent}-wait"), + submitted_at: at(submitted), + completed_at: completed.map(at), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + #[test] + fn a_waiter_still_parked_at_the_kill_is_counted() { + let records = vec![wait_record("w-1", 90, Some(150))]; + let on_pod: BTreeSet = ["w-1".to_string()].into_iter().collect(); + assert_eq!( + parked_at_injection(&records, at(100), &on_pod), + ParkedAtInjection { + total: 1, + on_killed_executor: 1, + } + ); + } + + /// A round that had already woken before the kill was not disturbed by it, + /// and counting it would overstate what the fault landed in. + #[test] + fn a_waiter_that_woke_before_the_kill_is_not_counted() { + let records = vec![wait_record("w-1", 80, Some(90))]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 0); + } + + /// A `wait` that never returned at all is the most interesting case there + /// is, so it must not fall out of the count for want of a completion time. + #[test] + fn a_wait_that_never_returned_is_still_counted_as_parked() { + let records = vec![wait_record("w-1", 90, None)]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 1); + } + + /// A kill that caught waiters but none of its *own* is exactly as + /// uninformative as one that caught none at all — the affected group is + /// empty either way — and the first completed S11 run reported this shape as + /// ordinary context because the rule only looked at the total. + #[test] + fn a_kill_that_caught_none_of_its_own_waiters_needs_attention() { + let parked = ParkedAtInjection { + total: 200, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("none of them")); + } + + #[test] + fn a_kill_that_caught_nothing_parked_needs_attention() { + let parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("says nothing")); + } +} diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index d69779c6f4..537aa3375b 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -132,6 +132,11 @@ pub async fn run( ), termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index 65d65c40d6..f76d4b59a3 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -69,7 +69,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals, RestartEvent}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -138,7 +139,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run. Five rebalances in // five minutes cannot be read from phase boundaries. @@ -185,7 +186,7 @@ pub async fn run( fault_injected_at, ); summary.ownership = samples; - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -202,6 +203,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -295,7 +301,10 @@ pub async fn run( } let restarts = signals.read_restart_events(); - attention_extra.push(describe_restarts(&restarts)); + attention_extra.push(Note::leveled( + restarts.is_empty(), + describe_restarts(&restarts), + )); for event in &restarts { info!( "S13: restart {} at {} ({})", diff --git a/integration-tests/src/chaos/scenarios/s19.rs b/integration-tests/src/chaos/scenarios/s19.rs new file mode 100644 index 0000000000..fe74d85244 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s19.rs @@ -0,0 +1,720 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S19 — one executor's clock moved half a minute behind the cluster (GOL-383). +//! +//! [`crate::chaos::skew`] carries the argument: why a uniformly wrong clock is +//! invisible from inside, why the quota lease is the only place two clocks are +//! compared, why the offset is negative and why it is thirty seconds. This +//! module is the choreography that produces the numbers that argument needs. +//! +//! ### What is expected to happen +//! +//! Read out of `quota_state.rs` and `services/quota.rs`, and it is worth stating +//! ahead of the run because it decides what the phases have to be long enough to +//! contain. +//! +//! The shard-manager keeps one lease per pod per resource. The executor renews +//! when it believes fewer than `RENEWAL_THRESHOLD` remain of a +//! `LEASE_DURATION`-long lease, and it makes that judgement by subtracting its +//! own clock from an `expires_at` the shard-manager minted. A pod running thirty +//! seconds behind therefore believes it has thirty seconds more headroom than it +//! has, and renews after its lease has already expired by the granting clock. +//! +//! What happens next depends on whether anyone else touched the resource in +//! between, because expiry is lazy: `reclaim_expired` runs only inside +//! `acquire_lease` and `renew_lease`. If the healthy executor renews the same +//! resource inside that window, the skewed pod's lease is reclaimed and its own +//! renewal comes back `LeaseNotFound`. The executor marks the lease `Lost`, and +//! reservations queued against it park until the next loop re-acquires. That +//! park is the observable: a latency spike on the skewed executor's quota agents +//! and nothing at all on the other executor's. +//! +//! This is why the quota population spans **both** executors rather than being +//! pinned to the skewed one, which is where the scenario departs from the +//! ticket. A skewed pod alone renews late, rescues its own lease and nothing +//! ever notices. +//! +//! ### The choreography +//! +//! 1. **Warm up** the quota agents, the durable agents and the schedule +//! targets, so the skew lands on a live population. +//! 2. **Select** the executor owning the largest share of the *quota* agents, +//! and name it for the workflow. Quota rather than anything else because that +//! is the population the fault can actually reach. +//! 3. **Baseline** — mixed workload and registrations together, then a round of +//! clock probes that should read about zero on both sides. Those are archived +//! rather than judged, and they are what separates a broken probe from a +//! clock that never moved. +//! 4. **Fault** — keep driving, and probe both sides several times through the +//! window. Sample the assignment too: a clock skew has no business moving +//! shards, and a run where it did is a different experiment. +//! 5. **Heal**, then keep driving long enough for the quota stream to come back +//! to its baseline and for the scheduled backlog to drain under a corrected +//! clock. +//! 6. **Read back and probe** — the same completion, exactly-once and +//! scheduled-fire oracles the rest of the suite ends with. + +use crate::chaos::fires::ScheduleFireReport; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::pinned::owners_by_pod; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_back_agents, read_counters, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::skew::{ClockProbe, SkewInputs, SkewReport, SkewViolation}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, probe, scheduled, skew, workload}; +use chrono::Utc; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Where in the fault window the clock probes are taken, as fractions of it. +/// +/// Three rather than one, because the reading is a median and one probe round +/// that happened to catch a slow invocation would decide the run. Spread rather +/// than bunched, because the renewal cycle this scenario disturbs is tens of +/// seconds long and a burst of readings inside one cycle is one reading. +/// +/// None of them is at the very start: Chaos Mesh reports `AllInjected` when the +/// pod's clock has been stepped, but the executor only acts on it at its next +/// renewal, and a probe before then would read the skew correctly while the +/// quota cells around it described an undisturbed pod. +const PROBE_FRACTIONS: [f64; 3] = [0.25, 0.5, 0.75]; + +/// Where in the fault window the assignment is sampled, as a fraction of it. +/// +/// Late, and for S3's reason: this sample is checking that nothing moved, and a +/// table that still looks untouched three quarters of the way in says far more +/// than one that looks untouched immediately. +const OWNERSHIP_SAMPLE_FRACTION: f64 = 0.8; + +/// Runs S19 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let scheduled_config = config.require_scheduled()?; + let skew_config = config.require_skew()?; + let history = OperationHistory::new(ScenarioCode::S19.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S19); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let quota_agents: Vec = (0..workload_config.quota_agents) + .map(|index| ctx.agent_name(Stream::Quota, index)) + .collect(); + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut probes: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr, $fires:expr, $skew:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + if let Some(report) = $skew { + summary = summary.with_skew(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: selection.clone(), + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Reads, not increments, for the reason every scenario here warms up the + // same way: an increment would be invisible to the operation history and + // would leave every read-back off by one. A cold start also looks exactly + // like a stall from outside, and this scenario's whole signal is a latency + // comparison between two populations. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!( + "S19: warming {} quota agents and {} schedule targets", + quota_agents.len(), + targets.len() + ); + let warm: Vec<(Stream, String, ReadKind)> = quota_agents + .iter() + .map(|agent| (Stream::Quota, agent.clone(), ReadKind::QuotaCounter)) + .collect(); + let _ = read_back_agents(&ctx, &[], warm).await; + let warmed = scheduled::warm(&ctx, &targets).await; + info!("S19: warmed {warmed} scheduled agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + // + // On the quota agents, because they are the only population the fault can + // reach: a lease is the one thing here that one machine mints and another + // judges. The schedule targets are classified against whichever executor + // this picks rather than choosing it, since they are along to measure what + // the skew cost, not to decide where it lands. + let subject = split::quota_subject(&ctx); + let split = match split::select(subject, deps, "a_agents).await { + Ok(split) => split, + Err(e) => { + warn!("S19: cannot aim the clock skew: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + }; + selection = Some(split.clone()); + + // The healthy executor has to hold quota agents too, or the fault is inert + // by construction: the shard-manager only reclaims an expired lease when + // some *other* pod touches the same resource, so with every agent on one + // executor the skewed pod renews late, rescues its own lease and nothing + // disagrees. Refused before the window rather than reported after it. + if split.elsewhere.is_empty() { + warn!("S19: every quota agent landed on one executor, so nothing can contend for a lease"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!( + "all {} quota agents are owned by {}, and a stale lease is only reclaimed \ + when another pod touches the same resource, so a skew on that pod would \ + produce no disagreement to measure", + quota_agents.len(), + split.pod_address + ), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + // Which schedule targets sit on the executor about to be skewed. Computed + // from the same routing table the split came from, because ownership is per + // agent id and the targets are a different agent type: the executor holding + // most quota agents need not hold most targets. + let targets_on_faulted_pod = + match targets_on_pod(deps, &ctx, &targets, &split.pod_address).await { + Ok(on_pod) => on_pod, + Err(e) => { + warn!("S19: cannot place the schedule targets against the skewed executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "S19: {} of {} schedule targets sit on {}", + targets_on_faulted_pod.len(), + targets.len(), + split.pod_address + ); + // The fire account's group is called `on-killed-executor` because every + // scenario that read it before this one killed a pod. Said out loud in the + // result rather than left for a reader to trip over, since S19 kills + // nothing. + attention_extra.push(Note::context(format!( + "S19: the scheduled-fire account files the skewed executor's {} targets under \ + `on-killed-executor`. Nothing was killed — the group means the targets on the pod the \ + fault was aimed at, and the name is kept so archived results stay readable", + targets_on_faulted_pod.len() + ))); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S19: baseline phase, mixed workload plus {} registration loops, for {:?}", + targets.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let mixed = workload::start(ctx.clone(), workload_config); + let registrations = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + + // Archived rather than judged. If these read cleanly and the fault-window + // round reads nothing, the probe works and the clock did not move; if both + // read nothing, the instrument is broken and the run says so instead of + // blaming Chaos Mesh. + info!("S19: baseline clock probes, which should read about zero on both executors"); + probes.extend(skew::probe_round(&ctx, &split, skew_config.probes_per_round).await); + + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S19: baseline produced no confirmed operations, aborting before injection"); + mixed.stop().await; + registrations.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + let sampled = scheduled::sample_fire_count(&ctx, &targets).await; + if sampled == 0 { + warn!("S19: {baseline_operations} operations accepted and no scheduled action has fired"); + mixed.stop().await; + registrations.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + // A rebalance between selection and injection would leave the run naming + // the control group as the affected one and vice versa — a report that is + // not merely wrong but confidently inverted. + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S19: ownership drifted between selection and injection: {e:#}"); + mixed.stop().await; + registrations.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + info!( + "S19: baseline complete ({baseline_operations} confirmed ops, {sampled} fires across a \ + sample of {} targets), naming {} and signalling readiness", + scheduled::FIRE_PROOF_SAMPLE.min(targets.len()), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S19.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S19: no fault-injected signal arrived: {e}"); + mixed.stop().await; + registrations.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "S19: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // Probe rounds and one assignment sample, on a single schedule so the two + // never wait on each other. Elapsed is tracked against the fault window + // rather than slept blindly, because a probe round takes real time and + // three of them added to a fixed sleep would run past the heal. + let fault_window = config.phases.fault(); + let mut elapsed = Duration::ZERO; + let mut stops: Vec<(Duration, Stop)> = PROBE_FRACTIONS + .iter() + .map(|f| (fault_window.mul_f64(*f), Stop::Probe)) + .chain(std::iter::once(( + fault_window.mul_f64(OWNERSHIP_SAMPLE_FRACTION), + Stop::Ownership, + ))) + .collect(); + stops.sort_by_key(|(at, _)| *at); + + for (at, stop) in stops { + if let Some(wait) = at.checked_sub(elapsed) { + tokio::time::sleep(wait).await; + elapsed += wait; + } + match stop { + Stop::Probe => { + info!("S19: clock probes {elapsed:?} into the fault window"); + probes.extend(skew::probe_round(&ctx, &split, skew_config.probes_per_round).await); + } + Stop::Ownership => { + info!("S19: sampling assignment {elapsed:?} into the fault window"); + ownership + .push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + } + } + } + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S19: no fault-recovered signal arrived: {e}"); + mixed.stop().await; + registrations.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "S19: clock corrected at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S19: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let skipped = registrations.skipped(); + mixed.stop().await; + registrations.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "S19 skipped {skipped} registration ticks because targets still had their budget of \ + {} in flight — the offered rate was clamped by the platform, so the phase counts \ + understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = scheduled::settle_before_readback(scheduled_config); + info!("S19: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + history.record_fire_logs(logs.clone()); + + let fault = fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }); + + let skew_report = skew::build( + &records, + SkewInputs { + split: &split, + fault, + injected_offset_ms: skew_config.injected_offset_ms, + tolerance_ms: skew_config.tolerance_ms, + recovery_floor_percent: skew_config.recovery_floor_percent, + probes, + }, + ); + for line in skew_report.note_lines() { + info!("{line}"); + } + + let fires = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault, + &targets_on_faulted_pod, + scheduled_config.lease_budget(), + ); + info!( + "S19: scheduled-fire account — {} registrations accepted, {} fired once, {} \ + inconclusive, {} unverifiable, {} findings", + fires.registrations_confirmed, + fires.fired_once, + fires.inconclusive, + fires.unverifiable, + fires.findings.len() + ); + + let readback = read_back(&ctx, &records, "a_agents, workload_config, &targets).await; + let before_probe = read_counters(&ctx, &records).await; + let key_probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + + let exactly_once = ExactlyOnceReport::build( + &records, + &key_probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "S19: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ + the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reason = termination(&exactly_once, &fires, &skew_report, &records); + finish!( + reason, + &records, + readback, + Some(exactly_once), + Some(fires), + Some(skew_report) + ); +} + +/// What to do at one point in the fault window. +#[derive(Debug, Clone, Copy)] +enum Stop { + Probe, + Ownership, +} + +/// The verdict, in the order a reader would want it. +/// +/// Duplicate execution first, because it is the only harm visible from outside +/// the cluster and the ticket's headline guarantee. A skew that could not be +/// confirmed comes next: it is not a defect, it is a run that measured nothing, +/// and reporting the numbers underneath it as a pass is the failure this +/// scenario is most exposed to. +/// +/// [`SkewViolation::QuotaDidNotRecover`] is deliberately absent. Losing a lease +/// under skew is a legitimate response, and how long getting it back may take +/// is a judgement rather than a constant — so it is reported and left to the +/// operator, the same way the relay account's recovery finding is. +fn termination( + exactly_once: &ExactlyOnceReport, + fires: &ScheduleFireReport, + skew: &SkewReport, + records: &[OperationRecord], +) -> TerminationReason { + if let Some(reason) = exactly_once_termination(exactly_once) { + return reason; + } + if fires.has_violations() { + return TerminationReason::ScheduledFireViolated { + findings: fires.findings.len() as u64, + first: fires + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }; + } + if let Some(finding) = skew + .findings + .iter() + .find(|f| f.violation == SkewViolation::ClockNeverMoved) + { + return TerminationReason::FaultTargetUnverified { + detail: finding.detail.clone(), + }; + } + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + return TerminationReason::StreamNeverSucceeded { + stream: Stream::Quota.to_string(), + }; + } + TerminationReason::Completed +} + +/// The schedule targets owned by `pod_address`. +/// +/// Read from a fresh routing table rather than reusing the split's, because the +/// split was taken over a different agent type and only carries the agents it +/// was asked about. +async fn targets_on_pod( + deps: &BenchmarkTestDependencies, + ctx: &WorkloadContext, + targets: &[String], + pod_address: &str, +) -> anyhow::Result> { + let table = deps.shard_manager().get_routing_table().await?; + let by_pod = owners_by_pod(ctx, &table, workload::SCHEDULE_COUNTER_AGENT, targets); + Ok(by_pod + .get(pod_address) + .map(|agents| agents.iter().cloned().collect()) + .unwrap_or_default()) +} + +/// Durable state for every stream that keeps a count. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + quota_agents: &[String], + workload_config: &crate::chaos::WorkloadConfig, + targets: &[String], +) -> Vec { + let mut agents: Vec<(Stream, String, ReadKind)> = quota_agents + .iter() + .map(|agent| (Stream::Quota, agent.clone(), ReadKind::QuotaCounter)) + .collect(); + agents.extend( + (0..workload_config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .map(|agent| (Stream::Durable, agent, ReadKind::Counter)), + ); + agents.extend( + targets + .iter() + .map(|target| (Stream::Scheduled, target.clone(), ReadKind::Polls)), + ); + read_back_agents(ctx, records, agents).await +} diff --git a/integration-tests/src/chaos/scenarios/s3.rs b/integration-tests/src/chaos/scenarios/s3.rs new file mode 100644 index 0000000000..63351408a6 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s3.rs @@ -0,0 +1,455 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S3 — executor / worker-service network partition (GOL-370). +//! +//! S1 cuts the shard-manager off from an executor, and the cluster reacts: the +//! shard-manager stops hearing health checks and moves the shards. S3 cuts the +//! *other* link, and the cluster does not react at all — which is the point. +//! +//! The executor keeps talking to the shard-manager, so it keeps its shards and +//! stays in the routing table. worker-service therefore keeps being told, quite +//! correctly, that this executor owns those agents, and keeps trying to reach a +//! pod it cannot reach. There is no route around it, because as far as the +//! platform is concerned nothing is wrong. Every other scenario in the suite +//! ends with the platform recovering; this one asks what it does when there is +//! nothing to recover *from*, only something to wait out. +//! +//! ### What worker-service actually does +//! +//! Read out of `golem-worker-service/src/service/worker/routing_logic.rs`. A +//! call to an unreachable pod fails the 10s connect timeout, which is retriable, +//! so `call_worker_executor` invalidates the routing table and tries again — +//! forever. `get_delay` stops extending the backoff after five attempts and the +//! loop then settles at the 3s ceiling rather than giving up. The freshly +//! fetched table names the same unreachable pod every time, because the +//! shard-manager still believes in it. +//! +//! Two consequences, and S3 measures both: +//! +//! * **For the agents on that executor**, an invocation does not fail fast. It +//! hangs, until the caller's own attempt timeout ends it. That is the +//! "acceptance degradation with pending and timeout behaviour" the ticket +//! asks to see, and it is visible in the history as attempts that timed out +//! rather than as refusals. +//! * **For everyone else**, the routing table is one process-wide cache entry +//! per worker-service replica. Every stalled caller invalidating it costs +//! *every* caller a shard-manager round trip. `invalidation_min_delay` bounds +//! that to twice a second, so the cost should be small — but "should be" is +//! what the control group is there to check. +//! +//! ### The choreography +//! +//! 1. **Warm up** every agent, so the partition lands on a live population. +//! 2. **Select** the executor owning the largest share of them, exactly as S10 +//! and S11 pick their target, and name it for the workflow. +//! 3. **Baseline** — one emitter per agent, one operation each at a time. +//! 4. **Fault** — keep driving. Sample the routing table part-way in: S3's +//! premise is that the assignment does *not* move, and a run where it did is +//! a different experiment that has to be read differently. +//! 5. **Heal**, then keep driving so the isolated agents visibly come back. +//! 6. **Read back and probe** — the same completion and exactly-once oracles +//! every other scenario ends with. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_back_agents, read_counters, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::steady; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +/// Same reasoning as every other scenario: an increment still in flight has to +/// land, and reading early reports a mismatch that says nothing. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How far into the fault window the assignment sample is taken, as a fraction +/// of it, and the ceiling on that. +/// +/// Unlike S1 this sample is checking that nothing moved, so it is taken late +/// rather than early: a table that still looks untouched two minutes in is a +/// much stronger statement than one that looks untouched immediately. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.75; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(150); + +/// Runs S3 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let isolation = config.require_isolation()?; + let history = OperationHistory::new(ScenarioCode::S3.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S3); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = steady::agent_names(&ctx, isolation.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr, $reachability:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + if let Some(report) = $reachability { + summary = summary.with_reachability(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: selection.clone(), + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Construct every agent before measuring, for the reason S1 spells out: a + // cold start looks exactly like a stall from outside, and this scenario's + // entire signal is a comparison of throughput before and during the fault. + // A baseline that was still cold-starting would understate itself and make + // the fault look milder than it was. + // + // Reads, not increments. An increment here would be invisible to the + // operation history and would leave every read-back off by one. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S3: warming up {} counter agents", agents.len()); + let warm: Vec<(Stream, String, ReadKind)> = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + let _ = read_back_agents(&ctx, &[], warm).await; + info!( + "S3: warmed {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + // + // Chaos Mesh's `mode: one` would pick an executor at random, and a + // partition that cut off the executor owning six agents out of two hundred + // would still produce a confident-looking report. The driver names the pod + // instead; the workflow turns the IP into a pod name. + let subject = split::counter_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S3: cannot aim the partition: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S3: baseline phase, running {} emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = steady::start(ctx.clone(), isolation.agents, isolation.interval()); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S3: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None + ); + } + + // A rebalance between selection and injection would leave the run naming + // the control group as the affected one and vice versa — a report that is + // not merely wrong but confidently inverted. + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S3: ownership drifted between selection and injection: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + + info!( + "S3: baseline complete ({baseline_operations} confirmed ops), naming {} and signalling \ + readiness", + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S3.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S3: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // Evidence for the premise, not a verdict. The partitioned link is not the + // one the shard-manager uses, so the assignment is expected to be identical + // to the baseline. If it moved, the fault was wider than intended and every + // reading below has a second explanation. + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + info!("S3: sampling assignment {observe_after:?} into the fault window"); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S3: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: partition healed at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S3: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S3: letting the platform settle for {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + + let readback = read_back(&ctx, &records, &agents).await; + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "S3: exactly-once account — {} keys checked, {} with a final result, {} recovered by the \ + probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reachability = ReachabilityReport::build( + &records, + &split, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + isolation.isolated_ceiling_percent, + isolation.control_floor_percent, + isolation.recovery_budget(), + ); + info!( + "S3: reachability account — {} findings, {} isolated agents never recovered", + reachability.findings.len(), + reachability.agents_never_recovered.len() + ); + + // The assertion is the same one S1 makes, and for the same reason: a key + // that executed twice is the only harm visible from outside the cluster. + // Everything the reachability report says is reported for a human to judge, + // because "worker-service waited rather than failing fast" is a design + // question and not a defect the driver is entitled to rule on. + let reason = exactly_once_termination(&exactly_once).unwrap_or_else(|| { + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + } + } else { + TerminationReason::Completed + } + }); + + finish!( + reason, + &records, + readback, + Some(exactly_once), + Some(reachability) + ); +} + +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + agents: &[String], +) -> Vec { + let targets = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + read_back_agents(ctx, records, targets).await +} diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index 08568cdca1..d80e382490 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -76,7 +76,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ReadbackVerdict, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -150,7 +151,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); macro_rules! finish { ($reason:expr, $records:expr, $readback:expr) => {{ @@ -160,7 +161,7 @@ pub async fn run( routing_snapshots.clone(), fault_injected_at, ); - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); let result = build_result( config, ScenarioOutcome { @@ -174,6 +175,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -254,10 +260,13 @@ pub async fn run( ); let requested = request_updates(&ctx, workload_config, target_revision).await; - attention_extra.push(format!( - "update to revision {target_revision} requested for {requested} of {} durable agents \ - at {update_started_at}", - workload_config.durable_agents + attention_extra.push(Note::leveled( + requested < workload_config.durable_agents as usize, + format!( + "update to revision {target_revision} requested for {requested} of {} durable \ + agents at {update_started_at}", + workload_config.durable_agents + ), )); // ── Signal: ready for the fault ───────────────────────────────────────── @@ -290,9 +299,9 @@ pub async fn run( "S5: fault {} ({} on {}) reported active at {}, {into_update}ms into the update", injected.fault_id, injected.kind, injected.target, injected.injected_at ); - attention_extra.push(format!( + attention_extra.push(Note::context(format!( "the executor kill landed {into_update}ms into the update" - )); + ))); fault_injected_at = Some(injected.injected_at); fault_id = Some(injected.fault_id.clone()); fault_target_observed = Some(injected.target.clone()); @@ -342,12 +351,16 @@ pub async fn run( .map(|(agent, _)| agent) .collect(); let unreadable = versions.values().filter(|v| v.is_none()).count(); - attention_extra.push(format!( - "after recovery {} of {} durable agents report component version {}; {} could not be read", - versions.len() - stale.len(), - versions.len(), - EXPECTED_VERSION_AFTER_UPDATE, - unreadable + attention_extra.push(Note::leveled( + !stale.is_empty() || unreadable > 0, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + versions.len() - stale.len(), + versions.len(), + EXPECTED_VERSION_AFTER_UPDATE, + unreadable + ), )); // ── Verdict ───────────────────────────────────────────────────────────── diff --git a/integration-tests/src/chaos/scenarios/s6.rs b/integration-tests/src/chaos/scenarios/s6.rs new file mode 100644 index 0000000000..7afbb8f14e --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s6.rs @@ -0,0 +1,475 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S6 — executor pod kill during agent deletion (GOL-372). +//! +//! S7 asks the platform to forget some of an agent's work. S6 asks it to forget +//! the agent, and then kills the executor while it is doing so. +//! +//! ### Why this one can assert +//! +//! Invoking a deleted agent id creates a **new** agent, and a new counter starts +//! from nothing. So a round — increments to a known `V`, then a delete — has +//! exactly two legal answers when the slot is next used: +//! +//! * `1`, a fresh agent, meaning the deletion took +//! * `V + 1`, meaning the old agent is still there +//! +//! Neither is a defect on its own; a delete whose response was lost leaves the +//! question genuinely open. What makes one a defect is the platform's own answer +//! beside it. Confirmed, and the agent is still worth `V`, is the resurrection +//! this scenario is named for. See [`crate::chaos::resurrection`]. +//! +//! ### What is actually being killed into +//! +//! Deleting is four steps in `delete_worker_internal`: interrupt the running +//! worker, `start_deleting`, remove it from the worker service, remove it from +//! the active set. Only the third is durable. +//! +//! The interesting part is that the happy path is **already defended**. +//! `Worker::start_deleting` stops the background status flush and the +//! checkpointer first, specifically so neither can — in the executor's own +//! comment — "resurrect the cached status" after the removal. So the question +//! S6 asks is not whether anyone thought about resurrection, but whether that +//! defence survives the pod dying between the mark and the removal, when +//! whoever picks up the shard next has to decide what a worker marked for +//! deletion but never removed means. +//! +//! ### A smoke round before anything else +//! +//! The whole account rests on "a deleted id comes back as a new agent". If that +//! is not true, every round in the run reports a resurrection and the report is +//! worthless. So one throwaway agent is built, deleted and re-invoked before the +//! baseline starts, and a run whose premise is wrong aborts in seconds instead +//! of spending the maintenance window discovering it. That lesson is S11's. +//! +//! ### The choreography +//! +//! Like S8, S10, S11, S3 and S7 the driver names the pod: it picks the executor +//! owning the largest share of its agent slots and keeps driving the rest as a +//! control group. As in S7 the last thing before read-back is one read per slot, +//! which answers the round no following increment ever probed. + +use crate::chaos::deletions::{self, DeleteRound}; +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a slot still coming back from the last delete. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S6 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let delete_config = config.require_delete()?; + let history = OperationHistory::new(ScenarioCode::S6.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S6); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = deletions::agent_names(&ctx, delete_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $resurrection:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $resurrection { + summary = summary.with_resurrection(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would leave a slot holding an agent the round did not + // build. Reads here rather than increments, for the same reason every other + // scenario warms with reads: an increment would be invisible to the round + // arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S6: warming up {} delete agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S6: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Smoke round ───────────────────────────────────────────────────────── + // + // Everything below assumes a deleted id comes back as a new agent. If that + // is not true, every round reports a resurrection and the artifact is + // worthless — so one throwaway agent proves it before the baseline starts. + // The first S11 run is why this is here: a wrong premise otherwise costs + // the whole maintenance window before the numbers say so. + if let Err(e) = deletions::smoke_round(&ctx, delete_config).await { + warn!("S6: smoke round failed, aborting before the baseline: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::delete_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S6: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S6: baseline phase, running {} delete emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = deletions::start(ctx.clone(), delete_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S6: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S6: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + info!( + "S6: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S6.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S6: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + }; + info!( + "S6: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S6: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, None), + delete_config + )) + ); + } + }; + info!( + "S6: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S6: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S6: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let resurrection = build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + delete_config, + ); + info!( + "S6: resurrection account — {} rounds, {} deleted exactly, {} findings", + resurrection.rounds_recorded, + resurrection.deleted_exactly, + resurrection.findings.len() + ); + + let reason = if resurrection.has_violations() { + let first = resurrection + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::AgentResurrected { + findings: resurrection.findings.len() as u64 + resurrection.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Delete.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(resurrection)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_resurrection( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::DeleteConfig, +) -> ResurrectionReport { + ResurrectionReport::build(rounds, split, fault, config.increments_per_round) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any delete after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [DeleteRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S6: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S6: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s7.rs b/integration-tests/src/chaos/scenarios/s7.rs new file mode 100644 index 0000000000..db109fded3 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s7.rs @@ -0,0 +1,450 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S7 — executor pod kill during agent state revert (GOL-371). +//! +//! Every other scenario in this suite disturbs work that is trying to happen. +//! S7 disturbs work that is trying to be **undone**: each agent builds its +//! counter up with a run of increments and then asks the platform to take some +//! of them back, over and over, while an executor is killed underneath. +//! +//! ### Why this one can assert +//! +//! Read-backs elsewhere compare a counter against a range, and the width of the +//! range is the operations whose fate the driver could not determine. Here +//! there is no range. The last increment of a round *returns* the counter's +//! value, so the driver knows exactly what the agent was worth immediately +//! before the revert, and it asked for an exact number of invocations back. So +//! afterwards there are two legal values, `V` and `V - N`, and nothing between +//! them. See [`crate::chaos::truncation`] for what each other answer means. +//! +//! That is why S7 is one of the few scenarios whose read-back can fail the run +//! outright rather than being reported for a human to weigh. +//! +//! ### What is actually being killed into +//! +//! The truncation itself cannot tear: `RevertLastInvocations` commits a single +//! `OplogEntry::revert` marking a region deleted. The window worth aiming at is +//! the one around it — reverting takes `lock_stopped_worker`, so the worker is +//! stopped, the entry is committed, and only then is the worker status +//! reattached. An executor that dies between the commit and the reattach has +//! changed durable state and lost the thing that tells anyone about it. +//! +//! ### The choreography +//! +//! Like S8, S10, S11 and S3 the driver names the pod: it picks the executor +//! owning the largest share of its agents and keeps driving the rest as a +//! control group. Unlike them, the last thing it does before read-back is a +//! single read per agent, which answers the one round per agent that no +//! following increment ever probed. + +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::reverts::{self, RevertRound}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::truncation::TruncationReport; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a worker still coming back from the last revert. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S7 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let revert_config = config.require_revert()?; + let history = OperationHistory::new(ScenarioCode::S7.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S7); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = reverts::agent_names(&ctx, revert_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $truncation:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $truncation { + summary = summary.with_truncation(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: selection.clone(), + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would put an entry between the increments and the revert + // that counts them. Reads here rather than increments, for the same reason + // every other scenario warms with reads: an increment would be invisible to + // the round arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S7: warming up {} revert agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S7: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::revert_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S7: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S7: baseline phase, running {} revert emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = reverts::start(ctx.clone(), revert_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S7: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S7: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + info!( + "S7: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S7.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S7: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + }; + info!( + "S7: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S7: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, None), + revert_config + )) + ); + } + }; + info!( + "S7: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S7: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S7: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let truncation = build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + revert_config, + ); + info!( + "S7: truncation account — {} rounds, {} applied exactly, {} findings", + truncation.rounds_recorded, + truncation.applied_exactly, + truncation.findings.len() + ); + + let reason = if truncation.has_violations() { + let first = truncation + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::RevertTruncationViolated { + findings: truncation.findings.len() as u64 + truncation.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Revert.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(truncation)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_truncation( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::RevertConfig, +) -> TruncationReport { + TruncationReport::build( + rounds, + split, + fault, + config.increments_per_round, + config.revert_invocations, + ) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any revert after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [RevertRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S7: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S7: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index b53fde75f1..81a7717e76 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -152,6 +152,11 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: selection.clone(), + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; @@ -489,6 +494,7 @@ mod tests { // answer was no" — the only failure that is evidence about the key // rather than about the connection. error_class: final_value.is_none().then_some(ErrorClass::Response), + skipped: None, } } @@ -500,6 +506,7 @@ mod tests { final_value: None, error: Some(format!("{class} failure")), error_class: Some(class), + skipped: None, } } diff --git a/integration-tests/src/chaos/scenarios/s9.rs b/integration-tests/src/chaos/scenarios/s9.rs new file mode 100644 index 0000000000..6b3e09f6a0 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s9.rs @@ -0,0 +1,722 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S9 — executor pod kill during a component rollback (GOL-369). +//! +//! S5 moves agents forward onto a new build and kills an executor while that is +//! happening. S9 moves them forward, waits for that to land, and then moves them +//! **back**, killing an executor during the return leg. +//! +//! The return leg is the one that matters operationally. A rollback is what you +//! reach for when the new build is already going wrong, so a rollback happening +//! under a dying executor is not a contrived situation — it is the situation you +//! would actually be in. +//! +//! ## What a rollback is here +//! +//! A redeploy. The original artifact is uploaded again as a **new** revision and +//! every agent is asked to move to it. That is what rollback means in practice, +//! and it is what makes the evidence unambiguous: `Counter::component_version` +//! is compiled into each build — `1` in `agent-counters`, `2` in +//! `agent-counters-v2`, nothing else different — so an agent that has genuinely +//! returned reports `1` from the code that is running, not from metadata about +//! what the platform believes. +//! +//! ## The constraint that shapes this scenario +//! +//! An automatic update **replays the agent's oplog against the new build** and +//! aborts if any recorded invocation produces a different result. It can only +//! cross a build boundary that no recorded invocation can tell apart. +//! +//! That has a sharp operational consequence, and it is arguably S9's most +//! useful finding: **a behaviour-changing build cannot be rolled back +//! automatically.** If the new build ever returned a different answer for an +//! invocation still in an agent's oplog — which is usually *why* you are +//! rolling back — the automatic update is refused. A rollback in that situation +//! needs a snapshot-based update instead. +//! +//! It also dictates how this scenario may look at its own agents. The first S9 +//! run verified the forward leg by invoking `component_version`, which exists +//! precisely to differ between builds, and thereby wrote an entry into all 200 +//! oplogs that the rollback's replay could never reproduce. Every rollback was +//! refused with `Unexpected oplog entry: expected component_version => 1, got +//! 2`. The forward leg is now read from metadata, which leaves no trace; the +//! running code is asked only at the very end, where nothing depends on it. +//! See `an_automatic_update_rolls_an_agent_back_to_an_earlier_build` in +//! `golem-worker-executor/tests/hot_update.rs` for the minimal reproduction. +//! +//! ## Why the forward leg is verified first +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. Same instinct as S6's smoke round: a clean report from a +//! scenario that never happened is the worst artifact this suite can produce. +//! +//! ## What fails the run +//! +//! The same two things S5 asserts, reused rather than re-invented because they +//! are the same facts: +//! +//! - an agent whose durable state fell below what the driver was told succeeded, +//! or rose above what it could possibly have asked for; +//! - an agent still reporting the build it was rolled back *from*. +//! +//! An agent that cannot be read at all is reported rather than assumed either +//! way, and control-plane refusals are counted apart from workload retries — +//! see [`crate::chaos::rollback`] for why that separation is load-bearing. + +use crate::chaos::history::{OperationHistory, OperationRecord, Phase, Stream}; +use crate::chaos::prep::{COUNTERS_V2_WASM, COUNTERS_WASM, ChaosPrepManifest}; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, warm_up, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How many agents to update concurrently. +/// +/// Update requests are cheap to issue and the point is that many are in flight +/// when the executor dies, so this is wide rather than polite. +const UPDATE_CONCURRENCY: usize = 32; + +/// What the running code reports on each of the two builds. +/// +/// Compiled into the WASM rather than read from metadata, which is the whole +/// reason these numbers can be trusted: metadata says what the platform +/// believes, `component_version` says what is executing. +const VERSION_ON_THE_NEW_BUILD: u32 = 2; +const VERSION_AFTER_ROLLBACK: u32 = 1; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let rollback_config = config.require_rollback()?; + let history = OperationHistory::new(ScenarioCode::S9.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S9); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + let mut rollback_report: Option = None; + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = rollback_report.clone() { + summary = summary.with_rollback(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Same reason as S1: an agent's first invocation costs far more than its + // later ones, and a population still cold-starting is a different thing to + // update than a population that is running. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S9: warming up agents before the baseline"); + let warmed = warm_up(&ctx, workload_config).await; + info!("S9: warmed {warmed} agents, settling {:?}", WARMUP_SETTLE); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S9: baseline phase, running mixed workload for {:?}", + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = workload::start(ctx.clone(), workload_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S9: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new() + ); + } + + // ── Roll forward ──────────────────────────────────────────────────────── + // + // The workload keeps running throughout. This leg is not the experiment: it + // exists to put the agents somewhere they can be brought back *from*. + info!("S9: rolling forward — updating the counters component to {COUNTERS_V2_WASM}"); + let forward = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_V2_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: roll-forward component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("component update to {COUNTERS_V2_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let forward_revision = forward.revision; + info!( + "S9: component now at revision {forward_revision}, moving {} durable agents onto it", + workload_config.durable_agents + ); + let _ = request_updates(&ctx, workload_config, forward_revision, 0, Duration::ZERO).await; + + // Let the forward leg land before measuring it. Without this the census + // below reads a population still in transit, and the gate would refuse a + // rollback that would have been perfectly good. + info!( + "S9: letting the roll-forward settle for {:?}", + rollback_config.settle() + ); + tokio::time::sleep(rollback_config.settle()).await; + + // Read from metadata, NOT by invoking `component_version`. + // + // This is the correction the first S9 run forced, and it is not a + // downgrade of the evidence — it is the only way to gather it without + // destroying what comes next. An automatic update replays the agent's + // oplog against the new build and aborts if any recorded invocation + // produces a different result. `component_version` exists precisely to + // differ between builds, so invoking it here writes an entry into every + // agent's oplog that the rollback's replay can never reproduce. The first + // run did exactly that and all 200 rollbacks were refused with + // "Unexpected oplog entry: expected component_version => 1, got 2". + // + // The forward leg only has to establish that the agents moved, so that the + // rollback has something to undo. Which revision the platform has them on + // answers that, and leaves no trace. The end state is still judged on the + // running code, at the very end, where nothing depends on it. + let rolled_forward = VersionCensus::build( + "before-rollback", + forward_revision.get() as u32, + &read_revisions(&ctx, workload_config).await, + ); + info!( + "S9: {} of {} agents are on revision {} ({} unreadable)", + rolled_forward.on_expected, + rolled_forward.agents, + forward_revision, + rolled_forward.unreadable + ); + + // ── Roll back ─────────────────────────────────────────────────────────── + // + // A redeploy of the original artifact as a new revision. Uploading it again + // rather than pointing agents back at the old revision is what "rollback" + // means operationally, and it keeps the evidence in the running code. + info!("S9: rolling back — re-uploading {COUNTERS_WASM} as a new revision"); + let back = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: rollback component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("rollback upload of {COUNTERS_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let rollback_revision = back.revision; + + let mut report = RollbackReport { + forward_revision: forward_revision.get(), + rollback_revision: rollback_revision.get(), + forward_version: VERSION_ON_THE_NEW_BUILD, + rollback_version: VERSION_AFTER_ROLLBACK, + rolled_forward, + rolled_back: None, + control: ControlPlaneAttempts::default(), + rolled_forward_floor_percent: rollback_config.rolled_forward_floor_percent, + }; + + // Refuse to spend the maintenance window rolling agents back to a build + // they never left. Every check would pass and the run would prove nothing. + if !report.forward_leg_landed() { + warn!("S9: the roll-forward did not land, refusing to roll back"); + handle.stop().await; + let records = history.snapshot(); + let detail = report + .attention_lines() + .first() + .cloned() + .unwrap_or_else(|| "the roll-forward did not land".to_string()); + rollback_report = Some(report); + finish!( + TerminationReason::FaultTargetUnverified { detail }, + &records, + Vec::new() + ); + } + + info!( + "S9: rollback revision is {rollback_revision}, asking {} durable agents to return", + workload_config.durable_agents + ); + let rollback_started_at = Utc::now(); + report.control = request_updates( + &ctx, + workload_config, + rollback_revision, + rollback_config.control_retries, + rollback_config.control_retry_delay(), + ) + .await; + attention_extra.push(Note::leveled( + report.control.refused > 0, + format!( + "rollback to revision {rollback_revision} accepted for {} of {} durable agents at \ + {rollback_started_at}", + report.control.accepted(), + workload_config.durable_agents + ), + )); + rollback_report = Some(report); + + // ── Signal: ready for the fault ───────────────────────────────────────── + tokio::time::sleep(rollback_config.kill_delay()).await; + info!( + "S9: {:?} into the rollback, signalling readiness for the kill", + rollback_config.kill_delay() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S9.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Agents are mid-rollback across both executors, so killing either one + // interrupts a return in flight. Which one carries no information. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S9: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + let into_update = (injected.injected_at - rollback_started_at).num_milliseconds(); + info!( + "S9: fault {} ({} on {}) reported active at {}, {into_update}ms into the rollback", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + attention_extra.push(Note::context(format!( + "the executor kill landed {into_update}ms into the rollback" + ))); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S9: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + info!("S9: fault cleared at {}", recovered.recovered_at); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S9: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S9: settling {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + let readback = read_back(&ctx, &records, workload_config).await; + + // One census, read once. Both the count and the stale list below describe + // the same moment: reading twice let an agent be counted as rolled back in + // the first pass and listed as stale in the second, so the report could + // contradict itself over an agent that simply landed between the two. + let versions = read_versions(&ctx, workload_config).await; + let rolled_back = VersionCensus::build("after-recovery", VERSION_AFTER_ROLLBACK, &versions); + let stale: Vec = versions + .into_iter() + .filter(|(_, v)| *v == Some(VERSION_ON_THE_NEW_BUILD)) + .map(|(agent, _)| agent) + .collect(); + attention_extra.push(Note::leveled( + rolled_back.on_expected < rolled_back.agents, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + rolled_back.on_expected, + rolled_back.agents, + VERSION_AFTER_ROLLBACK, + rolled_back.unreadable + ), + )); + if let Some(report) = rollback_report.as_mut() { + report.rolled_back = Some(rolled_back); + } + + // ── Verdict ───────────────────────────────────────────────────────────── + let reason = if let Some(bad) = readback.iter().find(|r| { + matches!( + r.verdict, + ReadbackVerdict::LostWork | ReadbackVerdict::DuplicateExecution + ) + }) { + TerminationReason::UpdateStateInconsistent { + agent: bad.agent.clone(), + detail: format!( + "{:?}: observed {:?} against an expected range of {}..={}", + bad.verdict, bad.observed, bad.expected_min, bad.expected_max + ), + } + } else if let Some(agent) = stale.first() { + // An agent still answering with the build it was rolled back *from* is + // a real failure. One that could not be read at all is not: an + // unreadable agent is reported above and says nothing either way. + // + // `UpdateNotApplied` rather than a parallel rollback variant, because + // it is the same fact — an update that did not land — and inventing a + // second name for it would only make the two harder to search for. + TerminationReason::UpdateNotApplied { + agent: agent.clone(), + observed: Some(VERSION_ON_THE_NEW_BUILD), + expected: VERSION_AFTER_ROLLBACK, + } + } else if let Some(stream) = stream_that_never_succeeded(&ChaosSummary::build( + &records, + readback.clone(), + routing_snapshots.clone(), + fault_injected_at, + )) { + TerminationReason::StreamNeverSucceeded { + stream: stream.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback); +} + +/// Asks every durable agent to move to `target_revision`, concurrently. +/// +/// Returns how many requests the platform accepted. A refused request is +/// recorded and the run continues: the scenario is about what happens to the +/// updates that *did* start when the executor died. +/// Asks every durable agent to move to `target_revision`, retrying refusals. +/// +/// The retries here are the **control plane's**, counted apart from the +/// workload's. They answer a different question: a request refused because its +/// agent's executor just died says nothing about the platform's correctness, +/// but an agent nobody successfully asked to come back explains a stale agent +/// later without excusing one. Passing `0` retries makes this the same +/// fire-once call S5 does. +async fn request_updates( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, + target_revision: golem_common::model::component::ComponentRevision, + retries: u32, + delay: Duration, +) -> ControlPlaneAttempts { + let mut pending: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut account = ControlPlaneAttempts { + requested: pending.len() as u64, + max_retries: retries, + ..Default::default() + }; + + for attempt in 0..=retries { + if pending.is_empty() { + break; + } + if attempt > 0 && !delay.is_zero() { + tokio::time::sleep(delay).await; + } + + let mut refused = Vec::new(); + for chunk in pending.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + // `disable_wakeup: false` — the agent should be woken to + // process the update rather than waiting for its next + // invocation, because the kill is timed against the + // rollback starting, not against the next caller happening + // along. + ctx.user + .auto_update_worker(&id, target_revision, false) + .await + .map_err(|e| (agent, e)) + }); + } + while let Some(joined) = batch.join_next().await { + match joined { + Ok(Ok(())) => { + if attempt == 0 { + account.accepted_first_try += 1; + } else { + account.accepted_after_retry += 1; + } + } + Ok(Err((agent, e))) => { + warn!("S9: update request for {agent} refused: {e:#}"); + refused.push(agent); + } + Err(e) => warn!("S9: an update request task panicked: {e}"), + } + } + } + pending = refused; + } + + account.refused = pending.len() as u64; + account +} + +/// Asks the platform which component revision each durable agent is on. +/// +/// Deliberately metadata rather than an invocation. See the comment at the +/// forward-leg census: invoking a method whose result differs between builds +/// writes an oplog entry that the next automatic update's replay cannot +/// reproduce, which aborts that update. Reading metadata leaves no trace. +async fn read_revisions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + let observed = ctx + .user + .get_worker_metadata(&id) + .await + .ok() + .map(|m| m.component_revision.get() as u32); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + +/// Asks every durable agent which build it is running. +/// +/// `None` means the agent could not be read, which is reported rather than +/// counted against the update. +async fn read_versions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let observed = workload::read_component_version(&ctx, &agent).await.ok(); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + +/// Durable and scheduled state, compared against what the driver submitted. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for index in 0..config.scheduled_agents { + let target = ctx.schedule_target_name(index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == target); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_polls(ctx, &target).await; + readback.extend(readback_for(Stream::Scheduled, &target, scoped, observed)); + } + + readback +} diff --git a/integration-tests/src/chaos/scenarios/storage_fault.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs new file mode 100644 index 0000000000..adbb6f9900 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -0,0 +1,1213 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage fault: the shared choreography behind S14, S15, S16, S17, S18, S22 +//! and S23, plus the S15A/S15B/S15C eliminations and the composed MF1. +//! +//! All eleven codes run this module. They differ in which store the fault is +//! aimed at, what it does to that store and for how long, all of which are +//! suite settings, and in what a reader should expect of the result: +//! +//! * **S16** (GOL-379) cuts the key-value cluster for the length of an AWS +//! storage failover, about a minute. The key-value retry budget covers that, +//! so the claim is that the platform absorbs it: operations stall and then +//! complete, and no executor is lost. +//! * **S22** (GOL-499) cuts the same cluster for longer than that budget. The +//! budget is then exhausted, the recovery-index write in `status_flusher` +//! panics on purpose, and executors are replaced. The claim is not survival +//! but that the exit is the intended one and nothing is lost across it. +//! * **S14** (GOL-376) cuts the *other* Aurora cluster, the one carrying the +//! oplog, again for the length of a failover. golem-dev gives the indexed +//! retry 200 attempts with a 10s cap, so the budget is not the question here +//! the way it is in the other two, and the claim is again absorption. +//! * **S18** (GOL-384) cuts neither Aurora cluster but the Redis cache in front +//! of the key-value layer, and holds it past the caller's patience rather +//! than past any platform budget. There is no budget to exhaust: golem-dev +//! configures that client to retry forever. The claim is that a stall which +//! outlasts the caller does not turn one operation into two. +//! * **S17** (GOL-375) leaves that same cache reachable and makes it slow. The +//! only scenario here that breaks nothing, and a different question rather +//! than a milder version of S18's: not whether the platform survives losing +//! its worker-status store, but whether it degrades or breaks when that store +//! gets slower. Going quiet would be a failure here rather than the expected +//! result. +//! * **S15** (GOL-374) slows the key-value cluster instead, so it is to S16 +//! what S17 is to S18 and the mirror image of S17 at the same time. The +//! claim is the same one — degradation rather than breakage — but on the +//! opposite set of streams, and it carries a risk the Redis delay does not: +//! this side's connection pools are bounded and its failure path is a panic. +//! See the suite entry for the arithmetic. +//! * **S15A**, **S15B** and **S15C** are S15 with streams taken away rather +//! than scenarios of their own: same fault, same endpoint, same phases, one +//! stream added back per entry, so whichever addition first makes `ephemeral` +//! suffer names the interaction. S15A drives `ephemeral` alone and so names +//! no slowed stream at all: its whole claim sits on the other side of the +//! expectation, that `ephemeral` stays steady. That is the one case where an +//! empty `slowed` list is the claim rather than an omission. See the suite +//! entries. +//! * **S23** (GOL-525) slows the indexed cluster instead of cutting it, so it +//! is to S14 what S15 is to S16, and it fills the last empty cell of the +//! matrix. It is the only delay here with no control stream, because every +//! agent commits its oplog to the store being slowed — `ephemeral` included, +//! one layer down. That costs the run the routing check the other two delays +//! get for free, and the evidence has to come from the `svc`-labelled storage +//! series instead. It also meets the one concurrency gate the suite has: +//! `GOLEM__INDEXED_STORAGE__CONFIG__MAX_CONCURRENT_OPS`, a semaphore tighter +//! than the pool behind it. +//! +//! * **MF1** (GOL-381) is the odd one out and the only code here that injects +//! more than one fault. It cuts the key-value cluster exactly as S16 does and +//! kills a worker-executor half way through that window. Everything below is +//! the same choreography with three additions, each guarded on the presence +//! of a `composed` block: the kill is aimed, a second fault signal is waited +//! for inside the window, and the shard assignment is sampled around it. What +//! it asks is not in the single-fault matrix at all — a survivor has to take +//! over shards while the running-workers set it needs to do that is behind +//! the cut. See [`crate::chaos::composed`] for how a composition that missed +//! is told apart from one that worked. +//! +//! The driver is the same in all eleven because the difference is one of +//! expectation, not of choreography. Nothing below asserts on which outcome +//! happened: the account it produces answers all ten questions, and the +//! oracles that fail the build — the scheduled-fire account and the +//! exactly-once account — are the ones every one of them shares. +//! +//! The first scenarios in the suite that break something the platform depends +//! on rather than something the platform *is*. Every fault before these removed +//! a golem process or a link between two of them, and in each case some part of +//! the cluster stayed healthy and could be compared against. Here the executors +//! keep running, keep their shards, keep answering the shard-manager, and +//! simply cannot reach a database underneath them. +//! +//! ## What the fault takes away +//! +//! Three different pieces of the platform, depending on the code. The two +//! Aurora cuts are close to mirror images of each other; the Redis cut is a +//! third thing again. +//! +//! Reading the golem-dev executor deployment, the **key-value** cluster that +//! S16 and S22 cut carries four things: +//! +//! * promises, +//! * the running-workers set, +//! * user-defined key-value data, +//! * and the scheduler, in its own schema on the same cluster. +//! +//! The oplog lives on a different Aurora cluster and the worker-status hot +//! cache lives in Redis, and that partition touches neither. So the platform +//! keeps the ability to *record* what it did and loses the ability to know +//! *what it is doing*. +//! +//! The **indexed** cluster that S14 cuts and S23 slows carries the oplog and +//! nothing else. Promises still resolve, the scheduler still claims and +//! acknowledges on time, the running-workers set is still writable. What goes +//! is the ability to commit anything durable at all, which is the opposite +//! arrangement: the platform knows exactly what it is doing and cannot record +//! any of it. +//! +//! The **Redis cache** that S18 and S17 aim at is not a cluster at all but the +//! front half of the key-value layer. `NamespaceRoutedKeyValueStorage` sends the `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` namespaces to it and everything +//! else to Postgres, so S18 removes exactly the part S16 leaves standing and +//! S17 slows the same part instead. Both Aurora clusters stay reachable +//! throughout either. +//! +//! S15 aims at the back half of that same layer, which is the key-value cluster +//! again, and the split decides its streams too. `durable` waits on the +//! `RunningWorkers` recovery index, which `AgentStatusFlusher::on_status_changed` +//! updates synchronously whenever an agent crosses between tracked and +//! untracked; `promise` reads and writes promise keys; the scheduler registers +//! into its own schema on the same cluster. `ephemeral` reaches none of it, and +//! is the one stream that should not move. +//! +//! No stream is a control group under the Aurora *cuts*. A durable increment +//! needs the running-workers set before it can start, the worker-status cache +//! to resolve its mode, and the oplog before it can finish, so `durable` +//! degrades under both of those and must not be read as untouched. The three +//! scenarios aimed at one half of the key-value layer are the exception, and +//! deliberately so: there the streams that keep working — or keep their pace — +//! are evidence rather than noise, which is why they are judged by their own +//! expectations. See [`crate::chaos::OutageExpectation`]. +//! +//! S23 is neither, and it is worth being explicit about which of the two it +//! resembles. It aims at one store rather than at half of a split layer, but +//! every stream reaches that store, so it has the *cuts'* problem of having +//! nowhere to look for a control while making the *delays'* claim about +//! degradation. Its `steady` list is therefore empty and the routing check is +//! left to the reader with the storage series to do it against, which the +//! runbook names. See the suite entry for which write each stream makes. +//! +//! ## The control is the baseline, not another pod +//! +//! S1 and S3 keep executors on the healthy side of the cut and read the verdict +//! off the disagreement between the two groups. There is no healthy side here: +//! all three executors share one store. So the comparison runs along time +//! instead, and every stream is measured against its own before-fault rate. See +//! [`crate::chaos::outage`] for what that costs and what it still answers. +//! +//! The first question it has to answer is whether the outage landed at all. A +//! partition that failed to take hold produces a report full of healthy numbers +//! and no error anywhere, which is the worst artifact this suite can produce, +//! so [`OutageViolation::OutageNotObserved`] is a named finding rather than +//! something a reader is left to infer from the cells. +//! +//! ## Why S18 is held longer than the others +//! +//! The first three scenarios are sized against a platform budget: S16 and S14 +//! stay inside one, S22 deliberately runs past it. S18 has no budget to run +//! past. golem-dev sets +//! `GOLEM__KEY_VALUE_STORAGE__CONFIG__CACHE__CONFIG__RETRIES__MAX_ATTEMPTS` to +//! `0`, which `RedisPool::configured` hands to fred as a `ReconnectPolicy` with +//! unlimited attempts, and it passes no performance or connection config, so +//! fred's defaults apply: no command timeout and an unbounded command buffer. +//! A cache write during the cut therefore never fails. It waits, for as long as +//! the cut lasts. +//! +//! That makes the `unwrap_or_else(|err| panic!(...))` on the cache path in +//! `WorkerService` unreachable through this fault, and it makes a short cut +//! uninformative: at 60s every caller is still inside its 120s attempt timeout, +//! so the run would only show operations taking a minute longer and completing. +//! The window is set past that timeout instead, so callers give up and retry +//! under the same idempotency key while the original write is still sitting in +//! fred's buffer. Whether those two land as one operation or two is the +//! question the scenario exists to answer, and the exactly-once account is +//! where it shows. +//! +//! ## What is expected to stall, and what is not +//! +//! Less than the whole platform, and this is the part worth reading the result +//! carefully for. `AgentStatusFlusher` took the status blob off the commit path +//! deliberately: a status change only marks the agent dirty, and a background +//! sweeper coalesces the writes. So a durable agent that is already resident +//! commits to the oplog without touching Redis synchronously, and the blob it +//! cannot flush is derivable from the oplog anyway, which is what makes the +//! staleness safe rather than merely tolerated. +//! +//! What does cross Redis synchronously is a lifecycle boundary — suspend, evict, +//! reattach — and a `get_agent_mode` miss. Run 33130077355 settled which +//! streams that amounts to: `ephemeral` alone. Its agents are created and torn +//! down per operation, so every one crosses a boundary, and it was silent for +//! 99.997% of the window while `durable`, `scheduled` and `promise` held +//! 99.94–100.06% of their baselines. +//! +//! `promise` was expected to be the second and is not. The mixed workload's +//! promise stream is `get_promise+complete` in one round trip against a durable +//! agent and never suspends; the suspending variant is `promise-wait`, which +//! S11 drives and these scenarios do not enable. +//! +//! That partial shape is why S18 does not share the other three scenarios' +//! verdict. `shareOfBaselinePercent` sits far higher here — 77.83% against +//! S14's 22.39% — without the fault being weaker, and the run-wide quiet figure +//! the other three are judged on reads 0.05%, because `durable` never stopped. +//! Judged by that rule the run reported a partition that had plainly landed as +//! one that never happened. See [`crate::chaos::OutageExpectation`] for the +//! split, and for why the streams that keep working are asserted on rather than +//! merely exempted. +//! +//! ## Why the scheduled stream is driven separately +//! +//! The mixed workload's scheduled stream registers through `schedule_poll_at`, +//! which increments a counter and records nothing else. That is enough to ask +//! "did every registration eventually fire" and useless for asking "how late". +//! Scheduler lag is one of the two things GOL-379 asks these runs to record, +//! and the only place a due time survives is the target's own fire log, so +//! these scenarios drive the token-carrying registration loop from +//! [`crate::chaos::scheduled`] instead and leave `scheduledAgents` at zero in +//! the mixed workload. Setting both is refused at load time — see +//! `ScenarioConfig::require_storage`. +//! +//! An entry may also leave the loop undriven altogether by setting +//! `scheduled.targets` to zero, which is what the S15 eliminations do. The +//! fire-count gate then has nothing to gate, and is skipped rather than failing +//! a run for observing no fires it never asked for. +//! +//! S14 is where that lag reads most directly. Its cut leaves the scheduler on +//! the healthy cluster, so claims and acknowledgements keep their timing and +//! what the delays measure is purely how long the fired invocation could not +//! commit. Under S16 and S22 the scheduler is inside the outage and the two +//! costs are not separable. +//! +//! One consequence worth stating plainly for anyone reading the result: the +//! fire account's `group` axis is degenerate for the ten single-fault codes. +//! None of them names a pod to kill, so every target is reported as `elsewhere` +//! and only the `window` axis carries information. The delays to read are the +//! `during-fault` and `after-fault` cells against `before-fault`. +//! +//! MF1 is the exception and the reason the axis exists here at all. Its kill is +//! aimed at the executor owning the largest share of the targets, so the +//! `on-pod` group is the population whose actions had to be recovered by a +//! survivor that could not read the scheduler's schema, and the `elsewhere` +//! group is the control that was only ever inside the outage. +//! +//! ## What fails the run +//! +//! The same narrow set S3 fails on, and for the same reason: this suite reports +//! rather than judges, and the bar for failing outright is "the run produced +//! nothing worth interpreting". +//! +//! * A token-level scheduled-fire violation. An accepted action that never ran, +//! ran twice, or ran after being refused is a statement about one named +//! registration with no band of doubt around it. +//! * A key that the exactly-once probe shows executed twice. +//! * A workload that never confirmed anything at all. +//! +//! Everything the storage account finds is loud without being fatal: it lands +//! in `attention`, which CI annotates. An outage that did not land and a stream +//! that never came back are both things a human has to look at, and neither is +//! improved by turning the job red. +//! +//! Run-by-run findings live in the per-scenario runbooks in golem-cloud, not +//! here. + +use crate::chaos::composed::ComposedFaultReport; +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::outage::StorageFaultReport; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_counters, readback_for, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::scheduled::{self, ScheduledSelection}; +use crate::chaos::signal::{BaselineReady, FaultInjected, FaultSignals, FaultTarget}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the last scheduled action is due, before anything is read +/// back. +/// +/// The rest of the settle is derived from the configuration, the same way S10 +/// derives it: the final registration falls due one `lead` after the workload +/// stops, and an action the outage delayed can cost up to one lease budget on +/// top. Reading before that has elapsed would report actions as lost that were +/// merely late, which is the one mistake this scenario cannot afford — a +/// storage outage is precisely the thing that makes work late. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How long past the enclosing fault window a composed run keeps waiting for +/// its second fault. +/// +/// The workflow injects the second fault at a fraction of the window and then +/// waits for Chaos Mesh to report it active, which is not instant. Without this +/// margin a slow injection near the end of the window would be recorded as one +/// that never happened, which is a worse mistake than the one it describes: the +/// run would look like a broken harness rather than like a composition that +/// landed too late to mean anything. +/// +/// Sized against the workflow's own worst case rather than picked round. It +/// gives Chaos Mesh 120s to confirm the injection, so a signal can appear as +/// late as the injection fraction plus 120s. At MF1's half-way fraction that is +/// the whole window plus 60s, and a margin equal to that would be a tie. +/// Doubling it leaves the driver certain to see a signal the workflow actually +/// wrote, which matters because `secondary-outside-primary` tells a reader far +/// more than `secondary-never-injected` does. +const SECONDARY_WAIT_MARGIN: Duration = Duration::from_secs(120); + +/// How long after the second fault lands before the shard assignment is sampled +/// a second time. +/// +/// The first sample is taken the instant Chaos Mesh confirms the kill, which is +/// before anything can have reacted to it: the shard-manager has not noticed the +/// pod is gone, so that sample is the assignment as it stood at the kill. Useful +/// as a reference, useless as an observation. +/// +/// What MF1 is actually for shows up in the second one — shards revoked from the +/// dead executor, and a survivor that cannot complete the assignment because the +/// running-workers set it has to read is behind the cut. The shard-manager gives +/// `assign_shards` 5s and retries it 5 times, so the interesting state exists for +/// tens of seconds and this has to land inside them. +const REASSIGNMENT_SETTLE: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A smoke test rather than a measurement: if the scheduling path is broken +/// every target is equally broken, and the point is to fail before spending the +/// fault window on a run that would report a clean account of nothing. +const FIRE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + code: ScenarioCode, + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let scheduled_config = config.require_scheduled()?; + let storage_config = config.require_storage()?; + // Present only for the `MF` codes. Fetched through the checked accessor so + // a malformed block fails here rather than half way through the window. + let composed_config = match config.composed { + Some(_) => Some(config.require_composed()?), + None => None, + }; + let history = OperationHistory::new(code.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(code); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + // All four are empty for the ten single-fault codes and stay out of the + // result entirely, rather than appearing as empty accounts that would read + // as "checked, nothing found". + let mut ownership_samples: Vec = Vec::new(); + let mut selection: Option = None; + let mut secondary: Option = None; + let mut composed_report: Option = None; + + // Every early return below goes through `finish`, so an abort produces the + // same artifact shape as a completed run, with fewer phases filled in. + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr, $outage:expr, $exactly:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if !ownership_samples.is_empty() { + summary = summary.with_ownership(ownership_samples.clone()); + } + if let Some(report) = composed_report.clone() { + summary = summary.with_composed_fault(report); + } + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + if let Some(report) = $outage { + summary = summary.with_storage_fault(report); + } + if let Some(report) = $exactly { + summary = summary.with_exactly_once(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: selection.clone(), + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "{code}: warming {} schedule emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("{code}: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim the second fault ──────────────────────────────────────────────── + // + // Only the composed codes have one. The enclosing fault is aimed at every + // executor and needs nothing chosen, but the kill inside it does, and the + // choice has to be made before the baseline for the same reason S10 makes + // it there: a run that cannot be aimed should not spend a maintenance + // window proving it. + // + // Aimed at the executor owning the largest share of the schedule targets, + // which is exactly what S10 aims at. That is worth more here than anywhere + // else in this module: the fire account already splits its targets into the + // ones on the killed executor and the ones elsewhere, and in the ten + // single-fault codes that axis is degenerate because nothing is killed. A + // composed run is the first one that fills it in, and the survivors are its + // control group for a scheduler that cannot reach its own schema. + if composed_config.is_some() { + let chosen = match scheduled::select(&ctx, deps, &targets).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("{code}: could not aim the second fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "{code}: second fault aimed at {} ({} of {} targets on it)", + chosen.pod_address, + chosen.on_pod.len(), + targets.len() + ); + selection = Some(chosen); + } + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "{code}: baseline phase, mixed workload at {} ops/s plus {} schedule targets, for {:?}", + workload_config.rate_per_sec, + targets.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let mixed = workload::start(ctx.clone(), workload_config); + let schedules = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + // Only for the composed codes. The single-fault ones kill nothing, so every + // sample would report the same assignment and the findings that only exist + // between two samples would never fire. + if composed_config.is_some() { + ownership_samples.push(sample_ownership(deps, "before-fault", None, true).await); + } + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + // Both handles have to be stopped on every exit path, and forgetting one + // leaves its emitters submitting into a history that has already been + // snapshotted. + macro_rules! stop_workloads { + () => {{ + mixed.stop().await; + schedules.stop().await; + }}; + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + // Taking a database away from a workload that never worked would + // measure nothing. Stop before touching the cluster. + warn!("{code}: baseline produced no confirmed operations, aborting before injection"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + // Registering is not firing. The scheduler is one of the three mechanisms + // this scenario is about, and a platform that accepted every registration + // and ran none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never worked. + // + // Only when the run drives the stream at all. A scenario that registers + // nothing has no fire to observe, so holding it to this gate aborts it + // during the baseline and before the fault is ever injected. That is what + // happened to S15A's first run, and it cost a cluster run to learn. + let drives_scheduled = config.drives_stream(Stream::Scheduled); + let sampled = if drives_scheduled { + sample_fire_count(code, &ctx, &targets).await + } else { + 0 + }; + if drives_scheduled && sampled == 0 { + warn!( + "{code}: {baseline_operations} operations confirmed and no scheduled action has fired" + ); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + let fires = if drives_scheduled { + format!( + "{sampled} fires across a sample of {} targets", + FIRE_PROOF_SAMPLE.min(targets.len()) + ) + } else { + "scheduled stream not driven".to_string() + }; + info!( + "{code}: baseline complete ({baseline_operations} confirmed ops, {fires}), signalling readiness" + ); + + // ── Signal: ready for the fault ───────────────────────────────────────── + // + // A single-fault code has nothing to aim: the partition is between every + // executor and one endpoint outside the cluster, so there is no pod for the + // driver to choose and no ownership to verify. A composed code names the + // executor its kill has to hit, and re-checks the division first — the + // baseline has just run for five minutes, and a selection made before it is + // a claim about a cluster that has had time to move underneath it. + let mut fault_target = None; + if let Some(chosen) = &selection { + if let Err(e) = scheduled::verify_ownership(&ctx, deps, chosen).await { + warn!("{code}: target ownership no longer holds, refusing to inject: {e:#}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + fault_target = Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }); + } + signals.write_baseline_ready(&BaselineReady { + scenario_code: code.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("{code}: no fault-injected signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "{code}: fault {} ({} on {}) reported active at {}, {} is now unreachable from the executors", + injected.fault_id, + injected.kind, + injected.target, + injected.injected_at, + storage_config.endpoint + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // ── The second fault ──────────────────────────────────────────────────── + // + // Bounded by the enclosing window rather than by the generous signal + // timeout every other wait here uses. A second fault that has not landed by + // the time the first one heals is never going to land inside it, and a + // driver still blocked on it would sleep through the heal it measures + // recovery from. + // + // A wait that runs out is not an abort. The run still has an enclosing + // fault, a workload and every account below this one; what it does not have + // is the composition, and the report says exactly that rather than the run + // ending with nothing. + if composed_config.is_some() { + let deadline = config.phases.fault() + SECONDARY_WAIT_MARGIN; + match signals.await_secondary_fault(deadline).await { + Ok(signal) => { + info!( + "{code}: second fault {} ({} on {}) reported active at {}", + signal.fault_id, signal.kind, signal.target, signal.injected_at + ); + ownership_samples.push( + sample_ownership(deps, "after-kill", ownership_samples.last(), false).await, + ); + routing_snapshots.push(snapshot_routing(deps, "after-kill").await); + secondary = Some(signal); + + // Then again once the shard-manager has had time to react. + // Capped at half of what is left of the window, so the sample + // cannot land after the heal and describe a cluster that had + // its storage back — which would report the reassignment as + // having worked all along. + if let Some(composed) = composed_config { + let remaining = config + .phases + .fault() + .mul_f64((1.0 - composed.after_fraction).max(0.0)); + let settle = REASSIGNMENT_SETTLE.min(remaining / 2); + info!("{code}: sampling shard assignment again in {settle:?}"); + tokio::time::sleep(settle).await; + ownership_samples.push( + sample_ownership(deps, "during-fault", ownership_samples.last(), false) + .await, + ); + routing_snapshots.push(snapshot_routing(deps, "during-fault").await); + } + } + Err(e) => { + warn!("{code}: no secondary-fault signal arrived within {deadline:?}: {e}"); + } + } + } + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("{code}: no fault-recovered signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "{code}: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // Built here rather than at the end, so an abort during recovery or + // read-back still says whether the two faults ever met. Every account below + // this point describes a cluster that was under both of them, and a reader + // who cannot tell that from a reader who cannot tell the difference is the + // failure this report exists to prevent. + if let Some(composed) = composed_config { + let report = ComposedFaultReport::build( + &injected, + fault_recovered_at, + secondary.as_ref(), + composed.min_overlap(), + ); + for finding in &report.findings { + warn!("{code}: {}: {}", finding.violation, finding.detail); + } + composed_report = Some(report); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "{code}: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + // A composed run takes one more sample early in the recovery phase, and + // MF1's first run is why. It left the fault window with 564 of 1024 shards + // assigned to nobody and reached the end of a ten-minute recovery phase with + // all 1024 assigned again, so the one thing the run most wanted to say — how + // long after the storage came back the handover completed — was somewhere + // inside a ten-minute gap between two samples. Waiting the same settle the + // during-fault sample uses turns that into an answer or a bound. + if composed_config.is_some() { + let settle = REASSIGNMENT_SETTLE.min(config.phases.recovery()); + tokio::time::sleep(settle).await; + ownership_samples + .push(sample_ownership(deps, "after-heal", ownership_samples.last(), false).await); + routing_snapshots.push(snapshot_routing(deps, "after-heal").await); + tokio::time::sleep(config.phases.recovery() - settle).await; + } else { + tokio::time::sleep(config.phases.recovery()).await; + } + + let skipped = schedules.skipped(); + // Stopping waits for in-flight operations to record themselves rather than + // cancelling them: an operation cancelled mid-flight is one the history + // cannot classify, and during a storage outage those are exactly the + // interesting ones. + stop_workloads!(); + + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "{code} skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase counts \ + understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + if composed_config.is_some() { + ownership_samples + .push(sample_ownership(deps, "after-recovery", ownership_samples.first(), true).await); + } + + // ── Read-back ─────────────────────────────────────────────────────────── + let settle = settle_before_readback(scheduled_config); + info!("{code}: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report: the + // reduction is the part a later ticket is most likely to want to redo. + history.record_fire_logs(logs.clone()); + + let fault_window = fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }); + + let killed_targets: BTreeSet = selection + .as_ref() + .map(|chosen| chosen.on_pod.iter().cloned().collect()) + .unwrap_or_default(); + let fires = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_window, + // Empty for the ten single-fault codes: nothing is killed, so every + // target belongs to the report's `elsewhere` group and only its window + // axis carries meaning. A composed run fills it in with the targets the + // killed executor owned, which is what turns the survivors into a + // control group. + &killed_targets, + scheduled_config.lease_budget(), + ); + info!( + "{code}: scheduled-fire account — {} registrations accepted, {} fired once, {} \ + inconclusive, {} unverifiable, {} findings", + fires.registrations_confirmed, + fires.fired_once, + fires.inconclusive, + fires.unverifiable, + fires.findings.len() + ); + + let outage = StorageFaultReport::build( + &records, + fault_window, + &storage_config.endpoint, + storage_config.expect.clone(), + storage_config.recovery_budget(), + ); + info!( + "{code}: storage account — the streams expected to stop were silent for at least {:?}% of \ + the fault window (floor {}%) and the streams expected to carry on held at least {:?}% of \ + their baseline while {} was unreachable, {} findings", + outage.quietest_stream_percent, + outage.expect.quiet_floor_percent(), + outage.least_serving_stream_percent, + outage.endpoint, + outage.findings.len() + ); + for finding in &outage.findings { + warn!("{code}: {}: {}", finding.violation, finding.detail); + } + + let readback = read_back(&ctx, &records, workload_config, &logs).await; + + // The idempotency half of the account. A key that timed out while the + // database was gone may or may not have executed; re-invoking it under the + // same key says which, because a platform that stored the result replays it + // and one that did not runs the work again. + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "{code}: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ + the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reason = termination(&fires, &exactly_once, &records); + + finish!( + reason, + &records, + readback, + Some(fires), + Some(outage), + Some(exactly_once) + ); +} + +/// Why the run stopped. +/// +/// Kept apart from [`run`] so the precedence is testable without a cluster. The +/// order matters: a token-level fire violation and a duplicated key are both +/// statements about one named thing, and either is worth more to a reader than +/// "the workload never worked", which is a statement about the run rather than +/// about the platform. +fn termination( + fires: &ScheduleFireReport, + exactly_once: &ExactlyOnceReport, + records: &[OperationRecord], +) -> TerminationReason { + if fires.has_violations() { + return TerminationReason::ScheduledFireViolated { + findings: fires.findings.len() as u64, + first: fires + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }; + } + if let Some(reason) = exactly_once_termination(exactly_once) { + return reason; + } + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + return TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + }; + } + TerminationReason::Completed +} + +/// How long to wait after the workload stops before reading anything back. +/// +/// Same derivation as S10's, and load-bearing for the same reason: the last +/// registration falls due one `lead` after the loop stops, and a delayed action +/// can cost a lease budget on top of that. +fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Reads durable state back for the streams that keep a count. +/// +/// The durable counters come from the mixed workload's own agents; the +/// scheduled counts come from the fire logs that were already read, because +/// re-reading `polls` would be a second round trip for a number the log already +/// carries. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, + logs: &[crate::chaos::history::TargetFireLog], +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for log in logs { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback.extend(readback_for( + Stream::Scheduled, + &log.agent, + scoped, + observed, + )); + } + + readback +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +async fn sample_fire_count(code: ScenarioCode, ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("{code}: could not sample fires on {target}: {e}"), + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::OutageExpectation; + use crate::chaos::history::{AttemptRecord, FireRecord, TargetFireLog}; + use crate::chaos::outage::{OutageFinding, OutageViolation}; + use chrono::{DateTime, TimeDelta}; + use test_r::test; + + const TARGET: &str = "chaos-s16-schedule-target-0000"; + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(240); + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn registration(token: &str, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: TARGET.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at: t0(), + completed_at: Some(t0() + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: t0(), + duration_ms: 20, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn log(fired: &[&str]) -> TargetFireLog { + TargetFireLog { + agent: TARGET.to_string(), + polls: Some(fired.len() as u64), + fires: fired + .iter() + .map(|token| FireRecord { + token: (*token).to_string(), + scheduled_at: t0() + TimeDelta::seconds(10), + observed_at: t0() + TimeDelta::seconds(11), + }) + .collect(), + error: None, + } + } + + fn fires(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The empty account: nothing registered, nothing fired, nothing to say. + fn clean_exactly_once() -> ExactlyOnceReport { + ExactlyOnceReport::build( + &[], + &[], + Stream::Durable, + &std::collections::BTreeMap::new(), + &std::collections::BTreeMap::new(), + ) + } + + /// A registration the platform accepted whose action never ran is a + /// statement about one named token, and it outranks every aggregate. + #[test] + fn a_token_that_never_fired_fails_the_run() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&[])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A fire violation outranks "the workload never worked", because it says + /// something about the platform and the other says something about the run. + #[test] + fn a_fire_violation_outranks_a_workload_that_never_confirmed_anything() { + // Rejected registrations mean nothing confirmed anywhere, and a fire + // that happened anyway is the sharpest thing this suite can find. + let records = vec![registration("token-a", Outcome::Rejected)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A run where nothing confirmed at all says nothing about resilience, so + /// it fails rather than reporting a clean account of a workload that never + /// worked. + #[test] + fn a_workload_that_never_confirmed_anything_fails_the_run() { + let records = vec![registration("token-a", Outcome::Indeterminate)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::StreamNeverSucceeded { .. }), + "expected a never-succeeded reason, got {reason:?}" + ); + } + + /// The healthy shape completes, and the storage account's own findings do + /// not change that: an outage that failed to land is loud in `attention` + /// and deliberately not fatal. + #[test] + fn a_healthy_run_completes() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert_eq!(reason, TerminationReason::Completed); + } + + /// The storage account's findings never reach the termination reason. An + /// outage that failed to land is the loudest thing these scenarios can + /// report, and it is deliberately not fatal: turning the job red would say + /// the platform did something wrong, and what actually went wrong is the + /// experiment. + #[test] + fn a_storage_finding_does_not_change_the_termination_reason() { + let mut outage = StorageFaultReport::build( + &[], + None, + "db.example", + OutageExpectation::WholeWorkload { + quiet_floor_percent: 15.0, + }, + Duration::from_secs(120), + ); + outage.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: None, + detail: "kept working".to_string(), + }); + assert!(outage.has_findings()); + + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + assert_eq!(reason, TerminationReason::Completed); + } +} diff --git a/integration-tests/src/chaos/scheduled.rs b/integration-tests/src/chaos/scheduled.rs new file mode 100644 index 0000000000..176f4b791b --- /dev/null +++ b/integration-tests/src/chaos/scheduled.rs @@ -0,0 +1,476 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-registration workload (GOL-378). +//! +//! The mixed workload's scheduled stream registers a poll and counts the fires. +//! This one registers a `fire` carrying the registration's own idempotency key, +//! which is what lets [`crate::chaos::fires`] pair an individual action against +//! the invocation that asked for it. +//! +//! ## Why the cadence, and why the lead +//! +//! `lead x targets / interval` is the population that matters: actions the +//! platform has accepted and not yet run. At 100 targets registering every two +//! seconds, ten seconds ahead, that is five hundred of them standing at any +//! instant, spread across both executors' shards. A kill lands in the middle of +//! that population by construction, which is what makes the cadence and the +//! lead the two numbers that decide whether the run measures anything. +//! +//! The lead also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! action registered before the kill would already have fired by the time the +//! pod died. See [`crate::chaos::scenarios::s10`] for why the narrower +//! claim-to-acknowledge window is not something the driver tries to aim at. +//! +//! ## Why the target set is split rather than pinned +//! +//! [`crate::chaos::pinned`] drives *only* the agents its chosen executor owns, +//! because an S8 operation that was not on the dead pod says nothing. Here the +//! opposite is true: the actions on the surviving executor are the control +//! group. Every target is driven, the driver names the executor owning the +//! largest share, and the report splits the two so a lease recovery that took +//! its full TTL cannot hide behind the half of the population that was never +//! disturbed. + +use crate::chaos::ScheduledConfig; +use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; +use crate::chaos::workload::{self, SCHEDULE_EMITTER_AGENT, WorkloadContext}; +use chrono::{DateTime, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Registrations one target may have in flight at once. +/// +/// Small on purpose. The cadence is what sets the rate; this only keeps a +/// stalled platform from accumulating tasks, and a target that has spent its +/// budget skips its tick and says so rather than queueing behind itself. +pub const MAX_IN_FLIGHT_PER_TARGET: usize = 8; + +/// How many targets are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking them one at a time behind a per-read ceiling outlasts the +/// maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// The executor S10 aims at, and how its targets divide around it. +/// +/// An alias rather than a type of its own: S11 splits its waiters exactly the +/// same way, the logic lives in [`crate::chaos::split`], and the name is kept +/// here because `scheduledSelection` is the key the archived result and the +/// golem-cloud report both read. +pub type ScheduledSelection = crate::chaos::split::PodSplit; + +/// Chooses the executor to aim at. See [`crate::chaos::split::select`]. +pub async fn select( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + targets: &[String], +) -> anyhow::Result { + crate::chaos::split::select(crate::chaos::split::schedule_subject(ctx), deps, targets).await +} + +/// Re-checks the division immediately before injection. See +/// [`crate::chaos::split::verify_ownership`]. +pub async fn verify_ownership( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + selection: &ScheduledSelection, +) -> anyhow::Result<()> { + crate::chaos::split::verify_ownership( + crate::chaos::split::schedule_subject(ctx), + deps, + selection, + ) + .await +} + +/// Extra quiet after the last registration's action is due, before the fire +/// logs are read. +/// +/// The rest of the settle is derived from the configuration rather than fixed: +/// the final registration falls due one `lead` after the workload stops, and if +/// its target's executor was faulted while holding the claim, the recovery costs +/// up to one lease budget on top. Reading before that elapsed would report +/// actions as lost that were merely late, which is the one mistake a fire +/// account cannot afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// scheduling path is broken, every target is equally broken, and the point is +/// to fail before spending the fault window on a run that would report a clean +/// account of nothing. +pub const FIRE_PROOF_SAMPLE: usize = 5; + +/// How long to wait after the workload stops before reading the fire logs. +pub fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +/// +/// Registering is not firing. A platform that accepted every registration and +/// scheduled none of them would otherwise reach read-back and report a flawless +/// account of a mechanism that never ran. +pub async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match crate::chaos::workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("could not sample fires on {target}: {e}"), + } + } + total +} + +/// A running registration workload. As elsewhere, dropping the handle does not +/// stop it: call [`ScheduledHandle::stop`] so in-flight registrations record +/// themselves instead of being cancelled mid-flight. +pub struct ScheduledHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + skipped: Arc, +} + +impl ScheduledHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Ticks a target dropped because it already had its budget of + /// registrations in flight. Non-zero means the platform was slow enough to + /// clamp the cadence, which is context for reading everything else. + pub fn skipped(&self) -> u64 { + self.skipped.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos scheduled workload stopped after {} registrations ({} ticks skipped)", + self.submitted(), + self.skipped() + ); + } +} + +/// Starts one registration loop per target. +pub fn start( + ctx: WorkloadContext, + targets: &[String], + config: &ScheduledConfig, +) -> ScheduledHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let skipped = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + let interval = config.interval(); + let lead = config.lead(); + let count = targets.len().max(1); + + info!( + "Chaos scheduled workload starting: {} targets, one registration every {:?} each \ + ({:.1}/s overall), {:?} ahead", + targets.len(), + interval, + targets.len() as f64 / interval.as_secs_f64(), + lead + ); + + for (index, target) in targets.iter().enumerate() { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let skipped = skipped.clone(); + let target = target.clone(); + // Spread the loops across one interval so the whole population does not + // register in the same instant, which would make the offered rate a + // sawtooth instead of the constant the phase stats assume. + let stagger = interval.mul_f64(index as f64 / count as f64); + + tasks.spawn(async move { + tokio::time::sleep(stagger).await; + let budget = Arc::new(Semaphore::new(MAX_IN_FLIGHT_PER_TARGET)); + let mut in_flight = JoinSet::new(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut seq = 0u64; + + while stop.load(Ordering::Relaxed) == 0 { + ticker.tick().await; + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let Ok(permit) = budget.clone().try_acquire_owned() else { + skipped.fetch_add(1, Ordering::Relaxed); + continue; + }; + submitted.fetch_add(1, Ordering::Relaxed); + let ctx = ctx.clone(); + let target = target.clone(); + let this_seq = seq; + seq += 1; + in_flight.spawn(async move { + let _permit = permit; + register_one(&ctx, index as u32, &target, this_seq, lead).await; + }); + while in_flight.try_join_next().is_some() {} + } + + // Drain rather than cancel: a registration cancelled mid-flight is + // one the history cannot classify, and during a fault those are the + // interesting ones. + while in_flight.join_next().await.is_some() {} + }); + } + + ScheduledHandle { + stop, + tasks, + submitted, + skipped, + } +} + +/// Registers one action, `lead` from now, under a key the fire will carry back. +async fn register_one(ctx: &WorkloadContext, index: u32, target: &str, seq: u64, lead: Duration) { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let key = ctx.idempotency_key(target, seq); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter); + + // Computed once, outside the retry, so a retried registration asks for the + // same instant. A due time that moved with each attempt would make the fire + // log's `scheduledMillis` disagree with what the driver believes it asked + // for, and the delay measurement would be against the wrong baseline. + let fire_at = SystemTime::now() + lead; + let since_epoch = fire_at.duration_since(UNIX_EPOCH).unwrap_or_default(); + let (secs, nanos) = (since_epoch.as_secs(), since_epoch.subsec_nanos()); + + let ctx2 = ctx.clone(); + let target2 = target.to_string(); + // Recorded against the target rather than the emitter, because the target is + // where the fire lands and where the read-back looks. + workload::run_operation( + ctx, + Stream::Scheduled, + target.to_string(), + "schedule_fire_at", + key, + |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + let target = target2.clone(); + async move { + let token = k.value.clone(); + ctx.user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "schedule_fire_at", + data_value!(target, secs, nanos, token), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Creates every emitter and target before the baseline starts. +/// +/// Returns how many agents were touched. Residency matters here for the same +/// reason it does in S8: an agent that has to cold-start on first use would put +/// start-up cost inside the baseline the recovery is measured against. +pub async fn warm(ctx: &WorkloadContext, targets: &[String]) -> usize { + let mut warmed = 0usize; + for (offset, chunk) in targets.chunks(READ_CONCURRENCY).enumerate() { + let mut batch = JoinSet::new(); + for (position, target) in chunk.iter().cloned().enumerate() { + let ctx = ctx.clone(); + let index = (offset * READ_CONCURRENCY + position) as u32; + batch.spawn(async move { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter.clone()); + if let Err(e) = ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "warm", data_value!()) + .await + { + warn!("S10: could not warm emitter {emitter}: {e:#}"); + } + // Reading the target creates it without mutating it, which is + // exactly the side effect wanted. + if let Err(e) = workload::read_polls(&ctx, &target).await { + warn!("S10: could not warm target {target}: {e}"); + } + }); + } + while batch.join_next().await.is_some() { + warmed += 1; + } + } + warmed +} + +/// Reads every target's fire log back. +/// +/// A failed read is carried as [`TargetFireLog::error`] rather than dropped: an +/// agent that could not be read says nothing either way about its actions, and +/// the account has to be able to tell that apart from an agent that lost them. +pub async fn read_logs(ctx: &WorkloadContext, targets: &[String]) -> Vec { + let total = targets.len(); + let mut logs = Vec::with_capacity(total); + + for chunk in targets.chunks(READ_CONCURRENCY) { + let mut batch = JoinSet::new(); + for target in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let polls = workload::read_polls(&ctx, &target).await.ok(); + let fires = workload::read_fires(&ctx, &target).await; + match fires { + Ok(raw) => TargetFireLog { + agent: target, + polls, + fires: to_fire_records(&raw), + error: None, + }, + Err(e) => TargetFireLog { + agent: target, + polls, + fires: Vec::new(), + error: Some(e), + }, + } + }); + } + + let mut batch_results = Vec::new(); + while let Some(joined) = batch.join_next().await { + match joined { + Ok(log) => batch_results.push(log), + Err(e) => warn!("S10: a fire-log read task panicked: {e}"), + } + } + batch_results.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs.extend(batch_results); + info!("S10: read fire logs for {} of {total} targets", logs.len()); + } + + logs +} + +/// Turns the agent's raw triples into timestamps. +/// +/// An entry whose millis cannot be a timestamp is dropped rather than clamped. +/// Dropping leaves the log shorter than the target's own `polls`, which is what +/// the account reads as "this target cannot testify" — a clamped nonsense +/// timestamp would instead have produced a confident wrong delay. +pub fn to_fire_records(raw: &[(String, u64, u64)]) -> Vec { + let mut out = Vec::with_capacity(raw.len()); + for (token, scheduled_millis, observed_millis) in raw { + match ( + millis_to_time(*scheduled_millis), + millis_to_time(*observed_millis), + ) { + (Some(scheduled_at), Some(observed_at)) => out.push(FireRecord { + token: token.clone(), + scheduled_at, + observed_at, + }), + _ => warn!( + "S10: dropping fire log entry {token} with unreadable timestamps \ + ({scheduled_millis}, {observed_millis})" + ), + } + } + out +} + +fn millis_to_time(millis: u64) -> Option> { + DateTime::from_timestamp_millis(i64::try_from(millis).ok()?) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The happy path of the conversion the whole account is built on. + #[test] + fn raw_triples_become_timestamped_fire_records() { + let raw = vec![("t-0".to_string(), 1_800_000_000_000, 1_800_000_000_450)]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "t-0"); + assert_eq!(records[0].delay_ms(), 450); + } + + /// An unreadable entry has to shorten the log rather than become a + /// confident wrong delay: the shortfall against `polls` is what tells the + /// account this target cannot testify. + #[test] + fn an_entry_with_an_impossible_timestamp_is_dropped_rather_than_clamped() { + let raw = vec![ + ("good".to_string(), 1_800_000_000_000, 1_800_000_000_010), + ("bad".to_string(), u64::MAX, 1_800_000_000_010), + ]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "good"); + + let log = TargetFireLog { + agent: "target-0".to_string(), + polls: Some(2), + fires: records, + error: None, + }; + assert!( + !log.is_complete(), + "a dropped entry must leave the log short of polls" + ); + } + + /// The last registration falls due one lead after the workload stops, and a + /// recovery costs up to a lease on top. Reading before that would report + /// late actions as lost. + #[test] + fn the_settle_covers_a_full_lead_plus_a_full_lease_recovery() { + let config = ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 45, + }; + assert_eq!( + settle_before_readback(&config), + Duration::from_secs(10 + 45) + SETTLE_MARGIN + ); + } +} diff --git a/integration-tests/src/chaos/signal.rs b/integration-tests/src/chaos/signal.rs index b5bd78e28d..ff858290f3 100644 --- a/integration-tests/src/chaos/signal.rs +++ b/integration-tests/src/chaos/signal.rs @@ -22,6 +22,7 @@ //! | -- | -- | -- | //! | `baseline-ready.json` | driver | Baseline workload is at steady state — safe to inject | //! | `fault-injected.json` | workflow | The fault is applied *and verified active* | +//! | `secondary-fault.json` | workflow | A composed scenario's second fault is applied and verified active | //! | `fault-recovered.json` | workflow | The fault is removed and the target is healthy again | //! //! Keeping the driver on this side of the line is what lets the same scenario @@ -53,6 +54,15 @@ pub const EXECUTORS_SCALED_FILE: &str = "executors-scaled.json"; /// schedule. Republished atomically, so a driver reading it mid-schedule always /// sees a complete list rather than a tail it caught halfway. pub const EXECUTOR_RESTARTS_FILE: &str = "executor-restarts.json"; +/// File name the workflow writes once the *second* fault of a composed +/// scenario is applied and verified active. Optional: a scenario that composes +/// nothing never sees it. +/// +/// A file of its own rather than a second entry in `fault-injected.json`, +/// because the driver is already waiting on that one when the second fault is +/// due, and rewriting a signal another party is polling is how a reader ends up +/// with half of each. +pub const SECONDARY_FAULT_FILE: &str = "secondary-fault.json"; /// File name the workflow writes once the fault is removed and the target is /// healthy again. pub const FAULT_RECOVERED_FILE: &str = "fault-recovered.json"; @@ -263,6 +273,21 @@ impl FaultSignals { } } + /// Blocks until the workflow reports the second fault of a composed + /// scenario active, or `timeout` elapses. + /// + /// The timeout is the scenario's own fault window rather than the generous + /// signal timeout the other waits use. A second fault that has not landed + /// by the time the first one heals is never going to, and a driver still + /// blocked on it would miss the heal it is supposed to be measuring + /// recovery from. + pub async fn await_secondary_fault( + &self, + timeout: Duration, + ) -> Result { + self.await_file(SECONDARY_FAULT_FILE, timeout).await + } + /// Blocks until the workflow reports the fault cleared, or `timeout` elapses. pub async fn await_fault_recovered( &self, diff --git a/integration-tests/src/chaos/skew.rs b/integration-tests/src/chaos/skew.rs new file mode 100644 index 0000000000..36ca142cea --- /dev/null +++ b/integration-tests/src/chaos/skew.rs @@ -0,0 +1,896 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The clock-skew account (GOL-383). +//! +//! S19 moves one executor's wall clock back by half a minute and asks what +//! breaks. Most of the answer is "nothing you can see", and the interesting +//! work in this module is establishing *why* that is the honest answer rather +//! than the answer of an experiment that was never wired up. +//! +//! ## Skew is invisible to anything that judges its own timestamps +//! +//! A clock that is uniformly wrong is undetectable from inside. Every +//! comparison a skewed pod makes between two of its own readings gives the +//! right answer, because the error cancels. Only a comparison between *its* +//! clock and *someone else's* can go wrong, so those comparisons are the entire +//! surface a skew can attack, and there are exactly two of them on this +//! platform. +//! +//! **The scheduler is not one of them.** An executor claims scheduled actions +//! `WHERE shard_id = ANY(...) AND available_at_ms <= now`, and shard sets are +//! disjoint, so two executors can never contend for one action however far +//! apart their clocks are. There is no lease to lose and no double fire to +//! produce: the owner simply fires late, by exactly the skew. That is why S19 +//! is aimed at the quota lease and not at the recurring schedules the ticket +//! originally named. +//! +//! **The quota lease is.** The shard-manager mints `expires_at` on its own +//! clock and the executor judges it on the executor's, in two places +//! (`quota.rs`): whether to renew, and whether the lease is already dead. That +//! asymmetry is the fault surface, and the platform already ships a fence for +//! it — `LeaseEpoch` exists so "an executor must reject operations from a stale +//! epoch". S19 asks whether the fence holds when the two clocks disagree. +//! +//! ## Why thirty seconds, and why backwards +//! +//! Not a round number picked for the ticket. On golem-dev the lease runs for +//! `60s` and the executor renews once fewer than `20s` remain, so a renewal +//! normally lands 40 seconds into a 60-second lease with 20 seconds of headroom. +//! A skew smaller than that headroom keeps every renewal inside the valid window +//! and the fault is inert. Thirty seconds overshoots it by ten, so the skewed +//! executor renews **ten seconds after its own lease has, by the granting +//! authority's clock, already expired** — and does so on every cycle, for the +//! whole window. +//! +//! Backwards, because forwards does nothing. A clock that runs fast makes the +//! executor renew *early*: more RPCs, no disagreement. +//! +//! ## Why the control group also needs quota agents +//! +//! The shard-manager's expiry is lazy. `reclaim_expired` runs only inside +//! `acquire_lease` and `renew_lease`, and `renew_lease` refreshes the caller's +//! own `expires_at` *before* it housekeeps. So a skewed executor renewing late +//! rescues its own lease and nothing ever notices. The stale lease is only +//! reclaimed if some *other* pod touches the same resource inside that +//! ten-second window. +//! +//! This is why S19 departs from the ticket's "pin recurring agents to the +//! skewed executor": the quota population has to span both executors. The +//! skewed one holds the stale lease; the healthy one is what makes the +//! disagreement real. +//! +//! ## What can actually be measured, and by whom +//! +//! The scheduled stream's fire log is worth being careful about, because it +//! looks like it measures the skew and does not. +//! +//! Fire delay is `observed - scheduled`: the driver mints the due time, the +//! target agent stamps the fire. On a skewed pod *both* the decision to fire and +//! the stamp come from the same wrong clock. An action due at `D` fires at true +//! `D + 30` and is stamped `D`, so the delay reads **zero**. The two errors +//! cancel exactly. A run that trusted this number would report a perfectly +//! punctual scheduler while every fire was half a minute late. +//! +//! Two things do see it: +//! +//! * **The oplog probe.** The driver knows when it made a call on its own +//! clock; the executor stamps the oplog entry that call produced. The +//! difference is the skew, straight, with no cancellation because the two +//! clocks belong to different machines. This is [`ClockProbe`], and it is +//! what proves the fault landed. +//! * **The recovery edge.** When the skew lifts the clock jumps forward and the +//! backlog fires with a *corrected* stamp, so those fires show a real delay +//! of about the skew. The fire report is blind during the fault and sighted +//! the moment it ends, which is why its `after-fault` cells are the ones to +//! read. +//! +//! ## What the two findings mean, and which of them fails a run +//! +//! * [`SkewViolation::ClockNeverMoved`] — the probe could not confirm the +//! offset. This one **fails the run**, as +//! `TerminationReason::FaultTargetUnverified`, because it is that reason +//! exactly: the window was spent and nothing shows the fault reached its +//! target. A run whose fault was inert has measured nothing, and recording it +//! as a pass is the failure S2 was built to avoid. +//! * [`SkewViolation::QuotaDidNotRecover`] — the quota stream did not return to +//! its own baseline once the clock was fixed. Reported, not failed, the same +//! way `relay::RelayViolation::RelayDidNotRecover` is. Losing a lease under +//! skew is a legitimate response, and how long getting it back may take is a +//! judgement rather than a constant. +//! +//! Duplicate execution and lost accepted work — the ticket's headline +//! guarantees — are not checked here. They are checked by the exactly-once, +//! scheduled-fire and read-back oracles the scenario ends with, which is the +//! right place for them: a duplicate is a duplicate whatever caused it. + +use crate::chaos::history::{OperationRecord, Stream}; +use crate::chaos::pinned::routing_agent_id; +use crate::chaos::split::{self, FaultWindow, Group, PodSplit, Window}; +use crate::chaos::workload::{QUOTA_COUNTER_AGENT, WorkloadContext}; +use chrono::{DateTime, Utc}; +use golem_common::base_model::OplogIndex; +use golem_common::model::oplog::PublicOplogEntry; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tracing::{info, warn}; + +/// Scenario code, for the lines a reader eventually sees. Only one scenario +/// moves a clock, so this is a constant rather than a field. +const SCENARIO: &str = "S19"; + +/// Ceiling on one probe's invocation and on each of the two oplog reads around +/// it. +/// +/// A probe that hangs must cost one reading, not the round. Generous, because a +/// slow answer is still an answer and the offset it carries is tens of seconds +/// wide. +const PROBE_TIMEOUT: Duration = Duration::from_secs(30); + +/// One driver-versus-executor clock reading. +/// +/// The driver notes its own clock, invokes an agent, then reads back the +/// timestamp the executor wrote on the oplog entry that invocation produced. +/// `offsetMs` is `stamped - asked`, so a pod running half a minute behind +/// reports about `-30000`. +/// +/// The reading is not exact and does not need to be: it carries the invocation's +/// own round trip, which is milliseconds against an offset of tens of seconds. +/// What it has to distinguish is "the clock moved by roughly what we asked for" +/// from "the clock never moved", and it separates those by three orders of +/// magnitude. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ClockProbe { + pub agent: String, + /// Which executor owned the agent when the split was taken. + pub group: Group, + /// Which side of the fault the reading was taken on, filled in by + /// [`build`] from `askedAt`. + /// + /// The verdict reads only the probes taken *during* the fault. Probes from + /// the baseline are archived rather than judged, and they are what + /// distinguishes a broken probe from a clock that never moved: a baseline + /// round that read cleanly and a fault round that read nothing are two very + /// different reports. + pub window: Window, + /// The driver's clock, immediately before the invocation. + pub asked_at: DateTime, + /// The executor's stamp on the oplog entry, when it could be read. + #[serde(skip_serializing_if = "Option::is_none")] + pub stamped_at: Option>, + /// `stamped_at - asked_at`, in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub offset_ms: Option, + /// Why the probe produced nothing. An unreadable probe is not an offset of + /// zero, and conflating the two would let a broken read pass as proof that + /// the fault was inert. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// What a skewed run did that it should not have. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SkewViolation { + /// The probes could not show the executor's clock had moved, so the run + /// exercised nothing. + ClockNeverMoved, + /// The quota stream never came back to its baseline after the clock was + /// corrected. + QuotaDidNotRecover, +} + +impl SkewViolation { + pub fn as_str(self) -> &'static str { + match self { + SkewViolation::ClockNeverMoved => "clock-never-moved", + SkewViolation::QuotaDidNotRecover => "quota-did-not-recover", + } + } +} + +impl std::fmt::Display for SkewViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, with the arithmetic that produced it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkewFinding { + pub violation: SkewViolation, + pub detail: String, +} + +/// One (stream, group, window) cell of the quota account. +/// +/// The table itself lives in [`crate::chaos::split`], because S4 asks the same +/// question of the same stream with a different fault. Kept as an alias rather +/// than renamed at the use sites: `skew.cells` is what archived results call +/// this, and the name is what a reader looks for. +pub use crate::chaos::split::StreamCell as SkewCell; + +/// The clock-skew account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkewReport { + /// What the suite asked Chaos Mesh for, in milliseconds. Negative for a + /// clock set behind. + pub injected_offset_ms: i64, + /// How far the measured offset may sit from the injected one before the run + /// is treated as having failed to inject anything. + pub tolerance_ms: i64, + /// Percentage of its own baseline the quota stream's post-fault p50 may + /// reach before the run is called unrecovered. + pub recovery_floor_percent: f64, + /// Every reading taken, including the ones that failed. Archived in full + /// because the whole verdict rests on them. + pub probes: Vec, + /// Median offset over the probes on the skewed executor. + #[serde(skip_serializing_if = "Option::is_none")] + pub observed_offset_ms: Option, + /// Median offset over the probes on every other executor: the reading that + /// says how much of the number above is ordinary driver-to-cluster skew + /// rather than the fault. + #[serde(skip_serializing_if = "Option::is_none")] + pub control_offset_ms: Option, + pub cells: Vec, + /// Post-fault p50 as a percentage of the baseline p50, on the skewed + /// executor's quota agents. Reported whether or not it breaches the floor. + #[serde(skip_serializing_if = "Option::is_none")] + pub quota_recovery_percent: Option, + pub findings: Vec, +} + +impl SkewReport { + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() + } + + /// One cell, or `None` when the run produced no operations for it. + pub fn cell(&self, group: Group, window: Window) -> Option<&SkewCell> { + self.cells + .iter() + .find(|c| c.group == group && c.window == window) + } + + /// What a reader has to act on. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| format!("{SCENARIO} {}: {}", f.violation.as_str(), f.detail)) + .collect() + } + + /// Context a reader needs to judge the numbers, findings or not. + /// + /// The measured offset goes here rather than into + /// [`Self::attention_lines`], on every run including the good ones. It is + /// the only line that says the experiment happened at all, and this + /// scenario is the one where a clean report and a report of nothing look + /// identical. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + + match (self.observed_offset_ms, self.control_offset_ms) { + (Some(observed), control) => { + let control = control.unwrap_or(0); + lines.push(format!( + "{SCENARIO}: the skewed executor stamped its oplog {observed}ms from the \ + driver's clock and the healthy one {control}ms, so the fault moved the \ + clock by {}ms against the {}ms asked for", + observed - control, + self.injected_offset_ms + )); + } + (None, _) => lines.push(format!( + "{SCENARIO}: no readable clock probe inside the fault window, so nothing in this \ + run says the offset was applied" + )), + } + + // The fire log is worth a standing line because its during-fault + // numbers look wrong until you know why. Both the decision to fire and + // the stamp come from the same wrong clock, so the two errors cancel + // and the delay reads zero however late the action really was. + lines.push(format!( + "{SCENARIO}: fire delay is blind during the fault — a skewed pod stamps its own late \ + fire with its own late clock — so the scheduled cost of the skew is in the \ + after-fault cells, where the backlog drains under a corrected clock" + )); + + if let Some(percent) = self.quota_recovery_percent { + lines.push(format!( + "{SCENARIO}: quota latency on the skewed executor's agents ended at {percent}% of \ + its own baseline, against a {}% floor", + self.recovery_floor_percent + )); + } + + for group in [Group::OnPod, Group::Elsewhere] { + if let (Some(before), Some(during)) = ( + self.cell(group, Window::BeforeFault), + self.cell(group, Window::DuringFault), + ) { + lines.push(format!( + "{SCENARIO}: {} quota p50 went {}ms -> {}ms across the injection ({} \ + confirmed before, {} during, {} rejected during)", + group.as_str(), + before.latency.p50_ms, + during.latency.p50_ms, + before.confirmed, + during.confirmed, + during.rejected + )); + } + } + + lines + } +} + +/// Everything the caller has to decide, kept out of the suite YAML's way. +#[derive(Debug, Clone)] +pub struct SkewInputs<'a> { + pub split: &'a PodSplit, + pub fault: Option, + pub injected_offset_ms: i64, + pub tolerance_ms: i64, + pub recovery_floor_percent: f64, + pub probes: Vec, +} + +/// Builds the account. +pub fn build(records: &[OperationRecord], inputs: SkewInputs<'_>) -> SkewReport { + let cells = split::stream_cells(records, Stream::Quota, inputs.split, inputs.fault); + + // Classified here rather than by the caller, so the probes and the cells + // answer "which side of the fault" the same way and cannot drift. + let mut probes = inputs.probes; + for probe in &mut probes { + probe.window = Window::of(probe.asked_at, inputs.fault); + } + + let observed_offset_ms = median_offset(&probes, Group::OnPod); + let control_offset_ms = median_offset(&probes, Group::Elsewhere); + + let mut report = SkewReport { + injected_offset_ms: inputs.injected_offset_ms, + tolerance_ms: inputs.tolerance_ms, + recovery_floor_percent: inputs.recovery_floor_percent, + probes, + observed_offset_ms, + control_offset_ms, + cells, + quota_recovery_percent: None, + findings: Vec::new(), + }; + + report.quota_recovery_percent = split::recovery_percent(&report.cells); + report.findings = findings(&report); + report +} + +/// Median offset over the readable probes taken inside the fault window, in one +/// group. +/// +/// A median rather than a mean because a single probe that caught a slow +/// invocation would drag an average by seconds, and rather than the extreme +/// because one unreadable-but-parsed timestamp should not decide a run. +/// +/// Restricted to the fault window because the baseline probes read ~0 by +/// construction: averaging the two rounds together would halve the measured +/// offset and put a correctly injected skew outside its own tolerance. +fn median_offset(probes: &[ClockProbe], group: Group) -> Option { + let mut offsets: Vec = probes + .iter() + .filter(|p| p.group == group && p.window == Window::DuringFault) + .filter_map(|p| p.offset_ms) + .collect(); + if offsets.is_empty() { + return None; + } + offsets.sort_unstable(); + Some(offsets[offsets.len() / 2]) +} + +fn findings(report: &SkewReport) -> Vec { + let mut findings = Vec::new(); + + // ── Did the clock actually move? ──────────────────────────────────────── + // + // The control offset is subtracted rather than ignored. The driver runs on + // a GitHub runner and the cluster on EC2; neither is guaranteed to agree + // with the other to the millisecond, and the fault is the *difference* + // between the two executors, not the absolute reading of either. + let readable = report + .probes + .iter() + .filter(|p| p.window == Window::DuringFault && p.offset_ms.is_some()) + .count(); + let taken = report + .probes + .iter() + .filter(|p| p.window == Window::DuringFault) + .count(); + match report.observed_offset_ms { + None => findings.push(SkewFinding { + violation: SkewViolation::ClockNeverMoved, + detail: format!( + "no probe on the skewed executor produced a readable timestamp inside the fault \ + window ({readable} of {taken} readable there, {} taken over the whole run), so \ + the run cannot show the {}ms offset was ever applied", + report.probes.len(), + report.injected_offset_ms + ), + }), + Some(observed) => { + let control = report.control_offset_ms.unwrap_or(0); + let measured = observed - control; + let drift = (measured - report.injected_offset_ms).abs(); + if drift > report.tolerance_ms { + findings.push(SkewFinding { + violation: SkewViolation::ClockNeverMoved, + detail: format!( + "asked for {}ms of skew and measured {}ms (skewed executor {}ms, control \ + {}ms), which is {}ms out against a {}ms tolerance", + report.injected_offset_ms, + measured, + observed, + control, + drift, + report.tolerance_ms + ), + }); + } + } + } + + // ── Did the quota stream come back? ───────────────────────────────────── + if let Some(percent) = report.quota_recovery_percent + && percent >= report.recovery_floor_percent + { + findings.push(SkewFinding { + violation: SkewViolation::QuotaDidNotRecover, + detail: format!( + "quota latency on the skewed executor's agents stood at {percent}% of its own \ + baseline after the clock was corrected, at or above the {}% floor", + report.recovery_floor_percent + ), + }); + } + + findings +} + +/// Takes one clock reading against a quota agent. +/// +/// Reads the agent's last oplog index, notes the driver's clock, invokes the +/// agent's `count` method, then reads the entries that invocation produced and +/// takes the timestamp off the first of them. That timestamp was written by the +/// executor that owns the agent, so the difference between it and the driver's +/// reading is the difference between two machines' clocks and nothing else. +/// +/// `count` rather than `reserve_and_increment` on purpose. The probe must not +/// appear in the operation history, and a reservation that did would leave the +/// quota read-back short by however many probes the run took — a lost-work +/// finding manufactured by the instrument. +pub async fn probe_clock(ctx: &WorkloadContext, agent: &str, group: Group) -> ClockProbe { + let agent_id = routing_agent_id(ctx, QUOTA_COUNTER_AGENT, agent); + + let failed = |asked_at: DateTime, error: String| ClockProbe { + agent: agent.to_string(), + group, + window: Window::Unknown, + asked_at, + stamped_at: None, + offset_ms: None, + error: Some(error), + }; + + let before = + match tokio::time::timeout(PROBE_TIMEOUT, ctx.user.get_oplog_last_index(&agent_id)).await { + Ok(Ok(index)) => index, + Ok(Err(e)) => return failed(Utc::now(), format!("reading the oplog length: {e:#}")), + Err(_) => { + return failed( + Utc::now(), + format!("reading the oplog length timed out after {PROBE_TIMEOUT:?}"), + ); + } + }; + + // Taken as late as possible, so the interval this reading has to absorb is + // the invocation alone rather than the invocation plus the read before it. + let asked_at = Utc::now(); + let parsed = golem_common::agent_id!(QUOTA_COUNTER_AGENT, agent.to_string()); + if let Err(e) = tokio::time::timeout( + PROBE_TIMEOUT, + ctx.user.invoke_and_await_agent( + &ctx.counters, + &parsed, + "count", + golem_common::data_value!(), + ), + ) + .await + .map_err(|_| anyhow::anyhow!("timed out after {PROBE_TIMEOUT:?}")) + .and_then(|inner| inner) + { + return failed(asked_at, format!("invoking the agent: {e:#}")); + } + + let entries = match tokio::time::timeout( + PROBE_TIMEOUT, + ctx.user + .get_oplog(&agent_id, OplogIndex::from_u64(before + 1)), + ) + .await + { + Ok(Ok(entries)) => entries, + Ok(Err(e)) => return failed(asked_at, format!("reading back the oplog: {e:#}")), + Err(_) => { + return failed( + asked_at, + format!("reading back the oplog timed out after {PROBE_TIMEOUT:?}"), + ); + } + }; + + let Some(stamped_at) = entries.first().and_then(|e| entry_timestamp(&e.entry)) else { + return failed( + asked_at, + format!( + "the invocation added {} oplog entries and none of them carried a readable \ + timestamp", + entries.len() + ), + ); + }; + + ClockProbe { + agent: agent.to_string(), + group, + window: Window::Unknown, + asked_at, + stamped_at: Some(stamped_at), + offset_ms: Some((stamped_at - asked_at).num_milliseconds()), + error: None, + } +} + +/// The executor's stamp on one oplog entry. +/// +/// Every variant of `PublicOplogEntry` carries a `timestamp` in its parameters +/// and the enum is tagged rather than nested, so the field sits at the top level +/// of the serialised form whatever the entry turned out to be. Going through +/// JSON rather than matching thirty variants keeps this from needing an arm per +/// entry kind, which is a list that grows. +fn entry_timestamp(entry: &PublicOplogEntry) -> Option> { + let value = serde_json::to_value(entry).ok()?; + let raw = value.get("timestamp")?.as_str()?; + DateTime::parse_from_rfc3339(raw) + .ok() + .map(|at| at.with_timezone(&Utc)) +} + +/// Takes a round of readings, `per_group` from each side of the split. +/// +/// Both sides, always. The skewed executor's reading on its own cannot say how +/// much of itself is the fault: the driver runs on a GitHub runner and the +/// cluster on EC2, and nothing makes those two agree. The healthy executor's +/// reading is what subtracts that out. +pub async fn probe_round( + ctx: &WorkloadContext, + split: &PodSplit, + per_group: u32, +) -> Vec { + let mut probes = Vec::new(); + for (group, agents) in [ + (Group::OnPod, &split.on_pod), + (Group::Elsewhere, &split.elsewhere), + ] { + for agent in agents.iter().take(per_group as usize) { + let probe = probe_clock(ctx, agent, group).await; + match (&probe.offset_ms, &probe.error) { + (Some(offset), _) => info!( + "{SCENARIO}: clock probe on {} ({}) read {offset}ms", + probe.agent, + group.as_str() + ), + (None, Some(error)) => warn!( + "{SCENARIO}: clock probe on {} ({}) failed: {error}", + probe.agent, + group.as_str() + ), + (None, None) => {} + } + probes.push(probe); + } + } + probes +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{Outcome, Phase}; + use chrono::TimeDelta; + use std::collections::BTreeMap; + use test_r::test; + + const INJECTED_MS: i64 = -30_000; + const TOLERANCE_MS: i64 = 5_000; + const RECOVERY_FLOOR: f64 = 150.0; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + /// Injected at +100s, healed at +200s, so `at(50)` is baseline, `at(150)` + /// is inside the fault and `at(250)` is after it. + fn fault() -> FaultWindow { + FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + } + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["quota-skewed".to_string()], + elsewhere: vec!["quota-healthy".to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn probe(group: Group, asked_secs: i64, offset_ms: Option) -> ClockProbe { + ClockProbe { + agent: match group { + Group::OnPod => "quota-skewed".to_string(), + Group::Elsewhere => "quota-healthy".to_string(), + }, + group, + window: Window::Unknown, + asked_at: at(asked_secs), + stamped_at: offset_ms.map(|ms| at(asked_secs) + TimeDelta::milliseconds(ms)), + offset_ms, + error: offset_ms.is_none().then(|| "unreadable".to_string()), + } + } + + fn record(agent: &str, submitted_secs: i64, duration_ms: u64) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Quota, + phase: Phase::Fault, + agent: agent.to_string(), + method: "reserve_and_increment".to_string(), + idempotency_key: format!("{agent}-{submitted_secs}"), + submitted_at: at(submitted_secs), + completed_at: Some(at(submitted_secs)), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + fn build_with(records: &[OperationRecord], probes: Vec) -> SkewReport { + let split = split(); + build( + records, + SkewInputs { + split: &split, + fault: Some(fault()), + injected_offset_ms: INJECTED_MS, + tolerance_ms: TOLERANCE_MS, + recovery_floor_percent: RECOVERY_FLOOR, + probes, + }, + ) + } + + /// The baseline round reads about zero by construction, so averaging it in + /// with the fault round would halve the measured offset and put a correctly + /// injected skew outside its own tolerance. + /// + /// This is the test that stops the scenario failing every good run. + #[test] + fn baseline_probes_are_archived_without_diluting_the_measurement() { + let report = build_with( + &[], + vec![ + probe(Group::OnPod, 50, Some(0)), + probe(Group::OnPod, 50, Some(0)), + probe(Group::OnPod, 150, Some(-30_000)), + probe(Group::OnPod, 150, Some(-30_000)), + probe(Group::Elsewhere, 150, Some(0)), + ], + ); + assert_eq!(report.observed_offset_ms, Some(-30_000)); + assert_eq!(report.probes.len(), 5, "every reading is archived"); + assert!( + report.findings.is_empty(), + "a correctly injected skew must not be reported as one that never landed, got {:?}", + report.findings + ); + } + + /// The driver runs on a GitHub runner and the cluster on EC2, and nothing + /// makes those two agree. The fault is the *difference* between the two + /// executors, so a base offset shared by both must not count as skew. + #[test] + fn a_clock_offset_shared_by_both_executors_is_not_the_fault() { + // Both pods read 4s behind the driver; only one of them is also skewed. + let report = build_with( + &[], + vec![ + probe(Group::OnPod, 150, Some(-34_000)), + probe(Group::Elsewhere, 150, Some(-4_000)), + ], + ); + assert_eq!(report.observed_offset_ms, Some(-34_000)); + assert_eq!(report.control_offset_ms, Some(-4_000)); + assert!( + report.findings.is_empty(), + "the shared -4000ms belongs to the driver, not to the fault, got {:?}", + report.findings + ); + } + + /// A run whose fault never landed looks exactly like a run whose fault + /// landed and did no harm. This is the only thing that tells them apart. + #[test] + fn a_clock_that_never_moved_fails_the_run() { + let report = build_with( + &[], + vec![ + probe(Group::OnPod, 150, Some(-40)), + probe(Group::Elsewhere, 150, Some(-35)), + ], + ); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(SkewViolation::ClockNeverMoved) + ); + } + + /// An unreadable probe is not a reading of zero. Conflating the two would + /// let a broken oplog read pass as proof that the fault was inert. + #[test] + fn an_unreadable_probe_is_not_an_offset_of_zero() { + let report = build_with( + &[], + vec![ + probe(Group::OnPod, 150, None), + probe(Group::Elsewhere, 150, Some(0)), + ], + ); + assert_eq!(report.observed_offset_ms, None); + let finding = report + .findings + .first() + .expect("an unreadable probe has to be a finding"); + assert_eq!(finding.violation, SkewViolation::ClockNeverMoved); + assert!( + finding.detail.contains("readable"), + "the detail has to say the probe could not be read, got {:?}", + finding.detail + ); + } + + /// Losing a lease under skew is legitimate; never getting it back is not. + #[test] + fn quota_latency_that_stays_high_after_the_heal_is_a_finding() { + let records = vec![ + record("quota-skewed", 50, 10), + record("quota-skewed", 150, 400), + record("quota-skewed", 250, 30), + ]; + let report = build_with( + &records, + vec![ + probe(Group::OnPod, 150, Some(-30_000)), + probe(Group::Elsewhere, 150, Some(0)), + ], + ); + assert_eq!(report.quota_recovery_percent, Some(300.0)); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(SkewViolation::QuotaDidNotRecover) + ); + } + + /// A quota stream that came back is not a finding, however much it cost + /// while the clock was wrong. + #[test] + fn quota_latency_that_returns_to_its_baseline_is_not_a_finding() { + let records = vec![ + record("quota-skewed", 50, 20), + record("quota-skewed", 150, 5_000), + record("quota-skewed", 250, 22), + ]; + let report = build_with( + &records, + vec![ + probe(Group::OnPod, 150, Some(-30_000)), + probe(Group::Elsewhere, 150, Some(0)), + ], + ); + assert_eq!(report.quota_recovery_percent, Some(110.0)); + assert!(report.findings.is_empty(), "got {:?}", report.findings); + } + + /// Agents the split never placed cannot be attributed to either side, and + /// guessing would put the fault's own damage in the control group. + #[test] + fn operations_on_agents_the_split_never_saw_land_in_no_cell() { + let report = build_with(&[record("quota-unplaced", 150, 10)], Vec::new()); + assert!(report.cells.is_empty(), "got {:?}", report.cells); + } + + /// Latency is filed by completion, not submission: an operation held across + /// the heal was paid for on the far side of it. + #[test] + fn an_operation_held_across_the_heal_costs_the_window_it_finished_in() { + let mut held = record("quota-skewed", 150, 60_000); + held.completed_at = Some(at(250)); + let report = build_with(&[held], Vec::new()); + + let during = report + .cell(Group::OnPod, Window::DuringFault) + .expect("submitted during the fault, so it is offered there"); + assert_eq!(during.submitted, 1); + assert_eq!( + during.latency.p50_ms, 0, + "the cost belongs to the window it was paid in, not the one it was offered in" + ); + + let after = report + .cell(Group::OnPod, Window::AfterFault) + .expect("completed after the heal, so its cost is filed there"); + assert_eq!(after.submitted, 0); + assert_eq!(after.latency.p50_ms, 60_000); + } + + /// The verdict is only ever taken over the fault window, so a run that + /// probed nothing there has measured nothing however many baseline + /// readings it archived. + #[test] + fn baseline_probes_alone_cannot_carry_a_run() { + let report = build_with( + &[], + vec![ + probe(Group::OnPod, 50, Some(-30_000)), + probe(Group::Elsewhere, 50, Some(0)), + ], + ); + assert_eq!(report.observed_offset_ms, None); + assert_eq!( + report.findings.first().map(|f| f.violation), + Some(SkewViolation::ClockNeverMoved) + ); + } +} diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs new file mode 100644 index 0000000000..5cd887bd1d --- /dev/null +++ b/integration-tests/src/chaos/split.rs @@ -0,0 +1,641 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Aiming a kill at one executor while still driving the agents it does not own. +//! +//! [`crate::chaos::pinned`] answers a different question. There, an operation +//! that was not on the dead pod says nothing, so the driver keeps only the +//! agents its chosen executor owns and discards the rest. Here every agent is +//! driven and the ones elsewhere are the run's own control group: on a +//! two-executor cluster roughly half the population is never touched, and +//! reporting one percentile across both would let a recovery that took its full +//! budget hide behind the half that was never disturbed. +//! +//! Both scenarios that work this way — S10's schedule targets and S11's promise +//! waiters — need exactly the same three things, which is why they live here +//! rather than being written twice: pick the executor owning the largest share, +//! refuse to proceed if that share is too small to mean anything, and re-check +//! the division immediately before the fault is injected. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::pinned::{owners_by_pod_in, pod_ip_of, routing_agent_id_in}; +use crate::chaos::summary::LatencyStats; +use crate::chaos::workload::WorkloadContext; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use golem_common::model::component::ComponentDto; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use tracing::info; + +/// The smallest share of agents one executor must own for the run to mean +/// anything, as a divisor of the population. +/// +/// A two-executor cluster splits a hashed population roughly evenly, so a +/// quarter is a floor rather than an expectation. Below it the "affected" group +/// is too small for its percentile to say anything, and a run that reported one +/// anyway would be worse than one that refused. +const MIN_SHARE_DIVISOR: usize = 4; + +/// What the split is about, for the messages a reader eventually sees. +/// +/// Carried rather than hard-coded because the failure modes here are reported to +/// an operator mid-maintenance-window, and "the most-loaded executor owns only 6 +/// of 100 agents" is a worse thing to read at 3am than the same sentence naming +/// promise waiters. +#[derive(Debug, Clone, Copy)] +pub struct Subject<'a> { + /// Scenario code, used only to prefix log lines. + pub scenario: &'a str, + /// The component the agents live in. Ownership is per agent id and an agent + /// id contains its component, so this is not cosmetic. + pub component: &'a ComponentDto, + /// Agent type, e.g. `ScheduleCounter`. + pub agent_type: &'a str, + /// Plural noun for messages, e.g. `schedule targets`. + pub noun: &'a str, +} + +/// The fault window, as the workflow reported it. +#[derive(Debug, Clone, Copy)] +pub struct FaultWindow { + pub injected_at: DateTime, + /// Absent for a run that never saw the fault clear. + pub recovered_at: Option>, +} + +/// Which side of the fault an event fell on. +/// +/// Shared rather than written per scenario because the classification is the +/// same question every time — an event before the kill, while the executor was +/// gone, or after it came back — and because the three names end up in archived +/// results that a reader compares across scenarios. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Window { + BeforeFault, + DuringFault, + AfterFault, + /// The run never learned when the fault was injected, so nothing can be + /// placed relative to it. + Unknown, +} + +impl Window { + pub fn as_str(self) -> &'static str { + match self { + Window::BeforeFault => "before-fault", + Window::DuringFault => "during-fault", + Window::AfterFault => "after-fault", + Window::Unknown => "unknown", + } + } + + /// Where `at` falls relative to the fault. + pub fn of(at: DateTime, fault: Option) -> Self { + match fault { + None => Window::Unknown, + Some(window) if at < window.injected_at => Window::BeforeFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) if at >= recovered => Window::AfterFault, + Some(_) => Window::DuringFault, + } + } +} + +impl std::fmt::Display for Window { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// When a window began, for the windows that have a fixed start. +/// +/// The fault's own boundaries come from the workflow's timestamps, which is +/// what makes them comparable across runs. The baseline has no boundary of its +/// own, so it starts at the first operation the run offered. +pub fn window_start( + window: Window, + fault: Option, + first_submitted: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(_)) => first_submitted, + (Window::DuringFault, Some(w)) => Some(w.injected_at), + (Window::AfterFault, Some(w)) => w.recovered_at, + _ => None, + } +} + +/// When a window closed. +/// +/// The mirror of [`window_start`], and needed wherever a gap has to be measured +/// against the window's own edges rather than against the first and last thing +/// that happened inside it: a stream that went silent for the whole back half +/// of a window has no later timestamp to compare against, so the edge is the +/// only thing that can show it. +pub fn window_end( + window: Window, + fault: Option, + last_completed: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(w)) => Some(w.injected_at), + (Window::DuringFault, Some(w)) => w.recovered_at.or(last_completed), + (Window::AfterFault, Some(_)) => last_completed, + _ => None, + } +} + +/// The longest stretch of a window in which nothing was answered. +/// +/// The window's own edges are the first and last comparison points, which is +/// what separates "answered steadily but slowly" from "answered nothing for two +/// minutes and then caught up in a burst". Those two produce the same count and +/// the same rate; only this tells them apart. +pub fn longest_silence_ms( + served_at: &[DateTime], + start: Option>, + end: Option>, +) -> Option { + let (start, end) = (start?, end?); + let mut marks: Vec> = served_at + .iter() + .copied() + .filter(|at| *at >= start && *at <= end) + .collect(); + marks.sort_unstable(); + marks.insert(0, start); + marks.push(end); + Some( + marks + .windows(2) + .map(|pair| (pair[1] - pair[0]).num_milliseconds().max(0) as u64) + .max() + .unwrap_or(0), + ) +} + +/// How long a window lasted, in seconds. +/// +/// The fault's own windows come from the workflow's timestamps, which is what +/// makes them comparable across runs. The two open-ended ones are bounded by +/// the workload instead: the baseline starts at the first operation offered, +/// and recovery ends at the last one that came back. +pub fn window_secs( + window: Window, + fault: Option, + first_submitted: Option>, + last_completed: Option>, +) -> f64 { + let seconds = |from: DateTime, to: DateTime| { + (to - from).num_milliseconds().max(0) as f64 / 1000.0 + }; + match (window, fault) { + (Window::BeforeFault, Some(w)) => first_submitted + .map(|first| seconds(first, w.injected_at)) + .unwrap_or(0.0), + (Window::DuringFault, Some(w)) => match (w.recovered_at, last_completed) { + (Some(recovered), _) => seconds(w.injected_at, recovered), + // A run that never saw the heal: the fault ran to whatever the last + // operation saw, which is the most that can be claimed. + (None, Some(last)) => seconds(w.injected_at, last), + (None, None) => 0.0, + }, + ( + Window::AfterFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }), + ) => last_completed + .map(|last| seconds(recovered, last)) + .unwrap_or(0.0), + _ => 0.0, + } +} + +/// Two decimal places, for the rates and shares that end up in a result. +pub fn round2(value: f64) -> f64 { + (value * 100.0).round() / 100.0 +} + +/// The executor the fault will be aimed at, and how the agents divide around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PodSplit { + /// The executor endpoint as the shard-manager names it, e.g. + /// `10.0.14.207:9000`. + pub pod_address: String, + /// Host part of the address, which is what a Kubernetes `status.podIP` + /// field selector matches. + pub pod_ip: String, + /// Agents this executor owns: the population that has to survive recovery. + pub on_pod: Vec, + /// Agents owned by any other executor: the run's own control group. + pub elsewhere: Vec, + /// How the agents spread across executors, so a run that refused to proceed + /// says whether the cluster was lopsided or the pool too small. + pub targets_per_pod: BTreeMap, + /// Shard count the routing table reported. Ownership is a hash modulo this, + /// so a selection cannot be re-derived later without it. + pub number_of_shards: usize, +} + +impl PodSplit { + /// Which group an agent belongs to, or `None` for one the selection never + /// saw. + pub fn group_of(&self, agent: &str) -> Option { + if self.on_pod.iter().any(|a| a == agent) { + Some(Group::OnPod) + } else if self.elsewhere.iter().any(|a| a == agent) { + Some(Group::Elsewhere) + } else { + None + } + } +} + +/// Which side of the kill an agent was on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Group { + /// Owned by the executor the fault was aimed at. + OnPod, + /// Owned by some other executor: the control group. + Elsewhere, +} + +impl Group { + pub fn as_str(self) -> &'static str { + match self { + Group::OnPod => "on-pod", + Group::Elsewhere => "elsewhere", + } + } +} + +/// Chooses the executor to aim at: the one owning the largest share of agents. +/// +/// Fails rather than proceeding unaimed. Chaos Mesh's `mode: one` would pick a +/// pod at random, and a run that killed an executor owning six agents out of a +/// hundred would still produce a confident-looking report about recovery. +pub async fn select( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + agents: &[String], +) -> anyhow::Result { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "reading the routing table to aim the {} fault", + subject.noun + ) + })?; + + let by_pod = owners_by_pod_in(subject.component, &table, subject.agent_type, agents); + let targets_per_pod: BTreeMap = by_pod + .iter() + .map(|(pod, xs)| (pod.clone(), xs.len())) + .collect(); + + let (pod_address, on_pod) = by_pod + .iter() + .max_by_key(|(_, agents)| agents.len()) + .map(|(pod, agents)| (pod.clone(), agents.clone())) + .ok_or_else(|| { + anyhow::anyhow!( + "routing table assigned none of the {} {} to any executor", + agents.len(), + subject.noun + ) + })?; + + let floor = (agents.len() / MIN_SHARE_DIVISOR).max(1); + if on_pod.len() < floor { + anyhow::bail!( + "the most-loaded executor owns only {} of {} {}, below the {floor} needed for its \ + share to be worth measuring: {targets_per_pod:?}", + on_pod.len(), + agents.len(), + subject.noun + ); + } + + let elsewhere: Vec = agents + .iter() + .filter(|t| !on_pod.contains(t)) + .cloned() + .collect(); + + info!( + "{}: aiming at executor {pod_address}, which owns {} of {} {} ({} elsewhere, across {} \ + executors)", + subject.scenario, + on_pod.len(), + agents.len(), + subject.noun, + elsewhere.len(), + targets_per_pod.len() + ); + + Ok(PodSplit { + pod_ip: pod_ip_of(&pod_address), + pod_address, + on_pod, + elsewhere, + targets_per_pod, + number_of_shards: table.number_of_shards.value, + }) +} + +/// Re-checks, against a freshly read routing table, that the agents are still +/// divided the way the selection says. +/// +/// Called immediately before the readiness signal, for the same reason +/// [`crate::chaos::pinned`] does it: a rebalance between selection and injection +/// would leave the run reporting a control group that was actually the affected +/// one. +pub async fn verify_ownership( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + split: &PodSplit, +) -> anyhow::Result<()> { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "re-reading the routing table to verify {} ownership", + subject.noun + ) + })?; + + let mut drifted = Vec::new(); + for agent in &split.on_pod { + let owner = table + .lookup(&routing_agent_id_in( + subject.component, + subject.agent_type, + agent, + )) + .map(|pod| pod.to_string()); + if owner.as_deref() != Some(split.pod_address.as_str()) { + drifted.push(format!( + "{agent} now owned by {}", + owner.unwrap_or_else(|| "nobody".to_string()) + )); + } + } + + if !drifted.is_empty() { + anyhow::bail!( + "{} of {} {} are no longer owned by {}: {}", + drifted.len(), + split.on_pod.len(), + subject.noun, + split.pod_address, + drifted.join(", ") + ); + } + + info!( + "{}: verified all {} {} are still owned by {}", + subject.scenario, + split.on_pod.len(), + subject.noun, + split.pod_address + ); + Ok(()) +} + +/// The counters component's schedule targets, as S10 aims at them. +/// One (stream, group, window) cell of a per-executor stream account. +/// +/// Serialised into archived results as `skew.cells`, so the field names are +/// load-bearing across runs that were written before this type had a second +/// caller. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamCell { + pub stream: Stream, + pub group: Group, + pub window: Window, + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Latency over confirmed operations only, filed by completion time. + pub latency: LatencyStats, +} + +#[derive(Default)] +struct Tally { + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + latencies: Vec, +} + +/// Tallies one stream by which executor owned the agent and which side of the +/// fault the operation fell on. +/// +/// The one table both fault-on-one-executor accounts are built from — S19's +/// clock skew and S4's DNS failure. Neither scenario's headline number is a +/// count: both are asking whether the group on the faulted pod moved away from +/// the group that was never touched, and that comparison only exists if both +/// halves are tallied the same way. +/// +/// Two decisions are baked in here rather than left to callers, so the two +/// scenarios cannot answer them differently: +/// +/// An agent the selection never saw is dropped rather than guessed at. +/// Attributing it would put the fault's own damage into the control group, +/// which is the one error that makes the comparison read backwards. +/// +/// Latency is filed by **completion**, not submission. An operation held across +/// the recovery edge was paid for on the far side of it, and filing it by +/// submission would credit the fault window with a cost the recovery bore. +pub fn stream_cells( + records: &[OperationRecord], + stream: Stream, + split: &PodSplit, + fault: Option, +) -> Vec { + let mut tallies: BTreeMap<(Group, Window), Tally> = BTreeMap::new(); + + for record in records.iter().filter(|r| r.stream == stream) { + let Some(group) = split.group_of(&record.agent) else { + continue; + }; + + let offered = tallies + .entry((group, Window::of(record.submitted_at, fault))) + .or_default(); + offered.submitted += 1; + match record.outcome { + Outcome::Confirmed => offered.confirmed += 1, + Outcome::Rejected => offered.rejected += 1, + Outcome::Indeterminate => offered.indeterminate += 1, + } + + if record.outcome == Outcome::Confirmed + && let Some(completed_at) = record.completed_at + { + tallies + .entry((group, Window::of(completed_at, fault))) + .or_default() + .latencies + .push(record.duration_ms); + } + } + + tallies + .into_iter() + .map(|((group, window), tally)| StreamCell { + stream, + group, + window, + submitted: tally.submitted, + confirmed: tally.confirmed, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + latency: LatencyStats::from_durations(tally.latencies), + }) + .collect() +} + +/// Post-fault p50 as a percentage of the baseline p50, on the faulted executor. +/// +/// `None` when either window produced no confirmed operation there, or when the +/// baseline p50 was zero — a percentage of nothing is not a recovery reading, +/// and reporting one would be worse than reporting none. +pub fn recovery_percent(cells: &[StreamCell]) -> Option { + let at = |window: Window| { + cells + .iter() + .find(|c| c.group == Group::OnPod && c.window == window) + .map(|c| c.latency.p50_ms as f64) + }; + let baseline = at(Window::BeforeFault)?; + let after = at(Window::AfterFault)?; + (baseline > 0.0).then(|| round2(100.0 * after / baseline)) +} + +pub fn schedule_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S10", + component: &ctx.counters, + agent_type: crate::chaos::workload::SCHEDULE_COUNTER_AGENT, + noun: "schedule targets", + } +} + +/// The counters component's durable agents, as S3 aims at them. +/// +/// The same agent type the mixed workload's durable stream drives, because it +/// is the same population: S3's emitters exist to pace it per agent, not to +/// invent a new kind of agent. +pub fn counter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S3", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "counter agents", + } +} + +/// The counters component's revert agents, as S7 aims at them. +pub fn revert_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S7", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "revert agents", + } +} + +/// The counters component's delete slots, as S6 aims at them. +pub fn delete_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S6", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "delete agents", + } +} + +/// The counters component's quota agents, as S19 aims at them. +/// +/// The quota stream rather than the durable one because the lease is the only +/// thing on this platform that one machine mints and another judges, and a +/// clock skew can only be seen where two clocks are compared. +pub fn quota_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S19", + component: &ctx.counters, + agent_type: crate::chaos::workload::QUOTA_COUNTER_AGENT, + noun: "quota agents", + } +} + +/// The promise component's waiters, as S11 aims at them. +pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S11", + component: &ctx.promise, + agent_type: crate::chaos::waiters::PROMISE_WAITER_AGENT, + noun: "promise waiters", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["a".to_string(), "b".to_string()], + elsewhere: vec!["c".to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + #[test] + fn a_split_places_each_agent_in_exactly_one_group() { + let split = split(); + assert_eq!(split.group_of("a"), Some(Group::OnPod)); + assert_eq!(split.group_of("c"), Some(Group::Elsewhere)); + } + + /// An agent the selection never saw is not silently counted as a control: + /// the caller has to decide what an unknown agent means, because in every + /// scenario here it means the population drifted. + #[test] + fn an_agent_outside_the_selection_belongs_to_no_group() { + assert_eq!(split().group_of("z"), None); + } +} diff --git a/integration-tests/src/chaos/steady.rs b/integration-tests/src/chaos/steady.rs new file mode 100644 index 0000000000..f2d4a1247c --- /dev/null +++ b/integration-tests/src/chaos/steady.rs @@ -0,0 +1,122 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One emitter per agent, each holding at most one operation (GOL-370). +//! +//! [`crate::chaos::workload`] drives streams: a shared rate, a shared in-flight +//! budget, and agents picked round-robin out of a pool. That is the right shape +//! when the thing being disturbed is a *stream*, and the wrong one when it is a +//! *place*. +//! +//! S3 cuts one executor off from worker-service. Every agent that executor owns +//! stalls, and those agents are spread evenly through every stream, so a shared +//! budget would be consumed by stalled operations within seconds and the agents +//! on the reachable executor would stop being submitted too. The run would then +//! show the undisturbed half degrading in lockstep with the disturbed one, and +//! the cause would be the driver rather than the platform. The mixed workload +//! already carries a comment about exactly this failure along the stream axis; +//! this module is the same lesson along the ownership axis. +//! +//! So: one task per agent, one operation in flight at a time, and a cadence +//! measured from the end of the previous operation rather than from a shared +//! clock. An agent whose executor is unreachable then contributes nothing and +//! costs nothing, which is what makes the two groups' throughput comparable. + +use crate::chaos::history::Stream; +use crate::chaos::workload::{self, WorkloadContext}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::info; + +/// The agents a run of `count` emitters drives, in index order. +/// +/// Delegates to [`WorkloadContext::agent_name`] rather than formatting a name +/// of its own, so the two cannot drift. That matters: the shared read-back and +/// exactly-once machinery both select on [`Stream::Durable`] and on names of +/// exactly that shape, and a second copy of the format here would empty both +/// silently rather than fail. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect() +} + +/// A running steady workload. Dropping the handle does not stop it — call +/// [`SteadyHandle::stop`], so operations in flight record themselves rather than +/// being cancelled. During a partition those are precisely the operations the +/// run exists to describe. +pub struct SteadyHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, +} + +impl SteadyHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos steady workload stopped after {} operations", + self.submitted() + ); + } +} + +/// Starts one emitter per agent and keeps them running until +/// [`SteadyHandle::stop`]. +pub fn start(ctx: WorkloadContext, agents: u32, interval: Duration) -> SteadyHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + info!( + "Chaos steady workload starting: {agents} emitters, one operation each, {interval:?} \ + between them" + ); + + for index in 0..agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + + tasks.spawn(async move { + let mut seq: u64 = 0; + while stop.load(Ordering::Relaxed) == 0 { + submitted.fetch_add(1, Ordering::Relaxed); + workload::submit_one(&ctx, Stream::Durable, index, seq).await; + seq += 1; + // From the end of the operation, not from a fixed clock. An + // agent that just spent two minutes stalled must not then fire + // a burst of catch-up operations: that would put its recovery + // throughput above its baseline and make the recovery cell + // unreadable. + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(interval).await; + } + } + }); + } + + SteadyHandle { + stop, + tasks, + submitted, + } +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 79416c03fd..552844f536 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -48,9 +48,20 @@ //! handful of suspect keys on it, which the workflow turns into ready-made trace //! queries. One global counter would have produced a haystack instead. +use crate::chaos::composed::ComposedFaultReport; +use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; +use crate::chaos::outage::StorageFaultReport; use crate::chaos::ownership::OwnershipSample; -use crate::chaos::probe::KeyProbe; +use crate::chaos::probe::{KeyProbe, SkipReason}; +use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::relay::RelayReport; +use crate::chaos::resolution::ResolutionReport; +use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::rollback::RollbackReport; +use crate::chaos::skew::SkewReport; +use crate::chaos::truncation::TruncationReport; +use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::time::Duration; @@ -358,6 +369,16 @@ pub struct ExactlyOnceReport { /// report so a clean verdict over a large number of them can be read for /// what it is: a weaker claim. pub keys_inconclusive: u64, + /// Of the inconclusive keys, the ones the probe pass never asked about at + /// all, by reason. + /// + /// Separated from the rest because the cause is different: an inconclusive + /// key was asked about and the exchange failed, a skipped key was never + /// asked. The pass only skips when it gives up — an agent that stopped + /// answering, or a pass that ran out of budget — so an entry here means the + /// run measured less than it set out to, whatever the verdict says. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub keys_skipped: BTreeMap, /// Keys that had a final result after recovery. pub keys_with_final_result: u64, /// Keys the driver never got a result for, but which the platform produced @@ -427,6 +448,9 @@ impl ExactlyOnceReport { .is_some_and(|class| class.is_definite_rejection()); if !definitive { report.keys_inconclusive += 1; + if let Some(reason) = probe.skipped { + *report.keys_skipped.entry(reason).or_default() += 1; + } continue; } report.findings.push(ExactlyOnceFinding { @@ -484,6 +508,62 @@ impl ExactlyOnceReport { } } +/// Whether a line a scenario reports is a finding or context. +/// +/// The distinction exists because CI branches on it. An annotation that fires +/// on every run — and one fires on every run if "routing settled before we +/// measured" counts as something needing review — trains its readers to ignore +/// it, which is worse than not having it. Context still reaches the report; it +/// just does not claim a human has to act. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoteLevel { + /// Something a human should look at before trusting the run. + Attention, + /// Something a human needs in order to read the run, but which is not + /// itself a problem. + Context, +} + +/// One operator-facing line, and which of the two lists it belongs in. +/// +/// Scenarios build these as they go and hand the whole batch to +/// [`ChaosSummary::absorb`] at the end, so the classification lives next to the +/// condition that produced it rather than in whatever reads the result later. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Note { + pub level: NoteLevel, + pub message: String, +} + +impl Note { + /// A line that means a human should look before trusting the run. + pub fn attention(message: impl Into) -> Self { + Self { + level: NoteLevel::Attention, + message: message.into(), + } + } + + /// A line that a human needs in order to read the run, but which is not + /// itself a problem. + pub fn context(message: impl Into) -> Self { + Self { + level: NoteLevel::Context, + message: message.into(), + } + } + + /// Picks the level from a condition, for the common case where the same + /// sentence is a finding or context depending on the numbers in it. + pub fn leveled(needs_attention: bool, message: impl Into) -> Self { + if needs_attention { + Self::attention(message) + } else { + Self::context(message) + } + } +} + /// Everything the driver reports for a scenario. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -503,6 +583,67 @@ pub struct ChaosSummary { /// read as "checked, nothing found". #[serde(skip_serializing_if = "Option::is_none")] pub exactly_once: Option, + /// The scheduled-fire account, for scenarios that pair scheduled actions + /// against the registrations that made them. Absent for scenarios that do + /// not, rather than an empty report that would read as "checked, nothing + /// found". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_fires: Option, + /// The promise-wakeup account, for scenarios that pair completions against + /// the waiters they were supposed to resume. Absent for scenarios that do + /// not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_wakeups: Option, + /// The reachability account, for scenarios that cut one executor off from + /// the tier that routes to it. Absent for scenarios that do not, for the + /// same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reachability: Option, + /// The cross-pod RPC account, for the one scenario that cuts two executors + /// off from each other. `None` everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relay: Option, + /// The clock-skew account, for the one scenario that moves an executor's + /// clock away from the rest of the cluster. `None` everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub skew: Option, + /// The name-resolution account, for the one scenario that stops a name + /// resolving on one executor. `None` everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution: Option, + /// The truncation account, for scenarios that revert agent state. Absent + /// for scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncation: Option, + /// The resurrection account, for scenarios that delete agents. Absent for + /// scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resurrection: Option, + /// The rollback account, for scenarios that move agents between builds and + /// back. Absent for scenarios that do not. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, + /// The storage-fault account, for scenarios that break a storage + /// dependency underneath every executor at once — by taking it away or by + /// slowing it down. Absent for scenarios that do not, for the same reason + /// as `scheduleFires`. + /// + /// Serialised as `storageOutage` rather than under the field's own name. + /// The wire name predates the scenarios that degrade a store rather than + /// removing one, and every archived result and the report generator that + /// reads them use it. Renaming the field would cost a schema bump and + /// silently stop rendering the runs already in the bucket, which is a worse + /// outcome than one name that has outlived its accuracy. + #[serde( + rename = "storageOutage", + default, + skip_serializing_if = "Option::is_none" + )] + pub storage_fault: Option, + /// How the two faults of a composed scenario lined up. Absent for the + /// scenarios that inject one, which is all of them but the `MF` codes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub composed_fault: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -512,8 +653,19 @@ pub struct ChaosSummary { /// it interpretable. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ownership: Vec, - /// The read-back verdicts that need a human, hoisted for scanning. + /// The verdicts and findings that need a human, hoisted for scanning. + /// + /// CI raises an annotation when this is non-empty, so nothing belongs here + /// that is true of a healthy run. Context goes in [`Self::notes`]. pub attention: Vec, + /// Lines a human needs in order to read the run, which are not themselves + /// problems: how the routing table looked before measuring, how much of the + /// mechanism under test the fault actually landed in, and so on. + /// + /// Kept out of [`Self::attention`] so that list keeps meaning "look at + /// this". Absent from older results, hence `default`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, } impl ChaosSummary { @@ -621,14 +773,42 @@ impl ChaosSummary { .collect(), routing_snapshots, exactly_once: None, + schedule_fires: None, + promise_wakeups: None, + reachability: None, + relay: None, + skew: None, + resolution: None, + truncation: None, + resurrection: None, + rollback: None, + storage_fault: None, + composed_fault: None, ownership: Vec::new(), attention, + notes: Vec::new(), + } + } + + /// Files a batch of scenario notes into the two lists by their level. + pub fn absorb(&mut self, notes: impl IntoIterator) { + for note in notes { + match note.level { + NoteLevel::Attention => self.attention.push(note.message), + NoteLevel::Context => self.notes.push(note.message), + } } } /// Attaches the exactly-once account and hoists its findings into /// [`Self::attention`], so a reader scanning the top of a report sees them /// next to the read-back verdicts rather than further down. + /// Skipped keys are hoisted alongside the findings, unlike the rest of the + /// numbers. They do not fail a run, but a clean verdict computed without + /// them is a weaker claim than the same verdict computed over everything, + /// and that has to be visible next to the verdict rather than only in the + /// account underneath it. A healthy pass skips nothing, so this stays quiet + /// on runs where it would only be noise. pub fn with_exactly_once(mut self, report: ExactlyOnceReport) -> Self { for finding in &report.findings { self.attention.push(format!( @@ -636,10 +816,173 @@ impl ChaosSummary { finding.violation, finding.idempotency_key, finding.agent, finding.detail )); } + for (reason, count) in &report.keys_skipped { + self.attention.push(format!( + "the probe pass never asked about {count} keys — {}", + reason.describe() + )); + } self.exactly_once = Some(report); self } + /// Attaches the scheduled-fire account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// More than the findings, unlike [`Self::with_exactly_once`]: an + /// unreadable target or a truncated fire log weakens every verdict the + /// report makes, and that has to be visible next to the verdicts rather + /// than only in the numbers underneath them. + pub fn with_schedule_fires(mut self, report: ScheduleFireReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.schedule_fires = Some(report); + self + } + + /// Attaches the promise-wakeup account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`], and one extra reason for it: + /// a waiter that could not be read is normally the weakest outcome there is, + /// but for a suspended waiter it can be the strongest evidence in the run. + /// The report decides which, and only what it classifies as attention lands + /// there. + pub fn with_promise_wakeups(mut self, report: WakeupReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.promise_wakeups = Some(report); + self + } + + /// Attaches the reachability account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The one line worth calling + /// out is the inconclusive case: a partition that never cut the executor off + /// produces a report full of healthy numbers, and that has to read as + /// "this run tested nothing" rather than as a pass. + pub fn with_reachability(mut self, report: ReachabilityReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.reachability = Some(report); + self + } + + /// Attaches the cross-pod RPC account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_reachability`], with the polarity reversed: + /// there, healthy numbers under a partition that never landed read as + /// "this run tested nothing". Here healthy numbers are the expected result, + /// so what has to reach a reader is the evidence that the run could have + /// failed — which is why the note lines carry the split and the partition + /// evidence on every run, findings or not. + pub fn with_relay(mut self, report: RelayReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.relay = Some(report); + self + } + + /// Attaches the clock-skew account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_relay`], for a sharper version of the same + /// reason. A skew changes nothing a pod can see about itself, so a run whose + /// injection silently failed produces the same clean report as one whose + /// injection landed and did no harm. The measured offset therefore goes into + /// the notes on every run, findings or not: it is the only line that says + /// the experiment happened. + pub fn with_skew(mut self, report: SkewReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.skew = Some(report); + self + } + + /// Attaches the name-resolution account. + /// + /// Same split as [`Self::with_skew`], and the notes matter more here than + /// anywhere else: S4's expected result is that nothing changed, so without + /// a line saying what was compared, a clean result and a run that injected + /// nothing are the same document. + pub fn with_resolution(mut self, report: ResolutionReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.resolution = Some(report); + self + } + + /// Attaches the truncation account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The line worth calling out + /// here is the inconclusive one: a kill that caught no revert in flight + /// proves nothing about crashing during a revert, and every clean number + /// underneath it describes reverts that completed either side of the fault. + pub fn with_truncation(mut self, report: TruncationReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.truncation = Some(report); + self + } + + /// Attaches the resurrection account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_truncation`], including the inconclusive line: + /// a kill that caught no delete in flight proves nothing about crashing + /// during a deletion. + pub fn with_resurrection(mut self, report: ResurrectionReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.resurrection = Some(report); + self + } + + /// Attaches the rollback account and hoists everything it wants a human to + /// see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the forward-leg + /// one: a rollback of agents that never left the old build proves nothing, + /// and that has to read as inconclusive rather than as a pass. + pub fn with_rollback(mut self, report: RollbackReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.rollback = Some(report); + self + } + + /// Attaches the storage-outage account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the + /// outage-not-observed one: a partition that failed to take hold leaves + /// every cell underneath it describing an undisturbed cluster, and that has + /// to read as "this run tested nothing" rather than as a pass. + pub fn with_storage_fault(mut self, report: StorageFaultReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.storage_fault = Some(report); + self + } + + /// Attaches the composed-fault account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as the others, with one line that is context on every run + /// rather than only on a bad one: where in the enclosing window the second + /// fault landed. Every figure in the rest of the report was measured on a + /// cluster under two faults, and a reader who does not know when the second + /// one arrived cannot place any of them. + pub fn with_composed_fault(mut self, report: ComposedFaultReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.composed_fault = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -688,6 +1031,30 @@ pub enum TerminationReason { /// owners is an agent whose state can fork, and there is no instant at /// which that is legitimate. ShardOwnershipViolated { findings: u64, first: String }, + /// An agent the platform said it had deleted came back with its state, or a + /// deletion landed somewhere other than the two answers it was allowed. + /// Asserted for the same reason as `RevertTruncationViolated`: invoking a + /// deleted id creates a new agent, so there are exactly two legal values and + /// no band of doubt. See [`crate::chaos::resurrection`]. + AgentResurrected { findings: u64, first: String }, + /// A revert landed somewhere other than the two values it was allowed to. + /// Asserted rather than reported, and the only read-back in the suite that + /// earns that: the driver knows the counter's value before the revert and + /// exactly how many invocations it asked to take back, so there is no band + /// of doubt around the answer. See [`crate::chaos::truncation`]. + RevertTruncationViolated { findings: u64, first: String }, + /// A scheduled action the platform accepted never fired, fired twice, or + /// fired after being refused. Asserted rather than reported: unlike a + /// count-based read-back, each of these is a statement about one named + /// action paired with one named registration, with no band of doubt around + /// it. See [`crate::chaos::fires`]. + ScheduledFireViolated { findings: u64, first: String }, + /// A promise completion the platform accepted never woke its waiter, woke it + /// twice, or woke it after being refused. Asserted for the same reason as + /// [`Self::ScheduledFireViolated`]: each is a statement about one named + /// completion paired with one named waiter, with no band of doubt around it. + /// See [`crate::chaos::wakeups`]. + PromiseWakeupViolated { findings: u64, first: String }, /// An agent's durable state did not survive a component update. Asserted /// because an update is supposed to change what an agent runs and nothing /// about what it remembers — state that moved is the one outcome an update @@ -746,6 +1113,40 @@ mod tests { Utc.timestamp_opt(1_800_000_000 + secs, 0).unwrap() } + /// The field is `storage_fault` in Rust and `storageOutage` on disk, and + /// that mismatch is deliberate rather than an oversight. + /// + /// The wire name predates the scenarios that slow a store down instead of + /// removing one. Every result already in the bucket uses it, and so does + /// the report generator that renders them. Renaming it would stop those + /// runs rendering to buy nothing, so the `#[serde(rename)]` stays and this + /// test is what stops a later tidy-up from quietly dropping it. + #[test] + fn the_storage_fault_account_still_serialises_under_its_original_name() { + let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None).with_storage_fault( + StorageFaultReport::build( + &[], + None, + "db.example", + crate::chaos::OutageExpectation::WholeWorkload { + quiet_floor_percent: 50.0, + }, + std::time::Duration::from_secs(120), + ), + ); + + let json = serde_json::to_value(&summary).unwrap(); + assert!( + json.get("storageOutage").is_some(), + "the on-disk name must not drift, got keys: {:?}", + json.as_object().map(|o| o.keys().collect::>()) + ); + assert!( + json.get("storageFault").is_none(), + "renaming this field silently orphans every archived result" + ); + } + fn op(op_id: u64, stream: Stream, phase: Phase, outcome: Outcome) -> OperationRecord { OperationRecord { op_id, @@ -960,12 +1361,31 @@ mod tests { /// A reader must never have to wonder whether a stream was skipped or just /// had nothing to say. + /// + /// The waiter stream is here for a different reason from the other two, and + /// the distinction is worth keeping straight: those two have no durable + /// count to read, while this one has a count that is *weaker* than what the + /// scenario already does with it. Its absence from the count-based read-back + /// means the token pairing in `promiseWakeups` is the account, not that + /// nothing was checked. #[test] fn streams_without_readback_are_named_rather_than_omitted() { let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None); assert_eq!( summary.streams_without_readback, - vec![Stream::Ephemeral, Stream::Promise] + // `Revert` is here for a different reason from the other three. + // Those keep no comparable durable state; a revert agent does, but + // some of its acknowledged work was deliberately taken back, so a + // generic counter comparison would report every reverted increment + // as lost. `crate::chaos::truncation` judges those agents exactly + // instead, which is strictly stronger. + vec![ + Stream::Ephemeral, + Stream::Promise, + Stream::PromiseWait, + Stream::Revert, + Stream::Delete + ] ); } @@ -1061,4 +1481,64 @@ mod tests { assert_eq!(stats.count, 0); assert_eq!(stats.max_ms, 0); } + + fn empty_summary() -> ChaosSummary { + ChaosSummary::build(&[], Vec::new(), Vec::new(), None) + } + + #[test] + fn notes_are_filed_by_level_rather_than_all_into_attention() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("WARNING: measured against an unsettled cluster"), + ]); + + assert_eq!(summary.attention.len(), 1); + assert!(summary.attention[0].contains("unsettled")); + assert_eq!(summary.notes.len(), 1); + assert!(summary.notes[0].contains("settled before measuring")); + } + + /// The reason the split exists. CI raises an annotation when `attention` is + /// non-empty, so a clean run has to leave it empty — otherwise the + /// annotation fires every time and stops meaning anything. + #[test] + fn a_run_with_only_context_raises_nothing_for_ci() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::context("S10 killed the executor with 353 actions pending"), + ]); + + assert!(summary.attention.is_empty()); + assert_eq!(summary.notes.len(), 2); + } + + #[test] + fn leveled_picks_the_list_from_the_condition() { + assert_eq!(Note::leveled(true, "x").level, NoteLevel::Attention); + assert_eq!(Note::leveled(false, "x").level, NoteLevel::Context); + } + + /// Older results have no `notes` key at all, and must still deserialise. + #[test] + fn a_result_written_before_notes_existed_still_reads() { + let mut summary = empty_summary(); + summary.absorb([Note::context("context")]); + let mut json: serde_json::Value = serde_json::to_value(&summary).unwrap(); + assert!(json.get("notes").is_some(), "notes are serialised when set"); + + json.as_object_mut().unwrap().remove("notes"); + let back: ChaosSummary = serde_json::from_value(json).unwrap(); + assert!(back.notes.is_empty()); + } + + /// An empty `notes` is omitted rather than written as `[]`, matching how + /// every other optional block in this result behaves. + #[test] + fn an_empty_notes_list_is_not_serialised() { + let json = serde_json::to_value(empty_summary()).unwrap(); + assert!(json.get("notes").is_none()); + } } diff --git a/integration-tests/src/chaos/truncation.rs b/integration-tests/src/chaos/truncation.rs new file mode 100644 index 0000000000..d0e39c28dc --- /dev/null +++ b/integration-tests/src/chaos/truncation.rs @@ -0,0 +1,731 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every revert land on a boundary, or did one tear (GOL-371)? +//! +//! The sharpest oracle in the suite, because it has no band of doubt in it. +//! Every other read-back compares a counter against a *range* — the width of +//! the range is the operations whose fate the driver could not determine. Here +//! the driver knows the counter's value immediately before the revert, because +//! the last increment of the round returned it, and it knows exactly how many +//! invocations the revert was asked to take back. So afterwards there are +//! exactly **two** legal values and nothing in between: +//! +//! * `V` — the revert never committed +//! * `V - N` — the revert committed +//! +//! Anything else is a defect, and which kind it is says what went wrong. A +//! value strictly between the two is a truncation that tore. A value below +//! `V - N` took back more than it was asked for. A value above `V` means state +//! grew across a revert. +//! +//! ### Why a partial truncation should be impossible, and why that is worth +//! testing anyway +//! +//! Reading `Worker::revert` in the executor: `RevertLastInvocations` walks back +//! to the nth `AgentInvocationStarted` entry and then commits **one** +//! `OplogEntry::revert` marking the region deleted. One entry cannot tear, so +//! the truncation itself is atomic by construction. +//! +//! The window worth killing into is the one *around* it. Reverting takes +//! `lock_stopped_worker`, so the worker is stopped first; then the entry is +//! committed; then `reattach_worker_status` runs, because — in the executor's +//! own words — "this commit will detach the worker status, immediately reattach +//! it so we see the up to date status". An executor that dies between the +//! commit and the reattach has left durable state changed and in-memory state +//! stale, which is the same shape as the S11 promise defect: the durable half +//! landed and the half that tells anyone about it did not. +//! +//! So the two findings this account expects to be able to make, if the platform +//! has a bug here, are [`TruncationViolation::AcknowledgedButNotApplied`] and +//! its opposite — not a torn counter. + +use crate::chaos::history::Outcome; +use crate::chaos::reverts::RevertRound; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one revert. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TruncationViolation { + /// The counter landed strictly between the pre-revert and post-revert + /// values: some of the invocations were taken back and some were not. + PartialTruncation, + /// More was taken back than the revert was asked for. + OverTruncation, + /// The counter is higher after the revert than it was before it. + Divergent, + /// The platform confirmed the revert and the state never moved. The same + /// shape as an accepted promise completion that never woke its waiter. + AcknowledgedButNotApplied, + /// The platform refused the revert and the state moved anyway. + RefusedButApplied, +} + +impl TruncationViolation { + pub fn as_str(self) -> &'static str { + match self { + TruncationViolation::PartialTruncation => "partial-truncation", + TruncationViolation::OverTruncation => "over-truncation", + TruncationViolation::Divergent => "divergent", + TruncationViolation::AcknowledgedButNotApplied => "acknowledged-but-not-applied", + TruncationViolation::RefusedButApplied => "refused-but-applied", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationFinding { + pub violation: TruncationViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// The counter before the revert, what it should have become, and what it + /// actually became. Carried so a finding can be read without the history. + pub before: u64, + pub expected_after_commit: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Reverts the platform confirmed that landed on the post-revert value. + pub applied: u64, + /// Reverts that left the agent exactly where it was. Legitimate only when + /// the platform never confirmed them. + pub not_applied: u64, + pub violations: u64, + /// Rounds the driver cannot judge: an increment that did not answer, so the + /// pre-revert value is unknown. + pub unjudgeable: u64, + /// Rounds no following increment ever probed. + pub unprobed: u64, +} + +/// Reverts the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertsCaught { + pub group: Group, + /// Reverts submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about: a run that caught none of + /// them proves nothing about crash-during-revert, however clean it looks. + pub reverts: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The truncation account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationReport { + /// What each round was configured to do, so an archived finding can be read + /// without the suite YAML to hand. + pub increments_per_round: u32, + pub revert_invocations: u32, + pub rounds_recorded: u64, + pub reverts_confirmed: u64, + pub reverts_indeterminate: u64, + pub reverts_rejected: u64, + /// Confirmed reverts that landed exactly on the post-revert value. + pub applied_exactly: u64, + /// Reverts the driver never heard back about that applied anyway — doubt + /// the platform resolved in its own favour, not a defect. + pub indeterminate_that_applied: u64, + /// Reverts the driver never heard back about that did not apply. Also not a + /// defect: the call may never have landed. + pub indeterminate_that_did_not: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +/// One round's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Applied, + NotApplied, + Violation(TruncationViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &RevertRound) -> Verdict { + let Some(before) = round.before_revert else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + let expected = before.saturating_sub(round.asked_to_revert as u64); + + if observed == expected { + // Applied. Legitimate unless the platform said it refused. + if round.outcome == Outcome::Rejected { + return Verdict::Violation(TruncationViolation::RefusedButApplied); + } + return Verdict::Applied; + } + if observed == before { + // Not applied. Legitimate unless the platform said it had been. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(TruncationViolation::AcknowledgedButNotApplied); + } + return Verdict::NotApplied; + } + if observed > before { + return Verdict::Violation(TruncationViolation::Divergent); + } + if observed < expected { + return Verdict::Violation(TruncationViolation::OverTruncation); + } + Verdict::Violation(TruncationViolation::PartialTruncation) +} + +impl TruncationReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + revert_invocations: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), TruncationCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = TruncationReport { + increments_per_round, + revert_invocations, + rounds_recorded: rounds.len() as u64, + reverts_confirmed: 0, + reverts_indeterminate: 0, + reverts_rejected: 0, + applied_exactly: 0, + indeterminate_that_applied: 0, + indeterminate_that_did_not: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| TruncationCell { + group, + window, + rounds: 0, + applied: 0, + not_applied: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.reverts_confirmed += 1, + Outcome::Indeterminate => report.reverts_indeterminate += 1, + Outcome::Rejected => report.reverts_rejected += 1, + } + + match judge(round) { + Verdict::Applied => { + cell.applied += 1; + if round.outcome == Outcome::Confirmed { + report.applied_exactly += 1; + } else { + report.indeterminate_that_applied += 1; + } + } + Verdict::NotApplied => { + cell.not_applied += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_revert.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + let expected = before.saturating_sub(round.asked_to_revert as u64); + findings.push(TruncationFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + expected_after_commit: expected, + observed, + detail: detail_for(violation, round, before, expected, observed), + }); + } + } + } + + // ── What the kill landed in the middle of ─────────────────────────── + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: std::collections::BTreeSet<&str> = + caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(RevertsCaught { + group, + reverts: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S7 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S7: {} further truncation finding(s) were dropped from the report", + self.findings_omitted + )); + } + + // The S10 lesson: a run that caught none of the mechanism proves + // nothing about it, however clean every other number looks. + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.reverts) + .sum(); + if caught == 0 { + lines.push( + "S7: the kill caught no revert in flight on the targeted executor, so this run \ + says nothing about crashing during a revert. Every verdict below describes \ + reverts that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S7: {} rounds of {} increments then a revert of {}; {} confirmed, {} in doubt, {} \ + refused", + self.rounds_recorded, + self.increments_per_round, + self.revert_invocations, + self.reverts_confirmed, + self.reverts_indeterminate, + self.reverts_rejected + )]; + lines.push(format!( + "S7: {} confirmed reverts landed exactly on the post-revert value; {} in doubt \ + applied anyway, {} in doubt did not", + self.applied_exactly, self.indeterminate_that_applied, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S7: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S7 {}: {} reverts across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.reverts, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S7 {} {}: {} rounds, {} applied, {} not applied, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.applied, + cell.not_applied, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&RevertRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for( + violation: TruncationViolation, + round: &RevertRound, + before: u64, + expected: u64, + observed: u64, +) -> String { + let asked = round.asked_to_revert; + match violation { + TruncationViolation::PartialTruncation => format!( + "the agent was worth {before}, a revert of {asked} invocations should have left it \ + at {expected}, and it came back at {observed} — between the two, so part of the \ + truncation landed and part did not" + ), + TruncationViolation::OverTruncation => format!( + "the agent was worth {before} and a revert of {asked} invocations left it at \ + {observed}, below the {expected} it asked for: more was taken back than requested" + ), + TruncationViolation::Divergent => format!( + "the agent was worth {before} before a revert of {asked} invocations and came back \ + at {observed}, higher than it started: state grew across a revert" + ), + TruncationViolation::AcknowledgedButNotApplied => format!( + "the platform confirmed a revert of {asked} invocations and the agent is still \ + worth {before}, not the {expected} it acknowledged" + ), + TruncationViolation::RefusedButApplied => format!( + "the platform refused a revert of {asked} invocations and the agent moved from \ + {before} to {observed} anyway" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use std::collections::BTreeMap; + use test_r::test; + + const ON_POD: &str = "chaos-s7-revert-0000"; + const CONTROL: &str = "chaos-s7-revert-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + /// One round: the agent was worth `before`, asked for 2 invocations back, + /// the platform answered `outcome`, and afterwards it read `observed`. + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> RevertRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + RevertRound { + agent: agent.to_string(), + round: 0, + before_revert: before, + asked_to_revert: 2, + outcome, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(80)), + observed_after: observed, + } + } + + fn build(rounds: &[RevertRound]) -> TruncationReport { + TruncationReport::build(rounds, &split(), Some(fault()), 4, 2) + } + + fn violations(report: &TruncationReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said yes and the agent landed exactly on + /// the post-revert value. + #[test] + fn a_revert_that_landed_where_it_said_it_would_is_not_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8))]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.applied_exactly, 1); + assert!(!report.has_violations()); + } + + /// The headline finding this scenario exists to make: the counter landed + /// between the two values it was allowed to have. + #[test] + fn a_counter_between_the_two_legal_values_is_a_torn_truncation() { + // Worth 10, asked for 4 back so 6 was legal, came back at 8. + let mut r = round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)); + r.asked_to_revert = 4; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![TruncationViolation::PartialTruncation] + ); + let finding = &report.findings[0]; + assert_eq!(finding.before, 10); + assert_eq!(finding.expected_after_commit, 6); + assert_eq!(finding.observed, 8); + assert!(report.has_violations(), "this must be able to fail the run"); + } + + /// The same shape as an accepted promise completion that never woke its + /// waiter: the durable half was acknowledged and nothing moved. + #[test] + fn a_confirmed_revert_that_changed_nothing_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::AcknowledgedButNotApplied] + ); + } + + /// Its opposite, and just as serious: refused, and applied regardless. + #[test] + fn a_refused_revert_that_applied_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Rejected, Some(8))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::RefusedButApplied] + ); + } + + #[test] + fn taking_back_more_than_was_asked_for_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(5))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::OverTruncation] + ); + } + + #[test] + fn state_growing_across_a_revert_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(11))]); + assert_eq!(violations(&report), vec![TruncationViolation::Divergent]); + } + + /// A revert the driver never heard back about is doubt, not damage. Both + /// answers are legitimate and neither is a finding. + #[test] + fn a_revert_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Indeterminate, Some(8)), + round(CONTROL, -10, Some(10), Outcome::Indeterminate, Some(10)), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_applied, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// A round whose increments never answered has no pre-revert value, so it + /// cannot be judged. Counted, never guessed at. + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(8))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + /// The last round an agent ran has nothing after it to probe with. It is + /// counted rather than assumed clean. + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught none of the mechanism proves nothing + /// about it, and the run has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_revert_says_the_run_proved_nothing() { + // Both rounds resolved well before the kill. + let report = build(&[ + round(ON_POD, -100, Some(10), Outcome::Confirmed, Some(8)), + round(CONTROL, -100, Some(10), Outcome::Confirmed, Some(8)), + ]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A revert still unresolved when the pod died is the population the whole + /// scenario is about, and it has to be counted apart from the rest. + #[test] + fn reverts_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(10), Outcome::Indeterminate, Some(10)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a revert"); + assert_eq!(entry.reverts, 1); + assert_eq!(entry.agents, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")) + ); + } + + /// Rounds are split by which executor owned the agent and which side of the + /// kill they fell on, so a control group cannot hide a disturbed one. + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)), + round(ON_POD, 10, Some(12), Outcome::Confirmed, Some(10)), + round(CONTROL, 10, Some(10), Outcome::Confirmed, Some(8)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().rounds, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 7) + .map(|_| round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 7); + assert!(report.has_violations()); + } + + /// A finding is read by an operator mid-window, so it has to state the + /// three numbers that make it interpretable without the history to hand. + #[test] + fn a_finding_states_the_numbers_that_make_it_readable() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + let detail = &report.findings[0].detail; + assert!(detail.contains("10"), "{detail}"); + assert!(detail.contains('8'), "{detail}"); + assert!( + !detail.contains("Some("), + "Option formatting leaked: {detail}" + ); + } +} diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs new file mode 100644 index 0000000000..61aecaca1e --- /dev/null +++ b/integration-tests/src/chaos/waiters.rs @@ -0,0 +1,697 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The suspended-waiter workload (GOL-377). +//! +//! Every other stream in this suite keeps an agent *busy*. This one keeps a pool +//! of them asleep. Each waiter repeats one round: +//! +//! 1. `arm(token)` creates a promise and returns it. Fast, and the only part the +//! driver needs an answer from. +//! 2. `wait(token, promise)` parks the agent on that promise. The invocation +//! stays open — on the platform as a suspended worker, in the driver as a +//! task holding a connection. +//! 3. After `dwellMillis`, the driver completes the promise from the outside. +//! 4. The wakeup ends the round, and the waiter starts the next one. +//! +//! ## Why one promise per waiter +//! +//! An agent runs its invocations one at a time, so a waiter parked in `wait` +//! cannot be armed again until it wakes. That is not a limitation to work +//! around; it is what makes the pool size mean something exact. With `waiters` +//! agents and one promise each, the number of agents standing suspended at the +//! instant the pod dies is a known constant rather than a sample — the same +//! property [`crate::chaos::pinned`] gets from one in-flight operation per +//! agent, for the same reason. +//! +//! It also makes a lost wakeup visible while the run is still going. A waiter +//! whose completion never arrives can never start another round, so it simply +//! stops producing. See [`WaiterHandle::stalled`]. +//! +//! ## Why the dwell +//! +//! `dwellMillis` decides how the population divides at the moment of the kill. +//! Every waiter is either parked-and-waiting (the dwell) or being-woken (the +//! completion round trip), and the dwell is far the longer of the two — so the +//! kill lands mostly in the first, which is the state this scenario is about, +//! and occasionally in the second, which is the narrower race S8 already covers +//! for ordinary invocations. +//! +//! The dwell also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! promise armed before the kill would already have been completed by the time +//! the pod died, and the run would measure nothing. +//! +//! ## Why completions are retried like everything else +//! +//! The suite's retry policy — one same-key retry, transport failures only — +//! applies here too, and a completion is the one operation where that could +//! plausibly be accused of hiding the defect: `complete` writes with +//! `set_if_not_exists` and re-triggers the wakeup, so a retry can repair a +//! completion whose wakeup was lost. +//! +//! It cannot hide what this scenario asks, because the question is scoped to +//! completions the platform *confirmed*. A retry only happens when the previous +//! attempt did not return success, so a confirmed completion is always a single +//! accepted call. "The platform said yes and the waiter never woke" is exactly +//! as detectable with the retry as without it, and the retry keeps the workload +//! behaving like a client anyone would actually write. + +use crate::chaos::PromiseConfig; +use crate::chaos::history::{Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::workload::{self, WorkloadContext}; +use chrono::{DateTime, TimeZone, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::PromiseId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::dsl::TestDsl; +use golem_wasm::{FromValue, ValueAndType}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Agent type exported by the promise component for this workload. +pub const PROMISE_WAITER_AGENT: &str = "PromiseWaiter"; + +/// Payload a completion carries. Nothing reads it; the token is what identifies +/// a round, and it travels through the agent rather than through the promise. +const COMPLETION_PAYLOAD: &[u8] = b"chaos-s11"; + +/// How long a waiter's loop waits for its own wakeup before concluding the +/// waiter is parked for good and standing down. +/// +/// Deliberately a multiple of the wakeup budget rather than a fixed number: the +/// budget is already the run's statement about what a recovery may cost, and +/// anything that treats "slow" as "lost" would turn a bad p99 into a false +/// finding. Standing down is not a verdict either — the read-back still asks the +/// waiter what happened, and a waiter that woke late says so. +const STALL_MULTIPLE: u32 = 4; + +/// Floor under [`STALL_MULTIPLE`], for a configuration with a very short budget. +const MIN_STALL_TIMEOUT: Duration = Duration::from_secs(180); + +/// How many waiters are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking a few hundred of them one at a time behind a per-read ceiling +/// outlasts the maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// Ceiling on one wakeup-log read. Generous, because it happens on a cluster +/// that has just been through a fault, and because the answer carries every +/// wakeup the waiter recorded rather than one number. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// How long the smoke round waits for its one wakeup before giving up. +/// +/// Generous against a cluster that is merely cold, short enough that a broken +/// completion path costs seconds rather than a baseline. +const SMOKE_WAKE_TIMEOUT: Duration = Duration::from_secs(60); + +/// How often the smoke round asks whether its waiter came back. +const SMOKE_POLL_INTERVAL: Duration = Duration::from_secs(2); + +/// A running waiter workload. +/// +/// As elsewhere, dropping the handle does not stop it: call +/// [`WaiterHandle::stop`] so rounds in flight record themselves instead of being +/// cancelled mid-completion. +pub struct WaiterHandle { + tasks: JoinSet<()>, + running: Arc, + rounds: Arc, + stalled: Arc, +} + +impl WaiterHandle { + /// Rounds started across all waiters. + pub fn rounds(&self) -> u64 { + self.rounds.load(Ordering::Relaxed) + } + + /// Waiters that stood down because a wakeup never arrived. + /// + /// The live half of this scenario's oracle. A waiter counted here was parked + /// on a promise the driver had completed, and stayed parked long enough that + /// no recovery budget explains it. The read-back then decides whether it + /// eventually woke. + pub fn stalled(&self) -> u64 { + self.stalled.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.running.store(false, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + } +} + +/// Warms every waiter so the baseline measures resident agents rather than cold +/// starts. Returns how many answered. +pub async fn warm(ctx: &WorkloadContext, waiters: &[String]) -> usize { + let mut warmed = 0; + let mut set = JoinSet::new(); + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .is_ok() + }); + } + while let Some(result) = set.join_next().await { + if matches!(result, Ok(true)) { + warmed += 1; + } + } + warmed +} + +/// Drives one whole round against a throwaway waiter, before the run commits to +/// anything. +/// +/// The first S11 run spent its entire baseline and then aborted on a count, +/// because `arm` and `complete` were both healthy and only `wait` was refused — +/// so the operation totals looked normal right up until nothing had woken. The +/// scenario knew something was wrong three minutes after it could have. +/// +/// This is the cheap version of that question, asked first and answered with the +/// platform's own error rather than with a zero. It uses an agent outside the +/// waiter pool, so a failed smoke test leaves the measured population untouched. +/// +/// Errors are returned rather than recorded: this is a pre-flight check, not +/// part of the workload, and its whole value is in saying *why*. +pub async fn smoke_test(ctx: &WorkloadContext, dwell: Duration) -> Result<(), String> { + let waiter = format!("{}-promise-smoke", ctx.key_prefix); + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let token = format!("{waiter}-smoke"); + + let created = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "arm", data_value!(token.clone())) + .await + .map_err(|e| format!("arm failed: {e:#}"))?; + let promise = created + .into_return_value_and_type() + .ok_or_else(|| "arm returned no promise id".to_string())?; + let id = PromiseId::from_value(promise.value.clone()) + .map_err(|e| format!("arm returned a promise id the driver cannot parse: {e}"))?; + + // Fire-and-forget on purpose. A `wait` that parks correctly never returns + // while we are looking at it, so awaiting it here would prove nothing and + // block forever — but a `wait` the platform refuses fails at the API + // boundary, which is exactly the failure this is here to catch. + ctx.user + .invoke_agent( + &ctx.promise, + &parsed, + "wait", + data_value!(token.clone(), promise), + ) + .await + .map_err(|e| format!("wait was refused: {e:#}"))?; + + tokio::time::sleep(dwell).await; + ctx.user + .complete_promise(&id, COMPLETION_PAYLOAD.to_vec()) + .await + .map_err(|e| format!("complete_promise failed: {e:#}"))?; + + // The waiter has to actually come back. Polled rather than awaited for the + // same reason as above. + let deadline = tokio::time::Instant::now() + SMOKE_WAKE_TIMEOUT; + loop { + if let Ok(value) = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + && let Some(wakes) = value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + && wakes > 0 + { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "the smoke waiter was armed, told to wait and completed, and did not wake \ + within {SMOKE_WAKE_TIMEOUT:?}" + )); + } + tokio::time::sleep(SMOKE_POLL_INTERVAL).await; + } +} + +/// Starts one loop per waiter. +pub fn start(ctx: WorkloadContext, waiters: &[String], config: &PromiseConfig) -> WaiterHandle { + let running = Arc::new(AtomicBool::new(true)); + let rounds = Arc::new(AtomicU64::new(0)); + let stalled = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + let stall_timeout = (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT); + info!( + "S11: starting {} waiters, {:?} dwell, standing a waiter down after {stall_timeout:?} \ + without a wakeup", + waiters.len(), + config.dwell() + ); + + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let running = running.clone(); + let rounds = rounds.clone(); + let stalled = stalled.clone(); + let dwell = config.dwell(); + tasks.spawn(async move { + run_waiter(ctx, waiter, dwell, stall_timeout, running, rounds, stalled).await; + }); + } + + WaiterHandle { + tasks, + running, + rounds, + stalled, + } +} + +/// One waiter's rounds, until the workload stops or the waiter stands down. +async fn run_waiter( + ctx: WorkloadContext, + waiter: String, + dwell: Duration, + stall_timeout: Duration, + running: Arc, + rounds: Arc, + stalled: Arc, +) { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let mut round = 0u64; + + while running.load(Ordering::Relaxed) { + let token = ctx.idempotency_key(&waiter, round); + round += 1; + rounds.fetch_add(1, Ordering::Relaxed); + + let Some(armed) = arm(&ctx, &waiter, &parsed, &token).await else { + // Nothing to complete and nobody parked. The next round tries again + // after a dwell, which keeps a wholly unreachable platform from + // spinning. + tokio::time::sleep(dwell).await; + continue; + }; + + // Park the waiter. Held open deliberately: the driver's own view of the + // wakeup is one of the two independent answers this scenario collects, + // and the fault is expected to take it away. + let wait_task = { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let parsed = parsed.clone(); + let token = token.clone(); + let promise = armed.value.clone(); + tokio::spawn(async move { wait(&ctx, &waiter, &parsed, &token, &promise).await }) + }; + + tokio::time::sleep(dwell).await; + complete(&ctx, &waiter, &token, &armed.id).await; + + // The handle is dropped rather than aborted on the timeout path, which + // leaves the invocation running. That is deliberate: aborting it would + // throw away the one record that says how the round ended, and the agent + // is parked either way — the driver's task is not what is holding it. + match tokio::time::timeout(stall_timeout, wait_task).await { + Ok(_) => {} + Err(_) => { + warn!( + "S11: waiter {waiter} has not woken {stall_timeout:?} after its completion \ + on token {token}; standing it down" + ); + stalled.fetch_add(1, Ordering::Relaxed); + return; + } + } + } +} + +/// One armed promise, in the two forms the round needs it. +/// +/// Both halves are kept because the two consumers want different things and +/// only one of them will accept a re-encoding. +#[derive(Clone)] +struct Armed { + /// For [`TestDsl::complete_promise`], the external REST API, which takes a + /// typed promise id and serialises it itself. + id: PromiseId, + /// For the `wait` invocation, which passes the promise back *into* an + /// agent. + /// + /// This has to be the value the agent handed out, verbatim. Re-encoding the + /// parsed [`PromiseId`] does not work: `data_value!` derives the parameter's + /// type from `IntoValue`, which names the record's fields with their WIT + /// spellings (`agent-id`, `oplog-idx`), while the agent's generated + /// parameter schema declares them in the component model's own spelling + /// (`agent_id`, `oplog_idx`). The type checker rejects the call outright. + /// Threading the original value through sidesteps the question, because its + /// type came from the agent in the first place. + /// + /// `crate::benchmarks::density::promise` keeps both halves for exactly this + /// reason. + value: ValueAndType, +} + +/// Creates the round's promise, recording the invocation. +/// +/// The result comes back through a cell rather than a return value because +/// [`workload::run_operation`] owns the retry rule and the failure +/// classification for every stream in the suite, and it reports outcomes rather +/// than payloads. Duplicating it here to get one value back would put the two +/// load-bearing rules of every chaos scenario in two places. +async fn arm( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, +) -> Option { + let cell: Arc>> = Arc::new(std::sync::Mutex::new(None)); + let sink = cell.clone(); + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "arm", + format!("{token}-arm"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let sink = sink.clone(); + let token = token.to_string(); + async move { + let created = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "arm", + data_value!(token), + ) + .await?; + let value = created + .into_return_value_and_type() + .ok_or_else(|| anyhow::anyhow!("arm returned no promise id"))?; + let id = PromiseId::from_value(value.value.clone()) + .map_err(|e| anyhow::anyhow!("invalid promise id: {e}"))?; + *sink.lock().unwrap() = Some(Armed { id, value }); + Ok(None) + } + }, + ) + .await; + + cell.lock().unwrap().clone() +} + +/// Parks the waiter until its promise resolves. +async fn wait( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, + promise: &ValueAndType, +) { + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "wait", + format!("{token}-wait"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let token = token.to_string(); + let promise = promise.clone(); + async move { + ctx.user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "wait", + data_value!(token, promise), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Completes the round's promise from outside the agent. +/// +/// Recorded under the round's token rather than under a key of its own, which is +/// what joins this record to the wakeup the waiter logs. The completion API is +/// keyed by promise id and takes no idempotency key of its own — a retry is the +/// same completion because it names the same promise, and `set_if_not_exists` on +/// the platform side is what makes that true. +async fn complete(ctx: &WorkloadContext, waiter: &str, token: &str, promise_id: &PromiseId) { + let ctx2 = ctx.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "complete", + token.to_string(), + |_key| { + let ctx = ctx2.clone(); + let promise_id = promise_id.clone(); + async move { + ctx.user + .complete_promise(&promise_id, COMPLETION_PAYLOAD.to_vec()) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Reads every waiter's wakeup log. +pub async fn read_logs(ctx: &WorkloadContext, waiters: &[String]) -> Vec { + let mut logs = Vec::with_capacity(waiters.len()); + for chunk in waiters.chunks(READ_CONCURRENCY) { + let mut set = JoinSet::new(); + for waiter in chunk { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { read_log(&ctx, &waiter).await }); + } + while let Some(result) = set.join_next().await { + match result { + Ok(log) => logs.push(log), + Err(e) => warn!("S11: a wakeup-log read task failed: {e}"), + } + } + } + logs.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs +} + +async fn read_log(ctx: &WorkloadContext, waiter: &str) -> WaiterWakeupLog { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.to_string()); + + let wakes = match read_within(waiter, "wakes", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + .map(|v| v as u64), + Err(e) => { + return WaiterWakeupLog { + agent: waiter.to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some(e), + }; + } + }; + + match read_within(waiter, "wakeups", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakeups", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => { + let wakeups = value + .into_return_value_and_type() + .map(|v| parse_wakeups(v.value)) + .unwrap_or_default(); + WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups, + error: None, + } + } + Err(e) => WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups: Vec::new(), + error: Some(e), + }, + } +} + +/// A read-back invocation under [`READ_TIMEOUT`]. +/// +/// A timeout is a verdict, not an error to propagate. It is also the loudest +/// thing this scenario can observe: a waiter that will not answer `wakeups` is +/// usually a waiter still parked on a promise that was completed long ago, and +/// [`crate::chaos::wakeups`] treats that case differently from an ordinary +/// unreadable agent. +async fn read_within(waiter: &str, what: &str, read: F) -> Result +where + F: std::future::Future>, +{ + match tokio::time::timeout(READ_TIMEOUT, read).await { + Ok(result) => result, + Err(_) => { + warn!("S11: reading {what} on waiter {waiter} timed out after {READ_TIMEOUT:?}"); + Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + } + } +} + +/// Turns the agent's `(token, armed_millis, woken_millis)` triples into records. +fn parse_wakeups(value: golem_wasm::Value) -> Vec { + let raw: Vec<(String, u64, u64)> = match Vec::<(String, u64, u64)>::from_value(value) { + Ok(raw) => raw, + Err(e) => { + warn!("S11: could not read a wakeup log: {e}"); + return Vec::new(); + } + }; + raw.into_iter() + .map(|(token, armed_millis, woken_millis)| WakeupRecord { + token, + armed_at: from_millis(armed_millis), + woken_at: from_millis(woken_millis), + }) + .collect() +} + +/// Epoch milliseconds as the agent stamped them. +/// +/// A zero means the agent had no armed time for the token, which the component +/// only produces if its own arm log rolled over. It is kept as the epoch rather +/// than dropped so the resulting nonsense interval is visible instead of the +/// wakeup silently going missing. +fn from_millis(millis: u64) -> DateTime { + Utc.timestamp_millis_opt(millis as i64) + .single() + .unwrap_or_else(|| Utc.timestamp_nanos(0)) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + #[test] + fn a_stall_timeout_scales_with_the_wakeup_budget() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 120, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + Duration::from_secs(480) + ); + } + + /// A budget short enough that four of it would call an ordinary recovery a + /// stall still gets the floor. + #[test] + fn a_short_wakeup_budget_still_gets_the_floor() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 10, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + MIN_STALL_TIMEOUT + ); + } + + #[test] + fn an_agent_timestamp_of_zero_stays_visible_as_the_epoch() { + assert_eq!(from_millis(0).timestamp_millis(), 0); + } + + /// Why `wait` is handed the value `arm` returned rather than a re-encoded + /// [`PromiseId`]. + /// + /// This is not a style preference, and getting it wrong is not caught by + /// anything local: the first S11 run had every single `wait` refused by the + /// type checker in two milliseconds, with `arm` and `complete` both clean, + /// because a re-encoded promise id names its fields the way WIT does and the + /// agent's generated parameter schema names them the way the component model + /// does. The two spellings never meet. + /// + /// So the rule is: a value that came *out* of an agent goes back *into* one + /// verbatim. This test pins the mismatch that makes the rule necessary, so + /// that a future change to either derive shows up here rather than on a + /// cluster. + #[test] + fn a_re_encoded_promise_id_does_not_name_its_fields_the_way_an_agent_declares_them() { + use golem_common::model::PromiseId; + use golem_wasm::IntoValue; + use golem_wasm::analysis::AnalysedType; + + let AnalysedType::Record(record) = PromiseId::get_type() else { + panic!("a promise id is a record"); + }; + let names: Vec<&str> = record.fields.iter().map(|f| f.name.as_str()).collect(); + + // What `data_value!` would ship: the WIT spellings. + assert!(names.contains(&"agent-id"), "got {names:?}"); + assert!(names.contains(&"oplog-idx"), "got {names:?}"); + + // What the agent's parameter schema actually asks for, and does not get. + assert!(!names.contains(&"agent_id"), "got {names:?}"); + assert!(!names.contains(&"oplog_idx"), "got {names:?}"); + } +} diff --git a/integration-tests/src/chaos/wakeups.rs b/integration-tests/src/chaos/wakeups.rs new file mode 100644 index 0000000000..55506a84e2 --- /dev/null +++ b/integration-tests/src/chaos/wakeups.rs @@ -0,0 +1,1120 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pairing completions against wakeups (GOL-377). +//! +//! S11's question is not "how many promises resolved". It is, for each +//! completion the platform *accepted*, whether the agent suspended on that +//! promise was resumed — once, and only once. Counting cannot answer it: a lost +//! wakeup and a duplicate wakeup cancel out in a total, and neither can be +//! localised to an agent afterwards. +//! +//! So every round carries a token. The driver mints it, `arm` records it against +//! the promise it created, and the waiter writes it into its wakeup log when it +//! is resumed. Pairing the two turns each failure into a statement about one +//! named round: this completion, accepted at this time, against this waiter, +//! never woke it. +//! +//! ## What counts as a finding, and what only counts as doubt +//! +//! A completion the platform confirmed is a promise the platform made. If the +//! waiter never woke and its log is whole, that is a finding, full stop. +//! +//! A completion that failed in an *indeterminate* way is not. From the client +//! side a dropped connection is indistinguishable from a request that arrived +//! and executed, and a pod kill produces exactly these. If such a round woke, +//! the platform resolved the doubt in its own favour and the report says so; if +//! it did not, nothing is proven and the round is counted as inconclusive rather +//! than as a loss. +//! +//! ## The waiter that will not answer +//! +//! One case here is unlike anything the other scenarios see. A waiter is parked +//! *inside* an invocation, so a waiter that never wakes cannot answer a read +//! either — its `wakeups` read queues behind the `wait` that is still running. +//! +//! An unreadable agent is normally the weakest possible outcome: it means the +//! run cannot say. Here it is nearly the opposite. A waiter whose completion was +//! confirmed, which then stopped producing rounds *and* could not be read, is a +//! worker wedged on a promise that was resolved long ago. The report separates +//! that case ([`WakeupViolation::NeverWoke`], with the read failure as its +//! detail) from an agent that merely timed out while otherwise healthy. +//! +//! ## Two clocks, and which number to believe +//! +//! The waiter stamps `armedAt` and `wokenAt` from the executor's clock; the +//! driver stamps the completion from its own. So the headline delay — +//! completion accepted to waiter resumed — spans two clocks, exactly as the +//! scheduled-fire delay does in [`crate::chaos::fires`], and the same guard +//! applies: `minDelayMs` is reported per cell so skew shows up as a negative +//! number instead of quietly flattering a percentile. +//! +//! There is one cross-check S10 cannot make. `parkedMs` — armed to woken — is +//! stamped at both ends by the executor, so it carries no skew at all. It is not +//! the delay, because it also contains the round's deliberate dwell, but on a +//! healthy baseline `parked - dwell` and the cross-clock delay should agree. A +//! gap between them is skew, and the report carries both rather than picking. + +use crate::chaos::history::{OperationRecord, Outcome, Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::split::{FaultWindow, PodSplit, Window}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A run that lost every wakeup would otherwise produce tens of thousands of +/// them and an artifact nobody can open. The count is reported separately, so +/// truncation is stated rather than inferred from a suspiciously round number. +const MAX_FINDINGS: usize = 200; + +/// The method name the completion operations are recorded under. +const COMPLETE_METHOD: &str = "complete"; + +/// The method name the parking invocations are recorded under. +const WAIT_METHOD: &str = "wait"; + +/// What [`WAIT_METHOD`] operations append to their round's token to form their +/// own idempotency key. +const WAIT_KEY_SUFFIX: &str = "-wait"; + +/// Whether a waiter was on the executor the fault killed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WaiterGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl WaiterGroup { + pub fn as_str(self) -> &'static str { + match self { + WaiterGroup::OnKilledExecutor => "on-killed-executor", + WaiterGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for WaiterGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What went wrong with one round. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WakeupViolation { + /// A completion the platform accepted whose waiter was never resumed. + NeverWoke, + /// One completion, two or more wakeups. A promise resolved twice, or a + /// recovery that replayed the resume without deduplicating it. + WokeMoreThanOnce, + /// A wakeup for a round whose completion the platform definitively refused. + WokeDespiteRejection, +} + +impl WakeupViolation { + pub fn as_str(self) -> &'static str { + match self { + WakeupViolation::NeverWoke => "never-woke", + WakeupViolation::WokeMoreThanOnce => "woke-more-than-once", + WakeupViolation::WokeDespiteRejection => "woke-despite-rejection", + } + } +} + +impl std::fmt::Display for WakeupViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupFinding { + pub violation: WakeupViolation, + pub token: String, + pub agent: String, + pub window: Window, + pub detail: String, +} + +/// Wakeup delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupDelayStats { + pub group: WaiterGroup, + pub window: Window, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than a waiter woken before it was completed. + pub min_delay_ms: i64, + /// Wakeups whose delay exceeded the configured budget. + pub over_budget: u64, + /// Armed-to-woken, on the executor's clock alone. Carries the round's dwell + /// as well as the delay, and carries no skew. + pub parked: LatencyStats, + /// How much longer the *caller* waited than the platform actually took. + /// + /// The `wait` invocation's own duration, less the round's dwell, less the + /// delay the waiter recorded. On a healthy round this is a few milliseconds + /// of round trip. A large value means the platform woke the agent on time + /// and the answer did not come back — which is a different defect from a + /// slow wakeup, and invisible in [`Self::delay`]. + pub client_excess: LatencyStats, + /// Rounds whose caller waited longer than the whole wakeup budget *after* + /// the waiter had already woken. + pub client_stalled: u64, +} + +/// The promise-wakeup account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupReport { + /// What resuming a suspended waiter is allowed to cost, from the suite YAML. + /// Recorded so a percentile in an archived result can be read years later + /// against the number it was judged by rather than against today's config. + pub wakeup_budget_ms: u64, + /// The dwell each round held before its completion, so `parked` can be read + /// without the suite YAML to hand. + pub dwell_ms: u64, + pub completions_confirmed: u64, + pub completions_indeterminate: u64, + pub completions_rejected: u64, + /// Wakeups the waiters recorded, including any whose token is unknown. + pub wakeups_recorded: u64, + /// Accepted completions paired with exactly one wakeup. + pub woke_once: u64, + /// Completions the driver was never sure of that woke anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_woke: u64, + /// Completions the driver was never sure of that never woke. Not a finding: + /// the completion may never have landed. + pub inconclusive: u64, + /// Completions whose waiter could not testify, because its log was + /// unreadable or truncated *and* nothing else about it looked wrong. + pub unverifiable: u64, + /// Wakeups whose token no completion claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Waiters the read-back could not reach at all. + pub waiters_unreadable: Vec, + /// Waiters whose wakeup log hit the component's cap. + pub waiters_truncated: Vec, + /// Waiters that stopped producing rounds during the run because a wakeup + /// never arrived, as the workload itself observed. The live half of the + /// oracle — see [`crate::chaos::waiters::WaiterHandle::stalled`]. + pub waiters_stood_down: u64, + /// Waiters that stood down *and* could not then be read: parked inside an + /// invocation, which is what a wedged worker looks like from outside. + pub waiters_wedged: Vec, + pub delay: Vec, + /// Rounds across every cell whose caller waited past the budget after the + /// wakeup had already happened. See [`WakeupDelayStats::client_excess`]. + pub client_stalled_total: u64, + /// The worst such gap, in milliseconds. + pub client_stall_worst_ms: u64, + /// How many of those callers had to retry to get their answer at all. A + /// stall that only ends on a retry is a request that was never coming back. + pub client_stall_retried: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One round, as the pairing sees it. +struct Round<'a> { + token: &'a str, + agent: &'a str, + outcome: Outcome, + submitted_at: DateTime, +} + +impl WakeupReport { + /// Pairs completions against wakeups. + /// + /// `records` is the whole history; only the `complete` operations of the + /// waiter stream are considered — `arm` and `wait` are recorded for the + /// timeline, not for this account. + #[allow(clippy::too_many_arguments)] + pub fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + fault: Option, + dwell: Duration, + wakeup_budget: Duration, + stood_down: u64, + ) -> Self { + let budget_ms = wakeup_budget.as_millis().min(u64::MAX as u128) as u64; + let dwell_ms = dwell.as_millis().min(u64::MAX as u128) as u64; + + let mut wakeups_by_token: BTreeMap<&str, Vec<&WakeupRecord>> = BTreeMap::new(); + let mut complete_log: BTreeMap<&str, bool> = BTreeMap::new(); + let mut waiters_unreadable = Vec::new(); + let mut waiters_truncated = Vec::new(); + let mut wakeups_recorded = 0u64; + + for log in logs { + complete_log.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + waiters_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + waiters_truncated.push(log.agent.clone()); + } + for wakeup in &log.wakeups { + wakeups_recorded += 1; + wakeups_by_token + .entry(wakeup.token.as_str()) + .or_default() + .push(wakeup); + } + } + + let on_killed: BTreeSet<&str> = split.on_pod.iter().map(|s| s.as_str()).collect(); + let mut report = Self::empty(budget_ms, dwell_ms); + report.wakeups_recorded = wakeups_recorded; + report.waiters_stood_down = stood_down; + + // A stalled waiter that then could not be read is the wedged case. It is + // computed before the round loop because the loop uses it to decide + // whether an unreadable waiter excuses a missing wakeup or convicts it. + let unreadable: BTreeSet<&str> = waiters_unreadable.iter().map(|s| s.as_str()).collect(); + + let mut cells: BTreeMap<(WaiterGroup, Window), DelayCell> = BTreeMap::new(); + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let waits = wait_observations(records); + let mut stall_worst_ms = 0u64; + let mut stall_retried = 0u64; + + for round in completions(records) { + let group = if on_killed.contains(round.agent) { + WaiterGroup::OnKilledExecutor + } else { + WaiterGroup::Elsewhere + }; + let window = Window::of(round.submitted_at, fault); + let wakeups = wakeups_by_token + .get(round.token) + .cloned() + .unwrap_or_default(); + claimed.insert(round.token); + + match round.outcome { + Outcome::Confirmed => report.completions_confirmed += 1, + Outcome::Indeterminate => report.completions_indeterminate += 1, + Outcome::Rejected => report.completions_rejected += 1, + } + + if wakeups.len() > 1 { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeMoreThanOnce, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "{} wakeups recorded for one completion, at {}", + wakeups.len(), + wakeups + .iter() + .map(|w| w.woken_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }); + } + + if let Some(wakeup) = wakeups.first() { + if round.outcome == Outcome::Rejected { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeDespiteRejection, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was definitively refused, and the waiter woke at {}", + wakeup.woken_at.to_rfc3339() + ), + }); + } else { + if round.outcome == Outcome::Confirmed { + report.woke_once += 1; + } else { + report.indeterminate_that_woke += 1; + } + let delay_ms = (wakeup.woken_at - round.submitted_at).num_milliseconds(); + let cell = cells.entry((group, window)).or_default(); + cell.push(delay_ms, wakeup.parked_ms(), budget_ms); + + // The caller's own view of the same round. `wait` covers the + // dwell as well as the wakeup, so the dwell comes off before + // the two are compared. + if let Some(observed) = waits.get(round.token) { + let client_ms = observed.duration_ms.saturating_sub(dwell_ms); + let excess = client_ms.saturating_sub(delay_ms.max(0) as u64); + cell.push_client(excess, budget_ms); + if excess > budget_ms { + stall_worst_ms = stall_worst_ms.max(excess); + if observed.attempts > 1 { + stall_retried += 1; + } + } + } + } + continue; + } + + // No wakeup. What that proves depends on the completion's outcome + // and on whether the waiter could testify at all. + match round.outcome { + Outcome::Rejected => {} + Outcome::Indeterminate => report.inconclusive += 1, + Outcome::Confirmed => { + let log_whole = complete_log.get(round.agent).copied().unwrap_or(false); + if log_whole { + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter's whole wakeup \ + log has no entry for it", + round.submitted_at.to_rfc3339() + ), + }); + } else if unreadable.contains(round.agent) && stood_down > 0 { + // The wedged case: the workload watched this waiter stop + // producing, and the read-back then could not reach it. + // Both symptoms of one worker still parked on a promise + // that was resolved. + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter has answered \ + nothing since, which is what a worker parked on a resolved \ + promise looks like from outside", + round.submitted_at.to_rfc3339() + ), + }); + if !report.waiters_wedged.iter().any(|w| w == round.agent) { + report.waiters_wedged.push(round.agent.to_string()); + } + } else { + report.unverifiable += 1; + } + } + } + } + + report.unknown_tokens = wakeups_by_token + .keys() + .filter(|token| !claimed.contains(*token)) + .count() as u64; + report.waiters_unreadable = waiters_unreadable; + report.waiters_truncated = waiters_truncated; + report.delay = cells + .into_iter() + .map(|((group, window), cell)| cell.into_stats(group, window)) + .collect(); + report.client_stalled_total = report.delay.iter().map(|c| c.client_stalled).sum(); + report.client_stall_worst_ms = stall_worst_ms; + report.client_stall_retried = stall_retried; + report + } + + fn empty(wakeup_budget_ms: u64, dwell_ms: u64) -> Self { + Self { + wakeup_budget_ms, + dwell_ms, + completions_confirmed: 0, + completions_indeterminate: 0, + completions_rejected: 0, + wakeups_recorded: 0, + woke_once: 0, + indeterminate_that_woke: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + waiters_unreadable: Vec::new(), + waiters_truncated: Vec::new(), + waiters_stood_down: 0, + waiters_wedged: Vec::new(), + delay: Vec::new(), + client_stalled_total: 0, + client_stall_worst_ms: 0, + client_stall_retried: 0, + findings: Vec::new(), + findings_omitted: 0, + } + } + + fn push_finding(&mut self, finding: WakeupFinding) { + if self.findings.len() < MAX_FINDINGS { + self.findings.push(finding); + } else { + self.findings_omitted += 1; + } + } + + /// Whether the run found anything that fails it. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// Total findings, including any the report dropped. + pub fn violations(&self) -> u64 { + self.findings.len() as u64 + self.findings_omitted + } + + /// The p99 wakeup delay on the killed executor's waiters, during the fault: + /// the one number this scenario exists to produce. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|cell| { + cell.group == WaiterGroup::OnKilledExecutor && cell.window == Window::DuringFault + }) + .map(|cell| cell.delay.p99_ms) + } + + /// Lines an operator has to act on. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.has_violations() { + lines.push(format!( + "S11 found {} promise-wakeup violations: {}", + self.violations(), + self.findings + .iter() + .take(3) + .map(|f| format!("{} on {}", f.violation, f.token)) + .collect::>() + .join(", ") + )); + } + if !self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 left {} waiters wedged: {} — each stopped producing rounds and then \ + answered no read, which is a worker still parked on a resolved promise", + self.waiters_wedged.len(), + self.waiters_wedged.join(", ") + )); + } + if !self.waiters_unreadable.is_empty() || !self.waiters_truncated.is_empty() { + lines.push(format!( + "S11 could not take a whole account from {} waiters ({} unreadable, {} \ + truncated), so {} accepted completions are unverified either way", + self.waiters_unreadable.len() + self.waiters_truncated.len(), + self.waiters_unreadable.len(), + self.waiters_truncated.len(), + self.unverifiable + )); + } + if self.client_stalled_total > 0 { + lines.push(format!( + "S11: {} rounds woke on time and the caller was not told for up to {}ms — {} of \ + them only got an answer by retrying. The waiters' own logs say the platform \ + resumed them promptly, so this is the response path, not the wakeup", + self.client_stalled_total, self.client_stall_worst_ms, self.client_stall_retried + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "S11 recorded {} wakeups whose token no completion claims — agent names carry \ + the run nonce, so this should be impossible", + self.unknown_tokens + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.waiters_stood_down > 0 && self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 stood {} waiters down after a slow wakeup, and every one of them was \ + readable afterwards — late rather than lost", + self.waiters_stood_down + )); + } + if self.indeterminate_that_woke > 0 { + lines.push(format!( + "S11 had {} completions fail in a way that proves nothing, whose waiters woke \ + anyway", + self.indeterminate_that_woke + )); + } + if self.inconclusive > 0 { + lines.push(format!( + "S11 had {} completions that neither succeeded nor demonstrably landed, whose \ + waiters did not wake — not losses, because the completion may never have \ + arrived", + self.inconclusive + )); + } + lines + } +} + +/// Delays accumulated for one cell before they become percentiles. +#[derive(Default)] +struct DelayCell { + delays: Vec, + parked: Vec, + client_excess: Vec, + client_stalled: u64, + min_delay_ms: i64, + over_budget: u64, + any: bool, +} + +impl DelayCell { + fn push(&mut self, delay_ms: i64, parked_ms: i64, budget_ms: u64) { + if !self.any || delay_ms < self.min_delay_ms { + self.min_delay_ms = delay_ms; + } + self.any = true; + let clamped = delay_ms.max(0) as u64; + if clamped > budget_ms { + self.over_budget += 1; + } + self.delays.push(clamped); + self.parked.push(parked_ms.max(0) as u64); + } + + /// Records how much longer the caller waited than the platform took. + fn push_client(&mut self, excess_ms: u64, budget_ms: u64) { + if excess_ms > budget_ms { + self.client_stalled += 1; + } + self.client_excess.push(excess_ms); + } + + fn into_stats(self, group: WaiterGroup, window: Window) -> WakeupDelayStats { + WakeupDelayStats { + group, + window, + delay: LatencyStats::from_durations(self.delays), + min_delay_ms: self.min_delay_ms, + over_budget: self.over_budget, + parked: LatencyStats::from_durations(self.parked), + client_excess: LatencyStats::from_durations(self.client_excess), + client_stalled: self.client_stalled, + } + } +} + +/// What the driver's own `wait` invocation cost, per round. +/// +/// Keyed by the round's token: the `wait` operation is recorded under +/// `{token}-wait`, which is what lets the caller's view be joined to the +/// waiter's own. +struct WaitObservation { + duration_ms: u64, + attempts: u32, +} + +fn wait_observations(records: &[OperationRecord]) -> BTreeMap<&str, WaitObservation> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == WAIT_METHOD) + .filter_map(|r| { + r.idempotency_key + .strip_suffix(WAIT_KEY_SUFFIX) + .map(|token| { + ( + token, + WaitObservation { + duration_ms: r.duration_ms, + attempts: r.attempts, + }, + ) + }) + }) + .collect() +} + +/// The completion operations, which are the rounds this report is about. +fn completions(records: &[OperationRecord]) -> impl Iterator> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == COMPLETE_METHOD) + .map(|r| Round { + token: r.idempotency_key.as_str(), + agent: r.agent.as_str(), + outcome: r.outcome, + submitted_at: r.submitted_at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::Phase; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn at_ms(millis: i64) -> DateTime { + DateTime::from_timestamp_millis(millis).unwrap() + } + + /// The fault: injected at 100s, cleared at 200s. + fn fault() -> Option { + Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }) + } + + fn split_of(on_pod: &[&str], elsewhere: &[&str]) -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: on_pod.iter().map(|s| s.to_string()).collect(), + elsewhere: elsewhere.iter().map(|s| s.to_string()).collect(), + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn completion( + agent: &str, + token: &str, + submitted: DateTime, + outcome: Outcome, + ) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Fault, + agent: agent.to_string(), + method: COMPLETE_METHOD.to_string(), + idempotency_key: token.to_string(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + /// A whole log: `wakes` matches the entries, so an absent wakeup is a + /// statement rather than a gap. + fn log(agent: &str, wakeups: Vec) -> WaiterWakeupLog { + WaiterWakeupLog { + agent: agent.to_string(), + wakes: Some(wakeups.len() as u64), + wakeups, + error: None, + } + } + + fn wakeup(token: &str, armed: DateTime, woken: DateTime) -> WakeupRecord { + WakeupRecord { + token: token.to_string(), + armed_at: armed, + woken_at: woken, + } + } + + fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + stood_down: u64, + ) -> WakeupReport { + WakeupReport::build( + records, + logs, + split, + fault(), + Duration::from_secs(5), + Duration::from_secs(60), + stood_down, + ) + } + + #[test] + fn a_completion_paired_with_one_wakeup_is_clean() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + assert!(!report.has_violations()); + } + + /// The headline failure. An accepted completion is a promise the platform + /// made; a whole log with no entry for it is the platform not keeping it. + #[test] + fn an_accepted_completion_that_never_woke_its_waiter_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.findings[0].token, "t-1"); + assert_eq!(report.findings[0].window, Window::DuringFault); + } + + /// A truncated log cannot testify, so the same missing wakeup proves + /// nothing. Calling it a loss would turn a component-side cap into a + /// platform defect. + #[test] + fn a_missing_wakeup_on_a_truncated_log_is_unverifiable_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: Some(9000), + wakeups: Vec::new(), + error: None, + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.waiters_truncated, vec!["w-1".to_string()]); + } + + /// The wedged case, and the one place where an unreadable agent convicts + /// rather than excuses: the workload watched this waiter stop producing, and + /// the read-back then could not reach it. Both are symptoms of one worker + /// still parked on a promise that was resolved. + #[test] + fn a_waiter_that_stood_down_and_then_answered_nothing_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 1); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.waiters_wedged, vec!["w-1".to_string()]); + assert_eq!(report.unverifiable, 0); + } + + /// The same unreadable waiter, with nothing having stood down, is only an + /// unreadable waiter. A read that timed out on an otherwise healthy run says + /// nothing about whether the completion landed. + #[test] + fn an_unreadable_waiter_that_never_stood_down_is_unverifiable() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert!(report.waiters_wedged.is_empty()); + } + + #[test] + fn one_completion_and_two_wakeups_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-1", at(105), at(160)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeMoreThanOnce + ); + } + + #[test] + fn a_wakeup_for_a_refused_completion_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Rejected)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeDespiteRejection + ); + assert_eq!(report.completions_rejected, 1); + } + + /// A pod kill produces indeterminate completions by the dozen. One that woke + /// anyway is the platform resolving the doubt in its own favour, and the + /// report says so rather than counting it beside the confirmed ones. + #[test] + fn an_indeterminate_completion_that_woke_is_recorded_separately() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.indeterminate_that_woke, 1); + assert_eq!(report.woke_once, 0); + assert!(!report.has_violations()); + } + + /// And one that did not wake proves nothing at all: from the client side a + /// dropped connection is indistinguishable from a request that arrived. + #[test] + fn an_indeterminate_completion_that_never_woke_is_inconclusive_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.inconclusive, 1); + assert!(!report.has_violations()); + } + + /// The control group is the whole reason the kill is aimed. Mixing the two + /// would let a recovery that took its full budget hide behind the waiters + /// that were never touched. + #[test] + fn delays_are_split_by_group_and_by_window() { + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + completion("w-2", "t-2", at(110), Outcome::Confirmed), + completion("w-1", "t-3", at(50), Outcome::Confirmed), + ]; + let logs = vec![ + log( + "w-1", + vec![ + wakeup("t-1", at(105), at(150)), + wakeup("t-3", at(45), at(51)), + ], + ), + log("w-2", vec![wakeup("t-2", at(105), at(111))]), + ]; + let report = build(&records, &logs, &split_of(&["w-1"], &["w-2"]), 0); + + let killed_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::OnKilledExecutor && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(killed_during.delay.max_ms, 40_000); + + let control_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::Elsewhere && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(control_during.delay.max_ms, 1_000); + + assert!(report.delay.iter().any(|c| c.window == Window::BeforeFault)); + assert_eq!(report.fault_window_p99_ms(), Some(40_000)); + } + + /// The driver and the executor keep different clocks, so a wakeup can look + /// as though it happened before the completion that caused it. That is skew, + /// and it has to be visible rather than clamped silently into a percentile. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let records = vec![completion("w-1", "t-1", at_ms(110_000), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![wakeup("t-1", at_ms(105_000), at_ms(109_700))], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -300); + assert_eq!(cell.delay.max_ms, 0); + } + + /// `parked` is stamped at both ends by the executor, so it is the one number + /// in the report free of that skew — which is what makes it worth carrying + /// beside the delay rather than instead of it. + #[test] + fn the_parked_interval_is_reported_on_the_executors_own_clock() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(150))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].parked.max_ms, 45_000); + assert_eq!(report.dwell_ms, 5_000); + } + + #[test] + fn a_delay_past_the_budget_is_counted_without_failing_the_run() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(180))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].over_budget, 1); + assert!(!report.has_violations()); + } + + /// Agent names carry the run nonce, so a wakeup nobody asked for should be + /// impossible. If it happens the report says so instead of dropping it. + #[test] + fn a_wakeup_no_completion_claims_is_counted() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-stray", at(105), at(112)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.unknown_tokens, 1); + assert_eq!(report.wakeups_recorded, 2); + } + + /// `arm` and `wait` are recorded for the timeline. Counting them as rounds + /// would treble every total in the report. + #[test] + fn only_the_completion_operations_are_counted_as_rounds() { + let mut armed = completion("w-1", "t-1-arm", at(109), Outcome::Confirmed); + armed.method = "arm".to_string(); + let mut waited = completion("w-1", "t-1-wait", at(109), Outcome::Confirmed); + waited.method = "wait".to_string(); + let records = vec![ + armed, + waited, + completion("w-1", "t-1", at(110), Outcome::Confirmed), + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + } + + /// A run that lost everything must still produce an artifact somebody can + /// open, and must say that it truncated rather than leaving a suspiciously + /// round number of findings. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 10) + .map(|i| completion("w-1", &format!("t-{i}"), at(110), Outcome::Confirmed)) + .collect(); + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + assert_eq!(report.violations(), MAX_FINDINGS as u64 + 10); + } + + /// The caller's view and the waiter's own can disagree, and when they do the + /// waiter's is the one that describes the platform. + /// + /// This is not hypothetical: the first completed S11 run woke every one of + /// 21,863 completions on time, and 89 of those callers were not told for 125 + /// seconds. A report with only [`WakeupReport::delay`] in it would have + /// called that run flawless. + #[test] + fn a_wakeup_the_caller_was_not_told_about_is_counted_separately_from_the_delay() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + // Parked 5s, then 125s before the caller heard anything back. + waited.duration_ms = 130_000; + waited.attempts = 2; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + // The waiter itself woke one second after the completion. + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + // The platform did its job, and the delay table says so. + assert_eq!(report.woke_once, 1); + assert_eq!(report.delay[0].delay.max_ms, 1_000); + assert_eq!(report.delay[0].over_budget, 0); + assert!(!report.has_violations()); + + // And the caller still waited two minutes past that. + assert_eq!(report.client_stalled_total, 1); + assert_eq!(report.client_stall_worst_ms, 124_000); + assert_eq!(report.client_stall_retried, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("response path, not the wakeup")) + ); + } + + /// A healthy round's caller waits the dwell plus the wakeup and nothing + /// more, so it must not be counted as stalled. + #[test] + fn a_prompt_round_records_no_client_stall() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + waited.duration_ms = 5_040; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(110))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.client_stalled_total, 0); + assert!(report.attention_lines().is_empty()); + } + + /// Standing waiters down is normal on a run where recovery was slow. It only + /// becomes an attention line when those waiters then could not be read. + #[test] + fn waiters_that_stood_down_but_answered_afterwards_are_context_not_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(190))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 3); + + assert!(report.attention_lines().is_empty()); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains("late rather than lost")) + ); + } +} diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index d702805951..e5545ca6a6 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -62,13 +62,29 @@ use tokio::sync::Semaphore; use tokio::task::JoinSet; use tracing::{debug, info, warn}; +/// The agent an RPC caller invokes, which is *not* the agent the driver invokes +/// (GOL-368). +/// +/// The suffix is not the driver's choice. `Counter::increment_through_rpc` +/// builds its client as `CounterClient::get(format!("{}-inner", self.id))`, so +/// the callee's id follows from the caller's. The driver reproduces that +/// derivation rather than passing a target in, which is what lets it decide +/// *before* the run which executor will own each half of the pair. +/// +/// Free rather than a method on [`WorkloadContext`] because it depends on +/// nothing about the run — only on what the component does — and because that +/// is what makes the coupling testable without standing up a platform. +pub fn rpc_callee_name(caller: &str) -> String { + format!("{caller}-inner") +} + /// Agent type names exported by the counters component. -const COUNTER_AGENT: &str = "Counter"; +pub(crate) const COUNTER_AGENT: &str = "Counter"; const EPHEMERAL_COUNTER_AGENT: &str = "EphemeralCounter"; -const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; -const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; +pub(crate) const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; +pub(crate) const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; const PROMISE_AGENT: &str = "PromiseAgent"; -const QUOTA_COUNTER_AGENT: &str = "QuotaCounter"; +pub(crate) const QUOTA_COUNTER_AGENT: &str = "QuotaCounter"; /// How far ahead scheduled polls are registered. Long enough that registration /// and firing are distinct events (so a fault can land between them), short @@ -113,6 +129,10 @@ const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120); /// a fault and a slow answer is still an answer. const READ_TIMEOUT: Duration = Duration::from_secs(30); +/// Ceiling on one fire-log read. Larger than [`READ_TIMEOUT`] because the +/// answer carries every fire the target recorded rather than one number. +const FIRE_READ_TIMEOUT: Duration = Duration::from_secs(60); + /// Payload written when completing a promise. const PROMISE_PAYLOAD: &[u8] = b"chaos"; @@ -177,6 +197,12 @@ impl WorkloadContext { format!("{}-scheduled-target-{index:04}", self.key_prefix) } + /// The suspended-waiter scenario's agent, in the promise component rather + /// than the counters one (GOL-377). + pub fn waiter_name(&self, index: u32) -> String { + format!("{}-promise-waiter-{index:04}", self.key_prefix) + } + /// The deterministic key for one operation. Same inputs, same key — that is /// what makes a retry the same operation rather than a new one. pub fn idempotency_key(&self, agent: &str, seq: u64) -> String { @@ -192,6 +218,28 @@ struct AttemptResult { class: Option, } +/// What one operation ended up as. +/// +/// Returned rather than only recorded because one scenario has to chain on it: +/// [`crate::chaos::reverts`] judges a revert by the value the *next* increment +/// reports, so it needs each operation's answer in hand rather than having to +/// go looking for its own record in a history every other emitter is also +/// appending to. +#[derive(Debug, Clone)] +pub struct OperationOutcome { + pub outcome: Outcome, + /// The value the operation returned, for the methods that return one. + pub value: Option, + /// The last error, verbatim, for callers that need to tell one refusal from + /// another. [`crate::chaos::deletions`] does: a delete refused because the + /// agent was not there means something quite different from any other + /// refusal. + /// + /// Only ever `Some` on a failure, so the clone costs nothing on the path + /// every scenario actually runs. + pub error: Option, +} + /// Runs one operation with the configured bounded, same-key retry, and records /// it in the history. /// @@ -207,7 +255,8 @@ pub(crate) async fn run_operation( method: &str, key: String, invoke: F, -) where +) -> OperationOutcome +where F: Fn(IdempotencyKey) -> Fut, Fut: std::future::Future>>, { @@ -309,6 +358,7 @@ pub(crate) async fn run_operation( ); } + let error = last.error.as_ref().map(|e| format!("{e:#}")); ctx.history.record(OperationRecord { op_id, stream, @@ -323,10 +373,49 @@ pub(crate) async fn run_operation( duration_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64, returned_value: last.value, first_attempt_value, - error: last.error.as_ref().map(|e| format!("{e:#}")), + error: error.clone(), error_class: last.class, attempt_log, }); + + OperationOutcome { + outcome, + value: last.value, + error, + } +} + +/// One `Counter.increment`, recorded under `stream`. +/// +/// Split out of [`submit_one`] so [`crate::chaos::reverts`] can drive +/// increments on its own stream through exactly the same retry rule and +/// classification, rather than growing a second copy of them. +pub(crate) async fn increment_counter( + ctx: &WorkloadContext, + stream: Stream, + agent: &str, + key: String, +) -> OperationOutcome { + let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); + let ctx2 = ctx.clone(); + run_operation(ctx, stream, agent.to_string(), "increment", key, |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + async move { + let value = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "increment", + data_value!(), + ) + .await?; + Ok(as_u32(value)) + } + }) + .await } /// Extracts a `u32` return value, if the agent returned one. Absent values are @@ -384,6 +473,7 @@ pub fn start(ctx: WorkloadContext, config: &crate::chaos::WorkloadConfig) -> Wor (Stream::Scheduled, config.scheduled_agents), (Stream::Promise, config.promise_agents), (Stream::Quota, config.quota_agents), + (Stream::Rpc, config.rpc_agents), ] .into_iter() .filter(|(_, agents)| *agents > 0) @@ -481,6 +571,25 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 Stream::PinnedHttp => { warn!("Chaos mixed workload cannot drive the pinned stream; see chaos::pinned"); } + // Driven by `crate::chaos::waiters` for the same reason: its agents run + // a round at their own pace rather than at a shared rate, and each one + // is blocked until its promise resolves. + Stream::PromiseWait => { + warn!("Chaos mixed workload cannot drive the waiter stream; see chaos::waiters"); + } + // Driven by `crate::chaos::deletions`: a round builds an agent up and + // then deletes it outright, and the value the *next* round's first + // increment returns says whether it stayed deleted. + Stream::Delete => { + warn!("Chaos mixed workload cannot drive the delete stream; see chaos::deletions"); + } + // Driven by `crate::chaos::reverts`: a round is a run of increments + // followed by a revert that takes some of them back, and the value the + // *next* round's first increment returns is what says whether that + // revert landed. A shared rate cannot express that ordering. + Stream::Revert => { + warn!("Chaos mixed workload cannot drive the revert stream; see chaos::reverts"); + } Stream::Durable => { let agent = ctx.agent_name(Stream::Durable, index); let key = ctx.idempotency_key(&agent, seq); @@ -506,6 +615,43 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 }) .await; } + Stream::Rpc => { + // The driver invokes the caller; the caller invokes the callee. + // Only the second hop is the one under test, and it is the hop the + // driver cannot address directly — which is the whole point, since + // an agent-to-agent call is the only traffic that chooses its own + // executor rather than being routed to one. + let agent = ctx.agent_name(Stream::Rpc, index); + let key = ctx.idempotency_key(&agent, seq); + let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.clone()); + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + run_operation( + ctx, + Stream::Rpc, + agent.clone(), + "increment_through_rpc", + key, + |k| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + async move { + let value = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "increment_through_rpc", + data_value!(), + ) + .await?; + Ok(as_u32(value)) + } + }, + ) + .await; + } Stream::Ephemeral => { let agent = ctx.agent_name(Stream::Ephemeral, index); let key = ctx.idempotency_key(&agent, seq); @@ -671,15 +817,24 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 /// A timeout is reported as an unreadable agent rather than propagated: the /// read-back already models "could not be read" as a verdict of its own, and an /// agent that will not answer is exactly that. -async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +where + F: std::future::Future>, +{ + read_within(what, agent, READ_TIMEOUT, read).await +} + +/// [`read_with_timeout`] with the ceiling named by the caller, for reads whose +/// answer is much bigger than a counter. +async fn read_within(what: &str, agent: &str, timeout: Duration, read: F) -> Result where - F: std::future::Future>, + F: std::future::Future>, { - match tokio::time::timeout(READ_TIMEOUT, read).await { + match tokio::time::timeout(timeout, read).await { Ok(result) => result, Err(_) => { - warn!("Chaos: reading {what} on {agent} timed out after {READ_TIMEOUT:?}"); - Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + warn!("Chaos: reading {what} on {agent} timed out after {timeout:?}"); + Err(format!("{what} timed out after {timeout:?}")) } } } @@ -780,6 +935,34 @@ pub async fn read_quota_counter(ctx: &WorkloadContext, agent: &str) -> Result Result, String> { + read_within("fires", agent, FIRE_READ_TIMEOUT, async { + let parsed: ParsedAgentId = agent_id!(SCHEDULE_COUNTER_AGENT, agent.to_string()); + match ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "fires", data_value!()) + .await + { + Ok(value) => value + .into_return_value() + .and_then(|v| Vec::<(String, u64, u64)>::from_value(v).ok()) + .ok_or_else(|| "fires returned no readable value".to_string()), + Err(e) => Err(format!("{e:#}")), + } + }) + .await +} + /// Reads back how many scheduled polls actually fired on a target agent. pub async fn read_polls(ctx: &WorkloadContext, agent: &str) -> Result { read_with_timeout("polls", agent, async { @@ -813,6 +996,24 @@ mod tests { (agent, target, key) } + /// The callee's id is fixed by the component, not by the driver. + /// + /// `Counter::increment_through_rpc` builds its client as + /// `CounterClient::get(format!("{}-inner", self.id))` in + /// `test-components/agent-counters/src/lib.rs`, and the driver reproduces + /// that derivation to decide which executor owns each half of a pair + /// *before* the run. Nothing in the type system links the two sides, and + /// they live in different crates, so a change to that suffix would silently + /// leave S2 pairing agents that never call each other — a run that then + /// reports a clean control while testing nothing. + #[test] + fn the_rpc_callee_name_matches_what_the_component_derives() { + assert_eq!( + rpc_callee_name("chaos-s2-1234-1-rpc-0007"), + "chaos-s2-1234-1-rpc-0007-inner" + ); + } + #[test] fn phase_marker_round_trips_every_phase() { let marker = PhaseMarker::new(Phase::Baseline); diff --git a/test-components/agent-counters-v2/src/lib.rs b/test-components/agent-counters-v2/src/lib.rs index 83e5a4f656..6e448fb3b0 100644 --- a/test-components/agent-counters-v2/src/lib.rs +++ b/test-components/agent-counters-v2/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] diff --git a/test-components/agent-counters/src/lib.rs b/test-components/agent-counters/src/lib.rs index 75a4d0511c..8c38771bd1 100644 --- a/test-components/agent-counters/src/lib.rs +++ b/test-components/agent-counters/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] diff --git a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs index 6767bfd62e..0edc3e3a5d 100644 --- a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs +++ b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs @@ -30,3 +30,114 @@ impl PromiseAgent for PromiseAgentImpl { golem_rust::complete_promise(&promise_id, &vec![0; payload_size as usize]) } } + +/// Ceiling on the wakeup log below. +/// +/// Far above what a chaos run produces — a few hundred wakeups per waiter — so +/// it only stops a misconfigured cadence from growing agent state without +/// bound. `wakes` keeps counting past it, which is what makes truncation +/// visible: a reader that gets fewer entries than wakes knows the log is short +/// rather than the wakeups missing. +const MAX_WAKEUP_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S11 asks is +/// when the *platform* resumed the waiter. The read is durable, so an agent that +/// replays reports the original wakeup time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + +/// A durable agent that parks on a promise and records being woken (GOL-377). +/// +/// [`PromiseAgent`] above creates and completes promises in one breath, which is +/// what the mixed workload's promise stream needs and says nothing about +/// recovery. S11 needs the other half: an agent genuinely suspended on a promise +/// whose executor is then killed, and a durable record of whether the completion +/// ever reached it. +/// +/// The record has to live in agent state rather than in the invocation's return +/// value, because the interesting case is exactly the one where the caller's +/// connection died with the executor. A caller that got an error learns nothing; +/// the log outlives the connection and answers anyway. +#[agent_definition] +pub trait PromiseWaiter { + fn new(name: String) -> Self; + + /// Creates a promise for this waiter and returns it without waiting. + /// + /// Separate from [`PromiseWaiter::wait`] because the completer needs the + /// promise id, and an invocation that blocks cannot return one. `token` is + /// the driver's idempotency key for this round, and is what pairs a wakeup + /// back to the completion that caused it. + fn arm(&mut self, token: String) -> PromiseId; + + /// Blocks until `promise_id` is completed, then records the wakeup. + /// + /// Suspends the agent, so the executor holds no thread for it: this is the + /// state S11 kills an executor in. + fn wait(&mut self, token: String, promise_id: PromiseId); + + /// The wakeup log: one `(token, armed_millis, woken_millis)` per wakeup, in + /// the order the waiter was resumed. + fn wakeups(&self) -> Vec<(String, u64, u64)>; + + /// How many wakeups happened, whether or not the log kept them. + fn wakes(&self) -> u32; +} + +struct PromiseWaiterImpl { + _name: String, + /// When each token was armed, so a wakeup can report the interval it was + /// parked without the driver having to join two clocks. + armed: Vec<(String, u64)>, + wakeups: Vec<(String, u64, u64)>, + wakes: u32, +} + +#[agent_implementation] +impl PromiseWaiter for PromiseWaiterImpl { + fn new(name: String) -> Self { + Self { + _name: name, + armed: Vec::new(), + wakeups: Vec::new(), + wakes: 0, + } + } + + fn arm(&mut self, token: String) -> PromiseId { + if self.armed.len() >= MAX_WAKEUP_LOG { + self.armed.remove(0); + } + self.armed.push((token, now_millis())); + golem_rust::create_promise() + } + + fn wait(&mut self, token: String, promise_id: PromiseId) { + let _ = golem_rust::blocking_await_promise(&promise_id); + self.wakes += 1; + if self.wakeups.len() < MAX_WAKEUP_LOG { + let armed_millis = self + .armed + .iter() + .rev() + .find(|(armed_token, _)| armed_token == &token) + .map(|(_, at)| *at) + .unwrap_or(0); + self.wakeups.push((token, armed_millis, now_millis())); + } + } + + fn wakeups(&self) -> Vec<(String, u64, u64)> { + self.wakeups.clone() + } + + fn wakes(&self) -> u32 { + self.wakes + } +}