From 00405f1dc6cf020e9e9fc589fc5e1e47d0d77415 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:01:58 +0200 Subject: [PATCH 01/34] other(benchmarks): scaffold SDKConfigBenchmark tuist project Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmark/Project.swift | 90 +++++++++++++ .../Networking/HTTPClient/HTTPClient.swift | 3 + .../SDKConfigBenchmark/Main/main.swift | 3 + .../Sources/BenchmarkCommand.swift | 127 ++++++++++++++++++ .../Sources/BenchmarkError.swift | 23 ++++ .../Sources/BenchmarkMain.swift | 26 ++++ .../SimulatedTransportURLProtocol.swift | 19 +++ .../Tests/BenchmarkCommandTests.swift | 73 ++++++++++ .../Tests/BenchmarkTestCase.swift | 7 + Workspace.swift | 3 +- 10 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 Projects/SDKConfigBenchmark/Project.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift diff --git a/Projects/SDKConfigBenchmark/Project.swift b/Projects/SDKConfigBenchmark/Project.swift new file mode 100644 index 0000000000..e257a09a0b --- /dev/null +++ b/Projects/SDKConfigBenchmark/Project.swift @@ -0,0 +1,90 @@ +import ProjectDescription +import ProjectDescriptionHelpers + +// Benchmark harness for comparing the legacy offerings flow against the remote config +// endpoint flow. Compiles the SDK sources directly (like BinarySizeTest does for its +// local-source mode) so the benchmark can drive internal manager-level APIs without +// exposing new public SDK API. `SDK_CONFIG_BENCHMARK` installs a simulated transport +// in `HTTPClient`; `ENABLE_REMOTE_CONFIG` turns on the remote config gate. +let benchmarkSwiftConditions: SettingsDictionary = [ + "SWIFT_ACTIVE_COMPILATION_CONDITIONS": "$(inherited) SDK_CONFIG_BENCHMARK ENABLE_REMOTE_CONFIG" +] + +let project = Project( + name: "SDKConfigBenchmark", + organizationName: .revenueCatOrgName, + settings: .framework, + targets: [ + .target( + name: "SDKConfigBenchmarkCore", + destinations: [.mac], + product: .staticLibrary, + bundleId: "com.revenuecat.SDKConfigBenchmarkCore", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Sources/**/*.swift", + .glob( + "../../Sources/**/*.swift", + excluding: [ + "../../Sources/LocalReceiptParsing/ReceiptParser-only-files/**/*.swift" + ] + ) + ], + dependencies: [ + .storeKit + ], + settings: .settings( + base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + ) + ), + .target( + name: "SDKConfigBenchmark", + destinations: [.mac], + product: .commandLineTool, + bundleId: "com.revenuecat.SDKConfigBenchmark", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Main/**/*.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkCore") + ], + settings: .settings( + base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + ) + ), + .target( + name: "SDKConfigBenchmarkTests", + destinations: [.mac], + product: .unitTests, + bundleId: "com.revenuecat.SDKConfigBenchmarkTests", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Tests/**/*.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkCore") + ], + settings: .settings( + base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + ) + ) + ], + schemes: [ + .scheme( + name: "SDKConfigBenchmark", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmark"]), + runAction: .runAction(configuration: "Release") + ), + .scheme( + name: "SDKConfigBenchmarkTests", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmarkTests"]), + testAction: .targets(["SDKConfigBenchmarkTests"]) + ) + ] +) diff --git a/Sources/Networking/HTTPClient/HTTPClient.swift b/Sources/Networking/HTTPClient/HTTPClient.swift index 6fc130debc..4dc475d6a0 100644 --- a/Sources/Networking/HTTPClient/HTTPClient.swift +++ b/Sources/Networking/HTTPClient/HTTPClient.swift @@ -58,6 +58,9 @@ class HTTPClient { config.timeoutIntervalForRequest = requestTimeout config.timeoutIntervalForResource = requestTimeout config.urlCache = nil // We implement our own caching with `ETagManager`. + #if SDK_CONFIG_BENCHMARK + config.protocolClasses = [SimulatedTransportURLProtocol.self] + #endif self.session = URLSession(configuration: config, delegate: RedirectLoggerSessionDelegate(), delegateQueue: nil) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift b/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift new file mode 100644 index 0000000000..ea59f7ac5b --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift @@ -0,0 +1,3 @@ +import SDKConfigBenchmarkCore + +BenchmarkMain.run() diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift new file mode 100644 index 0000000000..696cf31287 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -0,0 +1,127 @@ +import Foundation + +enum BenchmarkMode: String, CaseIterable { + + case legacy + case config + case configKillswitch = "config-killswitch" + + /// Whether a `RemoteConfigManager` stack is wired into the offerings flow. + var usesRemoteConfig: Bool { + switch self { + case .legacy: return false + case .config, .configKillswitch: return true + } + } + +} + +enum BenchmarkScenario: String, CaseIterable { + + /// Every iteration starts with empty disk caches and a fresh app user ID. + case cold + + /// Disk caches are primed once, then every iteration simulates an app relaunch: + /// fresh in-memory state, retained disk state, same app user ID. The fixture + /// serves 304 (offerings) and 204 (config) when the SDK proves it has current data. + case warm + +} + +struct BenchmarkCommand { + + var mode: BenchmarkMode = .legacy + var scenario: BenchmarkScenario = .cold + var profileName: String = "ideal" + var lossPercent: Int = 0 + var iterations: Int = 25 + var warmupIterations: Int = 3 + var paywallCount: Int = 50 + var workflowCount: Int = 100 + var seed: UInt64 = 42 + var appUserID: String = "benchmark-user" + var apiKey: String = "appl_benchmark" + /// Extra key=value pairs echoed verbatim into the JSONL row (e.g. sdk_commit=abc123). + var annotations: [String: String] = [:] + + // swiftlint:disable:next cyclomatic_complexity function_body_length + static func parse(_ args: [String]) throws -> BenchmarkCommand { + var command = BenchmarkCommand() + var index = 0 + + func value(for flag: String) throws -> String { + index += 1 + guard index < args.count else { + throw BenchmarkError.invalidArgument("\(flag) requires a value") + } + return args[index] + } + + func intValue(for flag: String, in range: ClosedRange) throws -> Int { + let raw = try value(for: flag) + guard let parsed = Int(raw), range.contains(parsed) else { + throw BenchmarkError.invalidArgument("\(flag) must be an integer in \(range), got \(raw)") + } + return parsed + } + + while index < args.count { + let flag = args[index] + switch flag { + case "--mode": + let raw = try value(for: flag) + guard let mode = BenchmarkMode(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown mode \(raw)") + } + command.mode = mode + case "--scenario": + let raw = try value(for: flag) + guard let scenario = BenchmarkScenario(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown scenario \(raw)") + } + command.scenario = scenario + case "--profile": + command.profileName = try value(for: flag) + case "--loss-percent": + command.lossPercent = try intValue(for: flag, in: 0...100) + case "--iterations": + command.iterations = try intValue(for: flag, in: 1...100_000) + case "--warmup-iterations": + command.warmupIterations = try intValue(for: flag, in: 0...1_000) + case "--paywalls": + command.paywallCount = try intValue(for: flag, in: 1...10_000) + case "--workflows": + command.workflowCount = try intValue(for: flag, in: 0...10_000) + case "--seed": + let raw = try value(for: flag) + guard let seed = UInt64(raw) else { + throw BenchmarkError.invalidArgument("--seed must be an unsigned integer, got \(raw)") + } + command.seed = seed + case "--app-user-id": + command.appUserID = try value(for: flag) + case "--api-key": + command.apiKey = try value(for: flag) + case "--annotation": + let raw = try value(for: flag) + let parts = raw.split(separator: "=", maxSplits: 1).map(String.init) + guard parts.count == 2, !parts[0].isEmpty else { + throw BenchmarkError.invalidArgument("--annotation expects key=value, got \(raw)") + } + command.annotations[parts[0]] = parts[1] + default: + throw BenchmarkError.invalidArgument("unknown flag \(flag)") + } + index += 1 + } + + guard command.warmupIterations < command.iterations else { + throw BenchmarkError.invalidArgument( + "--warmup-iterations (\(command.warmupIterations)) must be below --iterations (\(command.iterations))" + ) + } + + return command + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift new file mode 100644 index 0000000000..94d0823d75 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift @@ -0,0 +1,23 @@ +import Foundation + +enum BenchmarkError: Error, CustomStringConvertible { + + case invalidArgument(String) + case invalidFixture(String) + case unsupportedPath(String) + case timeout(String) + case backendFailure(String) + case scenarioViolation(String) + + var description: String { + switch self { + case let .invalidArgument(message): return "Invalid argument: \(message)" + case let .invalidFixture(message): return "Invalid fixture: \(message)" + case let .unsupportedPath(path): return "Unsupported fixture path: \(path)" + case let .timeout(operation): return "Timed out waiting for \(operation)" + case let .backendFailure(message): return "Backend failure: \(message)" + case let .scenarioViolation(message): return "Scenario violation: \(message)" + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift new file mode 100644 index 0000000000..a1f8fa7848 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift @@ -0,0 +1,26 @@ +import Foundation + +// BenchmarkMain hosts only static members, but it cannot be the caseless enum that +// `convenience_type` wants because the repo bans new public enums outright. +// swiftlint:disable convenience_type + +/// The single public entry point for the `SDKConfigBenchmark` executable target. +/// Everything else in this module stays internal; the executable is one call into here. +public struct BenchmarkMain { + + /// Parses `CommandLine.arguments`, runs the requested benchmark, prints one JSONL row + /// to stdout, and terminates the process (0 on success, 1 on error). + public static func run() -> Never { + do { + let command = try BenchmarkCommand.parse(Array(CommandLine.arguments.dropFirst())) + // Placeholder output until the runner lands; proves parsing and target wiring. + print("parsed mode=\(command.mode.rawValue) scenario=\(command.scenario.rawValue) " + + "profile=\(command.profileName) iterations=\(command.iterations)") + exit(0) + } catch { + FileHandle.standardError.write(Data("\(error)\n".utf8)) + exit(1) + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift new file mode 100644 index 0000000000..2827bcce4f --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -0,0 +1,19 @@ +import Foundation + +/// Placeholder so the `HTTPClient` benchmark hook compiles; replaced with the real +/// simulated transport in a follow-up commit. +final class SimulatedTransportURLProtocol: URLProtocol { + + override class func canInit(with request: URLRequest) -> Bool { + return false + } + + override class func canonicalRequest(for request: URLRequest) -> URLRequest { + return request + } + + override func startLoading() {} + + override func stopLoading() {} + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift new file mode 100644 index 0000000000..526713ab18 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -0,0 +1,73 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkCommandTests: BenchmarkTestCase { + + func testParseDefaults() throws { + let command = try BenchmarkCommand.parse([]) + + XCTAssertEqual(command.mode, .legacy) + XCTAssertEqual(command.scenario, .cold) + XCTAssertEqual(command.profileName, "ideal") + XCTAssertEqual(command.lossPercent, 0) + XCTAssertEqual(command.iterations, 25) + XCTAssertEqual(command.warmupIterations, 3) + XCTAssertEqual(command.paywallCount, 50) + XCTAssertEqual(command.workflowCount, 100) + XCTAssertEqual(command.seed, 42) + XCTAssertTrue(command.annotations.isEmpty) + } + + func testParseFullFlagSet() throws { + let command = try BenchmarkCommand.parse([ + "--mode", "config-killswitch", + "--scenario", "warm", + "--profile", "lte", + "--loss-percent", "20", + "--iterations", "100", + "--warmup-iterations", "5", + "--paywalls", "10", + "--workflows", "25", + "--seed", "7", + "--app-user-id", "user-1", + "--api-key", "appl_x", + "--annotation", "sdk_commit=abc123" + ]) + + XCTAssertEqual(command.mode, .configKillswitch) + XCTAssertTrue(command.mode.usesRemoteConfig) + XCTAssertEqual(command.scenario, .warm) + XCTAssertEqual(command.profileName, "lte") + XCTAssertEqual(command.lossPercent, 20) + XCTAssertEqual(command.iterations, 100) + XCTAssertEqual(command.warmupIterations, 5) + XCTAssertEqual(command.paywallCount, 10) + XCTAssertEqual(command.workflowCount, 25) + XCTAssertEqual(command.seed, 7) + XCTAssertEqual(command.appUserID, "user-1") + XCTAssertEqual(command.apiKey, "appl_x") + XCTAssertEqual(command.annotations["sdk_commit"], "abc123") + } + + func testParseRejectsUnknownMode() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--mode", "turbo"])) + } + + func testParseRejectsUnknownFlag() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--nope"])) + } + + func testParseRejectsMissingValue() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--iterations"])) + } + + func testParseRejectsWarmupNotBelowIterations() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--iterations", "5", "--warmup-iterations", "5"])) + } + + func testParseRejectsOutOfRangeLoss() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--loss-percent", "101"])) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift new file mode 100644 index 0000000000..d07d496af4 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift @@ -0,0 +1,7 @@ +import XCTest + +/// Minimal base class for benchmark harness tests. The benchmark test target does not link +/// the main UnitTests infrastructure, so it carries its own `XCTestCase` subclass to satisfy +/// the repo convention that test classes never inherit `XCTestCase` directly. +// swiftlint:disable:next xctestcase_superclass +class BenchmarkTestCase: XCTestCase {} diff --git a/Workspace.swift b/Workspace.swift index 27bd021419..6924a7b772 100644 --- a/Workspace.swift +++ b/Workspace.swift @@ -12,7 +12,8 @@ var projects: [Path] = [ "./Projects/APITesters", "./Projects/PaywallValidationTester", "./Projects/BinarySizeTest", - "./Projects/RCTTester" + "./Projects/RCTTester", + "./Projects/SDKConfigBenchmark" ] // These projects depend on external packages (Nimble, SnapshotTesting, OHHTTPStubs, GoogleMobileAds). From 7ca95b854b0b19ea878d7ea2f4cd757b8fbc047d Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:03:38 +0200 Subject: [PATCH 02/34] other(benchmarks): add seeded rng and rc-container encoder Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/RCContainerEncoder.swift | 69 +++++++++++++++++++ .../Sources/SeededRandom.swift | 20 ++++++ .../Tests/RCContainerEncoderTests.swift | 69 +++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift new file mode 100644 index 0000000000..614f1e7c9d --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift @@ -0,0 +1,69 @@ +import Foundation + +/// Serializes RC-Container payloads for the fixture server, mirroring the wire format that +/// `RCContainer.Parser` reads: an 8-byte header followed by elements, each with a 24-byte +/// SHA-256-prefix checksum, little-endian wire size, encoding byte, 3 reserved bytes, the +/// payload, and zero padding to an 8-byte boundary. Only the `none` encoding is produced; +/// the benchmark measures transport and decode behavior, not compression codecs. +enum RCContainerEncoder { + + private static let version: UInt8 = 1 + private static let checksumSize = 24 + private static let paddingBoundary = 8 + + /// Builds a container whose first element is the config payload followed by the given + /// inline content elements, matching what `RemoteConfigContainer` expects. + static func container(config: Data, contentElements: [Data]) -> Data { + var data = Data([UInt8(ascii: "R"), UInt8(ascii: "C"), self.version, 0]) + data.append(contentsOf: [0, 0, 0, 0]) + + for element in [config] + contentElements { + self.appendElement(element, to: &data) + } + + return data + } + + /// The externally-referenced blob ref for a payload: SHA-256 truncated to 24 bytes, + /// URL-safe base64 without padding (32 characters). + static func blobRef(for data: Data) -> String { + return self.checksum(for: data) + .base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + +} + +private extension RCContainerEncoder { + + static func checksum(for data: Data) -> Data { + return data.sha256.prefix(self.checksumSize) + } + + static func appendElement(_ payload: Data, to data: inout Data) { + data.append(self.checksum(for: payload)) + data.appendLittleEndianUInt32(UInt32(payload.count)) + data.append(0) // ContentEncoding.none + data.append(contentsOf: [0, 0, 0]) + data.append(payload) + + let remainder = payload.count % self.paddingBoundary + if remainder != 0 { + data.append(Data(repeating: 0, count: self.paddingBoundary - remainder)) + } + } + +} + +private extension Data { + + mutating func appendLittleEndianUInt32(_ value: UInt32) { + self.append(UInt8(value & 0xff)) + self.append(UInt8((value >> 8) & 0xff)) + self.append(UInt8((value >> 16) & 0xff)) + self.append(UInt8((value >> 24) & 0xff)) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift new file mode 100644 index 0000000000..2044efd749 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift @@ -0,0 +1,20 @@ +import Foundation + +/// Deterministic SplitMix64 generator so benchmark runs are reproducible for a given `--seed`. +struct SeededRandom: RandomNumberGenerator { + + private var state: UInt64 + + init(seed: UInt64) { + self.state = seed + } + + mutating func next() -> UInt64 { + self.state &+= 0x9E3779B97F4A7C15 + var value = self.state + value = (value ^ (value >> 30)) &* 0xBF58476D1CE4E5B9 + value = (value ^ (value >> 27)) &* 0x94D049BB133111EB + return value ^ (value >> 31) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift new file mode 100644 index 0000000000..4c7e70e84e --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift @@ -0,0 +1,69 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class RCContainerEncoderTests: BenchmarkTestCase { + + private let config = Data(#"{"domain":"app","manifest":"m1"}"#.utf8) + private let blobA = Data(#"{"id":"blob-a"}"#.utf8) + private let blobB = Data(String(repeating: "x", count: 300).utf8) + + func testEncodedContainerRoundTripsThroughSDKParser() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: [self.blobA, self.blobB]) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements.count, 3) + XCTAssertEqual(try parsed.elements[0].withDecodedPayloadBytes { Data($0) }, self.config) + XCTAssertEqual(try parsed.elements[1].withDecodedPayloadBytes { Data($0) }, self.blobA) + XCTAssertEqual(try parsed.elements[2].withDecodedPayloadBytes { Data($0) }, self.blobB) + } + + func testEncodedElementChecksumsMatchBlobRefs() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: [self.blobA]) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements[1].checksum, RCContainerEncoder.blobRef(for: self.blobA)) + XCTAssertNotNil(parsed.elementsByChecksum[RCContainerEncoder.blobRef(for: self.blobA)]) + } + + func testBlobRefShape() { + let ref = RCContainerEncoder.blobRef(for: self.blobA) + + XCTAssertEqual(ref.count, 32) + XCTAssertNil(ref.rangeOfCharacter(from: CharacterSet(charactersIn: "+/="))) + } + + func testConfigOnlyContainerParses() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: []) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements.count, 1) + } + + func testSeededRandomIsDeterministic() { + var first = SeededRandom(seed: 7) + var second = SeededRandom(seed: 7) + var other = SeededRandom(seed: 8) + + let firstValues = (0..<8).map { _ in first.next() } + let secondValues = (0..<8).map { _ in second.next() } + let otherValues = (0..<8).map { _ in other.next() } + + XCTAssertEqual(firstValues, secondValues) + XCTAssertNotEqual(firstValues, otherValues) + } + + func testSeededRandomUniformDoubleStaysInRange() { + var rng = SeededRandom(seed: 1) + + for _ in 0..<1_000 { + let value = Double.random(in: 0..<1, using: &rng) + XCTAssertGreaterThanOrEqual(value, 0) + XCTAssertLessThan(value, 1) + } + } + +} From 592d6d64eedc762f947d10f50f41e358fd8773cc Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:09:07 +0200 Subject: [PATCH 03/34] other(benchmarks): add payload factory producing real sdk wire shapes Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkPayloadFactory.swift | 269 ++++++++++++++++++ .../Tests/BenchmarkPayloadFactoryTests.swift | 79 +++++ 2 files changed, 348 insertions(+) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift new file mode 100644 index 0000000000..1425125eb8 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift @@ -0,0 +1,269 @@ +import Foundation + +/// Generates the wire payloads the fixture server responds with: the legacy offerings JSON, +/// the `/v1/config/{domain}` RC-Container, and the content-addressed blobs it references. +/// All payloads are deterministic for a given (paywallCount, workflowCount), so byte counts +/// are comparable across runs and branches. +struct BenchmarkPayloadFactory { + + static let blobURLFormat = "https://cdn.revenuecat.local/blobs/{blob_ref}" + + let paywallCount: Int + let workflowCount: Int + let configManifest = "benchmark-manifest-v1" + + let offeringsData: Data + let configContainerData: Data + + private let blobsByRef: [String: Data] + + init(paywallCount: Int, workflowCount: Int) { + self.paywallCount = paywallCount + self.workflowCount = workflowCount + + let builder = PayloadBuilder(paywallCount: paywallCount, workflowCount: workflowCount) + self.offeringsData = builder.offeringsData() + + var blobs: [String: Data] = [:] + var workflowRefs: [String: String] = [:] + for index in 0.. Data? { + return self.blobsByRef[ref] + } + + var allBlobRefs: [String] { + return Array(self.blobsByRef.keys) + } + + var productIdentifiers: Set { + return Set((0.. String { + return "com.revenuecat.benchmark.product_\(index)" + } + + func offeringsData() -> Data { + return self.data([ + "current_offering_id": "offering_0", + "offerings": (0.. Data { + return self.data([ + "id": "workflow_\(index)", + "display_name": "Benchmark Workflow \(index)", + "initial_step_id": "step-1", + "single_step_fallback_id": "step-1", + "steps": [ + "step-1": [ + "id": "step-1", + "type": "paywall", + "screen_id": "screen-1" + ] + ], + "screens": [:], + "metadata": [ + "benchmark_index": index, + "benchmark_payload": String(repeating: "w", count: 256) + ] + ]) + } + + func uiConfigAppBlobData() -> Data { + return self.data([ + "colors": [:], + "fonts": [:] + ]) + } + + func uiConfigLocalizationsBlobData() -> Data { + return self.data([ + "en_US": [ + "benchmark.title": "Benchmark title", + "benchmark.subtitle": "Benchmark subtitle" + ] + ]) + } + + func configJSONData( + manifest: String, + workflowRefsById: [String: String], + uiConfigAppRef: String, + uiConfigLocalizationsRef: String + ) -> Data { + var workflowsTopic: [String: Any] = [:] + for (workflowId, ref) in workflowRefsById { + let index = Int(workflowId.split(separator: "_").last.map(String.init) ?? "0") ?? 0 + workflowsTopic[workflowId] = [ + "offering_identifier": "offering_\(self.paywallCount == 0 ? 0 : index % self.paywallCount)", + "blob_ref": ref, + "prefetch": true + ] + } + + // Sorted so the payload bytes are stable across processes (JSON arrays keep their order). + let prefetchBlobs = workflowRefsById.values.sorted() + [uiConfigAppRef, uiConfigLocalizationsRef] + + return self.data([ + "domain": "app", + "manifest": manifest, + "active_topics": ["sources", "workflows", "ui_config"], + "prefetch_blobs": prefetchBlobs, + "topics": [ + "sources": [ + "api": [ + "sources": [ + ["url": "https://api.revenuecat.com/", "priority": 0, "weight": 100] + ] + ], + "blob": [ + "sources": [ + ["url_format": BenchmarkPayloadFactory.blobURLFormat, "priority": 0, "weight": 100] + ] + ] + ], + "workflows": workflowsTopic, + "ui_config": [ + "app": ["blob_ref": uiConfigAppRef, "prefetch": true], + "localizations": ["blob_ref": uiConfigLocalizationsRef, "prefetch": true] + ] + ] + ]) + } + +} + +private extension PayloadBuilder { + + func offering(index: Int) -> [String: Any] { + let offeringIdentifier = "offering_\(index)" + return [ + "identifier": offeringIdentifier, + "description": "Benchmark offering \(index)", + "packages": [ + [ + "identifier": "$rc_monthly", + "platform_product_identifier": Self.productIdentifier(index: index) + ] + ], + "metadata": [ + "benchmark_index": index, + "benchmark_payload": String(repeating: "m", count: 64) + ], + "paywall_components": self.paywallComponents(id: "paywall_\(index)", offeringIdentifier: offeringIdentifier) + ] + } + + func paywallComponents(id: String, offeringIdentifier: String) -> [String: Any] { + return [ + "id": id, + "default_locale": "en_US", + "revision": 1, + "template_name": "benchmark_components", + "asset_base_url": "https://assets.revenuecat.com", + "components_localizations": [ + "en_US": [ + "title": "Benchmark title for \(offeringIdentifier)", + "cta": "Continue" + ] + ], + "components_config": [ + "base": [ + "stack": self.paywallStack(), + "background": [ + "type": "color", + "value": [ + "light": ["type": "hex", "value": "#FFFFFF"] + ] + ] + ] + ] + ] + } + + func paywallStack() -> [String: Any] { + return [ + "type": "stack", + "components": [self.paywallTitleComponent()], + "dimension": [ + "type": "vertical", + "alignment": "center", + "distribution": "center" + ], + "size": [ + "width": ["type": "fill"], + "height": ["type": "fill"] + ], + "padding": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "margin": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "spacing": 16 + ] + } + + func paywallTitleComponent() -> [String: Any] { + return [ + "type": "text", + "text_lid": "title", + "font_weight": "bold", + "font_size": 24, + "color": [ + "light": ["type": "hex", "value": "#111111"] + ], + "size": [ + "width": ["type": "fill"], + "height": ["type": "fit"] + ], + "padding": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "margin": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "horizontal_alignment": "center" + ] + } + + func data(_ object: Any) -> Data { + do { + return try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + } catch { + preconditionFailure("Invalid benchmark JSON fixture: \(error)") + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift new file mode 100644 index 0000000000..adddede45a --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift @@ -0,0 +1,79 @@ +import XCTest + +@_spi(Internal) @testable import SDKConfigBenchmarkCore + +final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 5, workflowCount: 7) + + func testOfferingsDataDecodesIntoSDKModel() throws { + let response = try JSONDecoder.default.decode(OfferingsResponse.self, from: self.factory.offeringsData) + + XCTAssertEqual(response.offerings.count, 5) + XCTAssertEqual(response.currentOfferingId, "offering_0") + XCTAssertEqual(response.productIdentifiers.count, 5) + + for offering in response.offerings { + let components = try XCTUnwrap(offering.paywallComponents) + XCTAssertNil(components.errorInfo, "paywall components must decode without collected errors") + XCTAssertEqual(components.componentsConfig.base.stack.components.count, 1) + } + } + + func testConfigContainerDecodesIntoRemoteConfiguration() throws { + let container = try RemoteConfigContainer(data: self.factory.configContainerData) + let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } + let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + + XCTAssertEqual(configuration.domain, "app") + XCTAssertEqual(configuration.manifest, self.factory.configManifest) + XCTAssertEqual(Set(configuration.activeTopics), ["sources", "workflows", "ui_config"]) + + let workflowsTopic = try XCTUnwrap(configuration.topics.entries["workflows"]) + XCTAssertEqual(workflowsTopic.count, 7) + for item in workflowsTopic.values { + XCTAssertTrue(item.prefetch) + XCTAssertNotNil(item.blobRef) + } + + let sourcesTopic = try XCTUnwrap(configuration.topics.entries["sources"]) + XCTAssertNotNil(sourcesTopic["api"]) + XCTAssertNotNil(sourcesTopic["blob"]) + } + + func testAllReferencedBlobsResolveAndValidate() throws { + let container = try RemoteConfigContainer(data: self.factory.configContainerData) + let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } + let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + + let referencedRefs = Set(configuration.prefetchBlobs) + XCTAssertEqual(referencedRefs.count, 7 + 2) + + for ref in referencedRefs { + let blob = try XCTUnwrap(self.factory.blobData(forRef: ref), "missing blob for ref \(ref)") + XCTAssertEqual(RCContainerEncoder.blobRef(for: blob), ref, "blob refs must be content-addressed") + } + } + + func testWorkflowBlobDecodesIntoPublishedWorkflow() throws { + let container = try RemoteConfigContainer(data: self.factory.configContainerData) + let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } + let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + + let workflowsTopic = try XCTUnwrap(configuration.topics.entries["workflows"]) + let ref = try XCTUnwrap(workflowsTopic.values.first?.blobRef) + let blob = try XCTUnwrap(self.factory.blobData(forRef: ref)) + + let workflow = try JSONDecoder.default.decode(PublishedWorkflow.self, from: blob) + XCTAssertEqual(workflow.initialStepId, "step-1") + } + + func testPayloadsAreDeterministic() { + let other = BenchmarkPayloadFactory(paywallCount: 5, workflowCount: 7) + + XCTAssertEqual(other.offeringsData, self.factory.offeringsData) + XCTAssertEqual(other.configContainerData, self.factory.configContainerData) + XCTAssertEqual(Set(other.allBlobRefs), Set(self.factory.allBlobRefs)) + } + +} From bb316f6426c4bc43813f8ac08b60e28130ebbad2 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:14:41 +0200 Subject: [PATCH 04/34] other(benchmarks): add simulated network transport and fixture server Async chunked delivery with per-class (api vs cdn) latency, seeded loss model, etag 304 and manifest 204 warm paths, and a config kill-switch 4xx mode. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/FixtureServer.swift | 116 ++++++++ .../Sources/NetworkProfile.swift | 107 ++++++++ .../SimulatedTransportURLProtocol.swift | 253 +++++++++++++++++- .../Tests/FixtureServerTests.swift | 183 +++++++++++++ .../Tests/NetworkProfileTests.swift | 93 +++++++ 5 files changed, 747 insertions(+), 5 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift new file mode 100644 index 0000000000..f289b6721d --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift @@ -0,0 +1,116 @@ +import Foundation + +/// Routes simulated requests to deterministic fixture responses. +/// +/// Routing is host-agnostic (path-suffix based) so the SDK's fallback hosts resolve to the +/// same fixtures as the primary host. Warm paths are supported the same way production is: +/// offerings replies 304 to a matching `X-RevenueCat-ETag`, config replies 204 to a matching +/// manifest token in the POST body. `killSwitchConfig` makes the config endpoint return a 4xx, +/// which trips `RemoteConfigManager`'s session kill switch. +final class FixtureServer { + + struct Response { + let statusCode: Int + let headers: [String: String] + let body: Data + } + + static let offeringsETag = "benchmark-offerings-v1" + + private let factory: BenchmarkPayloadFactory + private let killSwitchConfig: Bool + + init(factory: BenchmarkPayloadFactory, killSwitchConfig: Bool = false) { + self.factory = factory + self.killSwitchConfig = killSwitchConfig + } + + func response(for request: URLRequest, bodyData: Data?) -> Response { + guard let url = request.url else { + return Self.notFound() + } + let path = url.path + + if path.hasSuffix("/offerings"), path.contains("/subscribers/") || path.hasSuffix("/v1/offerings") { + return self.offeringsResponse(for: request) + } + + if path.hasSuffix("/config/app") { + return self.configResponse(bodyData: bodyData) + } + + if let blobRange = path.range(of: "/blobs/") { + let ref = String(path[blobRange.upperBound...]) + return self.blobResponse(forRef: ref) + } + + return Self.notFound() + } + +} + +private extension FixtureServer { + + func offeringsResponse(for request: URLRequest) -> Response { + let requestETag = request.value(forHTTPHeaderField: "X-RevenueCat-ETag") + if requestETag == Self.offeringsETag { + return Response( + statusCode: 304, + headers: self.jsonHeaders(eTag: Self.offeringsETag), + body: Data() + ) + } + return Response( + statusCode: 200, + headers: self.jsonHeaders(eTag: Self.offeringsETag), + body: self.factory.offeringsData + ) + } + + func configResponse(bodyData: Data?) -> Response { + if self.killSwitchConfig { + let errorBody = Data(#"{"code": 7000, "message": "benchmark kill switch"}"#.utf8) + return Response(statusCode: 400, headers: self.jsonHeaders(eTag: nil), body: errorBody) + } + + if let bodyData, + let body = try? JSONSerialization.jsonObject(with: bodyData) as? [String: Any], + body["manifest"] as? String == self.factory.configManifest { + return Response(statusCode: 204, headers: [:], body: Data()) + } + + return Response( + statusCode: 200, + headers: ["Content-Type": "application/octet-stream"], + body: self.factory.configContainerData + ) + } + + func blobResponse(forRef ref: String) -> Response { + guard let blob = self.factory.blobData(forRef: ref) else { + return Self.notFound() + } + return Response( + statusCode: 200, + headers: ["Content-Type": "application/json"], + body: blob + ) + } + + func jsonHeaders(eTag: String?) -> [String: String] { + var headers = ["Content-Type": "application/json"] + if let eTag { + headers["X-RevenueCat-ETag"] = eTag + } + return headers + } + + static func notFound() -> Response { + return Response( + statusCode: 404, + headers: ["Content-Type": "application/json"], + body: Data(#"{"code": 7259, "message": "benchmark fixture not found"}"#.utf8) + ) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift new file mode 100644 index 0000000000..4ff90d320b --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift @@ -0,0 +1,107 @@ +import Foundation + +/// A named network condition applied by `SimulatedTransportURLProtocol`. +/// +/// API endpoints (dynamic backend calls) and CDN endpoints (static blob downloads) get separate +/// round-trip ranges because the config system's premise is that blobs are served from cheap, +/// cacheable CDN edges while `/v1/config` and `/v1/subscribers/.../offerings` hit the backend. +/// A flat per-request latency would structurally bias results against whichever mode issues +/// more requests, regardless of what those requests cost in production. +struct NetworkProfile { + + let name: String + /// Round-trip time range for dynamic API requests, in milliseconds. Sampled uniformly. + let apiRTTMs: ClosedRange + /// Round-trip time range for CDN blob requests, in milliseconds. Sampled uniformly. + let cdnRTTMs: ClosedRange + /// Downlink throughput; `nil` means body transfer time is not modeled. + let bandwidthBytesPerSec: Double? + + static let ideal = NetworkProfile( + name: "ideal", + apiRTTMs: 0...0, + cdnRTTMs: 0...0, + bandwidthBytesPerSec: nil + ) + + /// Good home wifi: low RTT, ~50 Mbit/s downlink. + static let wifi = NetworkProfile( + name: "wifi", + apiRTTMs: 25...40, + cdnRTTMs: 10...20, + bandwidthBytesPerSec: 50_000_000 / 8 + ) + + /// Typical LTE: higher and jitterier RTT, ~12 Mbit/s downlink. + static let lte = NetworkProfile( + name: "lte", + apiRTTMs: 55...110, + cdnRTTMs: 30...70, + bandwidthBytesPerSec: 12_000_000 / 8 + ) + + static func named(_ name: String) -> NetworkProfile? { + switch name { + case Self.ideal.name: return .ideal + case Self.wifi.name: return .wifi + case Self.lte.name: return .lte + default: return nil + } + } + + func rttMs(forHost host: String, rng: inout SeededRandom) -> Double { + let range = Self.isCDNHost(host) ? self.cdnRTTMs : self.apiRTTMs + guard range.lowerBound < range.upperBound else { return range.lowerBound } + return Double.random(in: range, using: &rng) + } + + /// Transfer time for `byteCount` at the profile's bandwidth, in milliseconds. + func transferTimeMs(forByteCount byteCount: Int) -> Double { + guard let bandwidth = self.bandwidthBytesPerSec, bandwidth > 0 else { return 0 } + return Double(byteCount) / bandwidth * 1_000 + } + + static func isCDNHost(_ host: String) -> Bool { + return host.contains("cdn") + } + +} + +/// Approximates packet loss for the simulated transport. +/// +/// This deliberately does NOT model real TCP behavior. Loss on a real connection shows up as +/// retransmission delays, stalled transfers, and occasional timeouts, not as a proportional +/// request failure rate. The approximation here: +/// - per delivered chunk, with probability `lossPercent`, one retransmission delay of 1x-2x RTT +/// is added before the chunk arrives; +/// - per request, with probability `(lossPercent/100)^3` (all retries of a critical packet +/// lost), the whole request fails with `URLError(.timedOut)` after one RTO so the SDK's +/// retry and fallback-host paths are exercised at a plausible rate. +/// Good enough to compare the two fetch systems under identical degraded conditions; not a +/// substitute for link-conditioner testing when absolute numbers matter. +struct LossModel { + + let lossPercent: Int + + var lossProbability: Double { + return Double(self.lossPercent) / 100 + } + + var requestFailureProbability: Double { + let loss = self.lossProbability + return loss * loss * loss + } + + /// Extra delay (ms) to add before a chunk of the response body, given the connection RTT. + func chunkRetransmitDelayMs(rttMs: Double, rng: inout SeededRandom) -> Double { + guard self.lossPercent > 0 else { return 0 } + guard Double.random(in: 0..<1, using: &rng) < self.lossProbability else { return 0 } + return rttMs * Double.random(in: 1...2, using: &rng) + } + + func shouldFailRequest(rng: inout SeededRandom) -> Bool { + guard self.lossPercent > 0 else { return false } + return Double.random(in: 0..<1, using: &rng) < self.requestFailureProbability + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index 2827bcce4f..a10043a808 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -1,19 +1,262 @@ import Foundation -/// Placeholder so the `HTTPClient` benchmark hook compiles; replaced with the real -/// simulated transport in a follow-up commit. +/// One completed (or failed) simulated request, for phase attribution and byte accounting. +struct TransportEvent { + + let host: String + let path: String + let statusCode: Int + let bytesReceived: Int + let startedAt: DispatchTime + let endedAt: DispatchTime + let failed: Bool + +} + +/// In-process transport used by every URLSession in the benchmark. +/// +/// `canInit` claims EVERY http(s) request, so no benchmark run can ever touch the real network; +/// fallback-host retries (`api-production.8-lives-cat.io`) resolve against the same fixture +/// server as the primary host. Responses are delivered asynchronously on a private queue: +/// headers after one sampled RTT, then the body in chunks paced by the profile's bandwidth and +/// the loss model's retransmission delays. Nothing here ever blocks a thread, so concurrent +/// requests overlap the way they would on a real connection pool. final class SimulatedTransportURLProtocol: URLProtocol { + private struct Installation { + let server: FixtureServer + let profile: NetworkProfile + let loss: LossModel + } + + private static let lock = NSLock() + private static var installation: Installation? + private static var rng = SeededRandom(seed: 0) + private static var events: [TransportEvent] = [] + + private static let chunkSize = 16 * 1024 + + /// Serial per request so header, chunks, and completion arrive in order; separate requests + /// each get their own queue and overlap freely. + private let deliveryQueue = DispatchQueue(label: "com.revenuecat.benchmark.transport.request") + private var pendingWorkItems: [DispatchWorkItem] = [] + private let stateLock = NSLock() + + static func install(server: FixtureServer, profile: NetworkProfile, loss: LossModel, seed: UInt64) { + self.lock.withLock { + self.installation = Installation(server: server, profile: profile, loss: loss) + self.rng = SeededRandom(seed: seed) + self.events = [] + } + } + + static func uninstall() { + self.lock.withLock { + self.installation = nil + self.events = [] + } + } + + /// Returns all events recorded since the last drain, oldest first. + static func drainEvents() -> [TransportEvent] { + return self.lock.withLock { + let drained = self.events + self.events = [] + return drained + } + } + + /// A URLSession whose requests go through this transport, for injecting into SDK seams + /// that build their own sessions (e.g. the remote config blob downloader). + static func makeSession() -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [Self.self] + configuration.urlCache = nil + return URLSession(configuration: configuration) + } + + // MARK: - URLProtocol + override class func canInit(with request: URLRequest) -> Bool { - return false + guard let scheme = request.url?.scheme else { return false } + return scheme == "http" || scheme == "https" } override class func canonicalRequest(for request: URLRequest) -> URLRequest { return request } - override func startLoading() {} + override func startLoading() { + let started = DispatchTime.now() + guard let url = self.request.url, + let installation = Self.lock.withLock({ Self.installation }) else { + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.invalidFixture("Simulated transport not installed") + ) + return + } + + let host = url.host ?? "" + let bodyData = Self.bodyData(of: self.request) + let response = installation.server.response(for: self.request, bodyData: bodyData) - override func stopLoading() {} + // Sample every random decision up front, under one lock, so the request's timeline is + // fixed at start time regardless of delivery interleaving. + let plan: (rttMs: Double, fails: Bool, chunkDelaysMs: [Double]) = Self.lock.withLock { + let rttMs = installation.profile.rttMs(forHost: host, rng: &Self.rng) + let fails = installation.loss.shouldFailRequest(rng: &Self.rng) + var chunkDelaysMs: [Double] = [] + if !fails { + var offset = 0 + while offset < max(response.body.count, 1) { + chunkDelaysMs.append( + installation.loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &Self.rng) + ) + offset += Self.chunkSize + } + } + return (rttMs, fails, chunkDelaysMs) + } + + if plan.fails { + self.deliverFailure(host: host, path: url.path, rttMs: plan.rttMs, startedAt: started) + } else { + self.deliverResponse( + response, + url: url, + profile: installation.profile, + plan: (plan.rttMs, plan.chunkDelaysMs), + startedAt: started + ) + } + } + + override func stopLoading() { + self.stateLock.withLock { + for item in self.pendingWorkItems { + item.cancel() + } + self.pendingWorkItems = [] + } + } + +} + +private extension SimulatedTransportURLProtocol { + + /// One retransmission-timeout's worth of waiting before the failure surfaces, so failed + /// attempts cost time like they do on a real link. + func deliverFailure(host: String, path: String, rttMs: Double, startedAt: DispatchTime) { + let rtoMs = max(1_000, rttMs * 2) + self.schedule(afterMs: rtoMs) { [weak self] in + guard let self else { return } + Self.record(TransportEvent( + host: host, + path: path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: true + )) + self.client?.urlProtocol(self, didFailWithError: URLError(.timedOut)) + } + } + + func deliverResponse( + _ response: FixtureServer.Response, + url: URL, + profile: NetworkProfile, + plan: (rttMs: Double, chunkDelaysMs: [Double]), + startedAt: DispatchTime + ) { + let host = url.host ?? "" + guard let httpResponse = HTTPURLResponse( + url: url, + statusCode: response.statusCode, + httpVersion: "HTTP/1.1", + headerFields: response.headers + ) else { + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.invalidFixture("Could not create fixture HTTP response") + ) + return + } + + var deliveryOffsetMs = plan.rttMs + self.schedule(afterMs: deliveryOffsetMs) { [weak self] in + guard let self else { return } + self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + } + + var offset = 0 + var chunkIndex = 0 + while offset < response.body.count { + let end = min(offset + Self.chunkSize, response.body.count) + let chunk = response.body.subdata(in: offset.. Void) { + let item = DispatchWorkItem(block: work) + self.stateLock.withLock { + self.pendingWorkItems.append(item) + } + self.deliveryQueue.asyncAfter(deadline: .now() + delayMs / 1_000, execute: item) + } + + static func record(_ event: TransportEvent) { + self.lock.withLock { + self.events.append(event) + } + } + + /// URLProtocol surfaces POST bodies as a stream; drain it so the fixture server can + /// inspect request payloads (the config manifest check needs this). + static func bodyData(of request: URLRequest) -> Data? { + if let body = request.httpBody { + return body + } + guard let stream = request.httpBodyStream else { + return nil + } + + stream.open() + defer { stream.close() } + + var data = Data() + let bufferSize = 16 * 1024 + var buffer = [UInt8](repeating: 0, count: bufferSize) + while stream.hasBytesAvailable { + let read = stream.read(&buffer, maxLength: bufferSize) + guard read > 0 else { break } + data.append(buffer, count: read) + } + return data + } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift new file mode 100644 index 0000000000..f4c8d3aaa1 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift @@ -0,0 +1,183 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class FixtureServerTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 3, workflowCount: 4) + private var server: FixtureServer! + + override func setUp() { + super.setUp() + self.server = FixtureServer(factory: self.factory) + } + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + super.tearDown() + } + + private func request(_ urlString: String, eTag: String? = nil) throws -> URLRequest { + var request = URLRequest(url: try XCTUnwrap(URL(string: urlString))) + if let eTag { + request.setValue(eTag, forHTTPHeaderField: "X-RevenueCat-ETag") + } + return request + } + + func testOfferingsServes200WithETagThen304OnMatch() throws { + let url = "https://api.revenuecat.com/v1/subscribers/user-1/offerings" + + let cold = self.server.response(for: try self.request(url), bodyData: nil) + XCTAssertEqual(cold.statusCode, 200) + XCTAssertEqual(cold.headers["X-RevenueCat-ETag"], FixtureServer.offeringsETag) + XCTAssertEqual(cold.body, self.factory.offeringsData) + + let warm = self.server.response( + for: try self.request(url, eTag: FixtureServer.offeringsETag), + bodyData: nil + ) + XCTAssertEqual(warm.statusCode, 304) + XCTAssertEqual(warm.headers["X-RevenueCat-ETag"], FixtureServer.offeringsETag) + XCTAssertTrue(warm.body.isEmpty) + } + + func testOfferingsFallbackHostPathRoutesIdentically() throws { + let fallback = self.server.response( + for: try self.request("https://api-production.8-lives-cat.io/v1/offerings"), + bodyData: nil + ) + + XCTAssertEqual(fallback.statusCode, 200) + XCTAssertEqual(fallback.body, self.factory.offeringsData) + } + + func testConfigServes200ContainerThen204OnMatchingManifest() throws { + let url = "https://api.revenuecat.com/v1/config/app" + + let cold = self.server.response(for: try self.request(url), bodyData: Data(#"{"appUserId":"u"}"#.utf8)) + XCTAssertEqual(cold.statusCode, 200) + XCTAssertEqual(cold.body, self.factory.configContainerData) + + let warmBody = Data(#"{"appUserId":"u","manifest":"benchmark-manifest-v1"}"#.utf8) + let warm = self.server.response(for: try self.request(url), bodyData: warmBody) + XCTAssertEqual(warm.statusCode, 204) + XCTAssertTrue(warm.body.isEmpty) + } + + func testKillSwitchConfigReturns400ButOfferingsStillServe() throws { + let killServer = FixtureServer(factory: self.factory, killSwitchConfig: true) + + let config = killServer.response( + for: try self.request("https://api.revenuecat.com/v1/config/app"), + bodyData: nil + ) + XCTAssertEqual(config.statusCode, 400) + + let offerings = killServer.response( + for: try self.request("https://api.revenuecat.com/v1/subscribers/u/offerings"), + bodyData: nil + ) + XCTAssertEqual(offerings.statusCode, 200) + } + + func testBlobRefRoundTrip() throws { + let ref = try XCTUnwrap(self.factory.allBlobRefs.first) + + let response = self.server.response( + for: try self.request("https://cdn.revenuecat.local/blobs/\(ref)"), + bodyData: nil + ) + + XCTAssertEqual(response.statusCode, 200) + XCTAssertEqual(response.body, self.factory.blobData(forRef: ref)) + } + + func testUnknownPathReturns404() throws { + let response = self.server.response( + for: try self.request("https://api.revenuecat.com/v1/unknown"), + bodyData: nil + ) + + XCTAssertEqual(response.statusCode, 404) + } + + // MARK: - Transport integration + + func testTransportDeliversFixtureBodyAndRecordsEvent() throws { + SimulatedTransportURLProtocol.install( + server: self.server, + profile: .ideal, + loss: LossModel(lossPercent: 0), + seed: 42 + ) + let session = SimulatedTransportURLProtocol.makeSession() + let ref = try XCTUnwrap(self.factory.allBlobRefs.first) + let url = try XCTUnwrap(URL(string: "https://cdn.revenuecat.local/blobs/\(ref)")) + + let expectation = self.expectation(description: "request completes") + var received: (data: Data?, statusCode: Int?) + session.dataTask(with: url) { data, response, error in + XCTAssertNil(error) + received = (data, (response as? HTTPURLResponse)?.statusCode) + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: 5) + + XCTAssertEqual(received.statusCode, 200) + XCTAssertEqual(received.data, self.factory.blobData(forRef: ref)) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.host, "cdn.revenuecat.local") + XCTAssertEqual(events.first?.bytesReceived, self.factory.blobData(forRef: ref)?.count) + XCTAssertTrue(SimulatedTransportURLProtocol.drainEvents().isEmpty, "drain must clear events") + } + + func testTransportInterceptsRealAPIHostsSoNothingLeaks() throws { + SimulatedTransportURLProtocol.install( + server: self.server, + profile: .ideal, + loss: LossModel(lossPercent: 0), + seed: 42 + ) + let session = SimulatedTransportURLProtocol.makeSession() + let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + + let expectation = self.expectation(description: "request completes") + var received: Data? + session.dataTask(with: url) { data, _, _ in + received = data + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: 5) + + XCTAssertEqual(received, self.factory.offeringsData, "real API hosts must resolve to fixtures") + } + + func testTransportModelsLossAsTimedOutFailure() throws { + SimulatedTransportURLProtocol.install( + server: self.server, + profile: .ideal, + loss: LossModel(lossPercent: 100), + seed: 42 + ) + let session = SimulatedTransportURLProtocol.makeSession() + let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + + let expectation = self.expectation(description: "request fails") + var receivedError: Error? + session.dataTask(with: url) { _, _, error in + receivedError = error + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: 10) + + XCTAssertEqual((receivedError as? URLError)?.code, .timedOut) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertTrue(events.first?.failed == true) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift new file mode 100644 index 0000000000..4eb29b102c --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift @@ -0,0 +1,93 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class NetworkProfileTests: BenchmarkTestCase { + + func testProfileLookupByName() { + XCTAssertEqual(NetworkProfile.named("ideal")?.name, "ideal") + XCTAssertEqual(NetworkProfile.named("wifi")?.name, "wifi") + XCTAssertEqual(NetworkProfile.named("lte")?.name, "lte") + XCTAssertNil(NetworkProfile.named("5g")) + } + + func testCDNLatencyIsBelowAPILatencyForRealProfiles() { + for profile in [NetworkProfile.wifi, .lte] { + XCTAssertLessThan(profile.cdnRTTMs.upperBound, profile.apiRTTMs.upperBound, profile.name) + } + } + + func testRTTSamplingIsDeterministicForSameSeed() { + var first = SeededRandom(seed: 9) + var second = SeededRandom(seed: 9) + + let firstSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(forHost: "api.revenuecat.com", rng: &first) } + let secondSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(forHost: "api.revenuecat.com", rng: &second) } + + XCTAssertEqual(firstSamples, secondSamples) + for sample in firstSamples { + XCTAssertTrue(NetworkProfile.lte.apiRTTMs.contains(sample)) + } + } + + func testCDNHostsSampleFromCDNRange() { + var rng = SeededRandom(seed: 9) + + for _ in 0..<32 { + let sample = NetworkProfile.lte.rttMs(forHost: "cdn.revenuecat.local", rng: &rng) + XCTAssertTrue(NetworkProfile.lte.cdnRTTMs.contains(sample)) + } + } + + func testIdealProfileAddsNoDelay() { + var rng = SeededRandom(seed: 1) + + XCTAssertEqual(NetworkProfile.ideal.rttMs(forHost: "api.revenuecat.com", rng: &rng), 0) + XCTAssertEqual(NetworkProfile.ideal.transferTimeMs(forByteCount: 10_000_000), 0) + } + + func testTransferTimeScalesWithBytes() { + let profile = NetworkProfile.lte + + let small = profile.transferTimeMs(forByteCount: 15_000) + let large = profile.transferTimeMs(forByteCount: 1_500_000) + + XCTAssertGreaterThan(large, small * 90) + XCTAssertEqual(profile.transferTimeMs(forByteCount: 1_500_000), 1_000, accuracy: 1) + } + + func testZeroLossAddsNoDelaysAndNeverFails() { + var rng = SeededRandom(seed: 3) + let loss = LossModel(lossPercent: 0) + + for _ in 0..<10_000 { + XCTAssertEqual(loss.chunkRetransmitDelayMs(rttMs: 80, rng: &rng), 0) + XCTAssertFalse(loss.shouldFailRequest(rng: &rng)) + } + } + + func testLossFailureRateMatchesHeuristic() { + var rng = SeededRandom(seed: 4) + let loss = LossModel(lossPercent: 30) + let trials = 10_000 + + let failures = (0.. 0 }, "20% loss over 256 chunks should add some delay") + } + +} From 58b2313717c7ffed748ec3a346df6084ddfc53f0 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:19:04 +0200 Subject: [PATCH 05/34] other(benchmarks): wire real manager-level sdk stacks per benchmark mode legacy = OfferingsManager with no remote config manager, config modes = the full RemoteConfigManager stack with an injected simulated blob downloader. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkProductsManager.swift | 41 +++++ .../Sources/BenchmarkSDKStack.swift | 141 +++++++++++++++++ .../Tests/BenchmarkSDKStackTests.swift | 145 ++++++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift new file mode 100644 index 0000000000..519898a4a7 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift @@ -0,0 +1,41 @@ +import Foundation + +/// StoreKit-free `ProductsManagerType` so the benchmark measures the SDK's fetch and decode +/// paths, not App Store product lookups. Every requested identifier resolves immediately to a +/// synthesized monthly subscription, the same way UI preview mode fabricates products. +final class BenchmarkProductsManager: ProductsManagerType { + + let requestTimeout: TimeInterval = 30 + + func products( + withIdentifiers identifiers: Set, + completion: @escaping ProductsManagerType.Completion + ) { + let products = identifiers.map { identifier in + TestStoreProduct( + localizedTitle: "Benchmark Monthly", + price: 9.99, + localizedPriceString: "$9.99", + productIdentifier: identifier, + productType: .autoRenewableSubscription, + localizedDescription: "Benchmark subscription", + subscriptionGroupIdentifier: "benchmark.group", + subscriptionPeriod: SubscriptionPeriod(value: 1, unit: .month) + ).toStoreProduct() + } + completion(.success(Set(products))) + } + + @available(iOS 15.0, tvOS 15.0, watchOS 8.0, macOS 12.0, *) + func sk2Products( + withIdentifiers identifiers: Set, + completion: @escaping ProductsManagerType.SK2Completion + ) { + completion(.success([])) + } + + func cache(_ product: StoreProductType) {} + + func clearCache() {} + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift new file mode 100644 index 0000000000..7c2b9017e8 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift @@ -0,0 +1,141 @@ +import Foundation + +/// Builds the same manager-level object graph `Purchases` wires at configure time, minus +/// StoreKit and identity, so one simulated app launch is: construct a stack, kick the remote +/// config refresh (config modes), and fetch offerings through `OfferingsManager`. +/// +/// Mode selection mirrors production's `SystemInfo.remoteConfigEnabled` gate: `legacy` passes +/// `remoteConfigManager: nil` to `OfferingsManager` (gate off), the config modes wire a real +/// `RemoteConfigManager` stack (gate on). The kill-switch mode uses identical wiring; the +/// fixture server's 4xx is what trips the manager's session kill switch. +final class BenchmarkSDKStack { + + let offeringsManager: OfferingsManager + let remoteConfigManager: RemoteConfigManagerType? + let httpClient: HTTPClient + let deviceCache: DeviceCache + + private let appUserID: String + + init(mode: BenchmarkMode, apiKey: String, appUserID: String) { + self.appUserID = appUserID + + let operationDispatcher = OperationDispatcher.default + let systemInfo = Self.makeSystemInfo(apiKey: apiKey, operationDispatcher: operationDispatcher) + + let httpClient = HTTPClient( + systemInfo: systemInfo, + eTagManager: ETagManager(), + signing: Signing(apiKey: apiKey), + diagnosticsTracker: nil, + requestTimeout: 15, + operationDispatcher: operationDispatcher + ) + self.httpClient = httpClient + + let backendConfiguration = BackendConfiguration( + httpClient: httpClient, + operationDispatcher: operationDispatcher, + operationQueue: Backend.QueueProvider.createBackendQueue(), + diagnosticsQueue: Backend.QueueProvider.createDiagnosticsQueue(), + systemInfo: systemInfo, + offlineCustomerInfoCreator: nil + ) + let backend = Backend( + backendConfig: backendConfiguration, + attributionFetcher: AttributionFetcher( + attributionFactory: AttributionTypeFactory(), + systemInfo: systemInfo + ) + ) + + guard let userDefaults = UserDefaults(suiteName: "com.revenuecat.SDKConfigBenchmark") else { + preconditionFailure("Could not create benchmark UserDefaults suite") + } + let deviceCache = DeviceCache(systemInfo: systemInfo, userDefaults: userDefaults) + self.deviceCache = deviceCache + + let remoteConfigManager = mode.usesRemoteConfig + ? Self.makeRemoteConfigManager(backendConfiguration: backendConfiguration, appUserID: appUserID) + : nil + self.remoteConfigManager = remoteConfigManager + + self.offeringsManager = OfferingsManager( + deviceCache: deviceCache, + operationDispatcher: operationDispatcher, + systemInfo: systemInfo, + backend: backend, + offeringsFactory: OfferingsFactory(systemInfo: systemInfo), + productsManager: BenchmarkProductsManager(), + diagnosticsTracker: nil, + remoteConfigManager: remoteConfigManager + ) + } + + /// Kicks the remote config sync the way `Purchases.updateAllCaches` does on launch. + func refreshRemoteConfigIfWired() { + self.remoteConfigManager?.refreshRemoteConfig(isAppBackgrounded: false) + } + + /// Erases every piece of disk state a previous launch could have left behind, so a `cold` + /// iteration starts from nothing: ETags, the offerings disk cache, and the persisted remote + /// configuration with its blob store. + func clearAllDiskState() { + self.httpClient.clearCaches() + self.deviceCache.clearOfferingsCache(appUserID: self.appUserID) + self.remoteConfigManager?.clearCache() + } + +} + +private extension BenchmarkSDKStack { + + static func makeSystemInfo(apiKey: String, operationDispatcher: OperationDispatcher) -> SystemInfo { + return SystemInfo( + platformInfo: nil, + finishTransactions: true, + operationDispatcher: operationDispatcher, + storeKitVersion: .storeKit1, + apiKey: apiKey, + responseVerificationMode: .disabled, + isAppBackgrounded: false, + preferredLocalesProvider: PreferredLocalesProvider( + preferredLocaleOverride: "en_US", + systemPreferredLocalesGetter: { ["en_US"] } + ) + ) + } + + static func makeRemoteConfigManager( + backendConfiguration: BackendConfiguration, + appUserID: String + ) -> RemoteConfigManagerType { + let diskCache = RemoteConfigDiskCache() + let blobStore = RemoteConfigBlobStore() + let sourceProvider = RemoteConfigSourceProvider(topicStore: diskCache) + let blobFetcher = RemoteConfigBlobFetcher( + blobStore: blobStore, + sourceProvider: sourceProvider, + downloader: URLSessionRemoteConfigBlobDownloader( + session: SimulatedTransportURLProtocol.makeSession() + ) + ) + return RemoteConfigManager( + remoteConfigAPI: RemoteConfigAPI(backendConfig: backendConfiguration), + diskCache: diskCache, + blobStore: blobStore, + blobFetcher: blobFetcher, + currentUserProvider: BenchmarkCurrentUserProvider(appUserID: appUserID) + ) + } + +} + +private struct BenchmarkCurrentUserProvider: CurrentUserProvider { + + let appUserID: String + + var currentAppUserID: String { return self.appUserID } + var currentUserIsAnonymous: Bool { return false } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift new file mode 100644 index 0000000000..b3f76fdabf --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift @@ -0,0 +1,145 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +/// End-to-end: real `OfferingsManager` (and `RemoteConfigManager` for config modes) against +/// the simulated transport. These are the tests that prove the benchmark measures the actual +/// SDK flows rather than a hand-rolled imitation of them. +final class BenchmarkSDKStackTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 3, workflowCount: 4) + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + super.tearDown() + } + + private func install(killSwitch: Bool = false) { + SimulatedTransportURLProtocol.install( + server: FixtureServer(factory: self.factory, killSwitchConfig: killSwitch), + profile: .ideal, + loss: LossModel(lossPercent: 0), + seed: 42 + ) + } + + private func fetchOfferings( + _ stack: BenchmarkSDKStack, + appUserID: String, + timeout: TimeInterval = 15 + ) throws -> Offerings { + let expectation = self.expectation(description: "offerings delivered") + let result = LockedResult>() + stack.offeringsManager.offerings(appUserID: appUserID, fetchCurrent: true) { offerings in + result.set(offerings) + expectation.fulfill() + } + self.wait(for: [expectation], timeout: timeout) + return try XCTUnwrap(result.get()).get() + } + + func testLegacyModeFetchesOfferingsWithSingleRequest() throws { + self.install() + let stack = BenchmarkSDKStack(mode: .legacy, apiKey: "appl_benchmark", appUserID: "legacy-user") + stack.clearAllDiskState() + + let offerings = try self.fetchOfferings(stack, appUserID: "legacy-user") + + XCTAssertEqual(offerings.all.count, 3) + XCTAssertNotNil(offerings.current) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertTrue(events[0].path.hasSuffix("/offerings")) + } + + func testConfigModeFetchesConfigBlobsAndOfferings() throws { + self.install() + let stack = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "config-user") + stack.clearAllDiskState() + + stack.refreshRemoteConfigIfWired() + let offerings = try self.fetchOfferings(stack, appUserID: "config-user") + + XCTAssertEqual(offerings.all.count, 3) + + let events = SimulatedTransportURLProtocol.drainEvents() + let paths = events.map(\.path) + XCTAssertTrue(paths.contains { $0.hasSuffix("/config/app") }, "expected a config fetch in \(paths)") + XCTAssertTrue(paths.contains { $0.contains("/blobs/") }, "expected blob downloads in \(paths)") + XCTAssertTrue(paths.contains { $0.hasSuffix("/offerings") }, "expected an offerings fetch in \(paths)") + + // The workflows topic marks every workflow blob prefetch, so offerings delivery waits + // for all of them plus the two ui_config blobs. + let blobRequests = paths.filter { $0.contains("/blobs/") } + XCTAssertEqual(Set(blobRequests).count, 4 + 2) + } + + func testKillSwitchModeStillDeliversOfferings() throws { + self.install(killSwitch: true) + let stack = BenchmarkSDKStack(mode: .configKillswitch, apiKey: "appl_benchmark", appUserID: "kill-user") + stack.clearAllDiskState() + + stack.refreshRemoteConfigIfWired() + let offerings = try self.fetchOfferings(stack, appUserID: "kill-user") + + XCTAssertEqual(offerings.all.count, 3) + XCTAssertEqual(stack.remoteConfigManager?.isDisabled, true, "4xx must trip the session kill switch") + + let events = SimulatedTransportURLProtocol.drainEvents() + let paths = events.map(\.path) + XCTAssertTrue(paths.contains { $0.hasSuffix("/config/app") }) + XCTAssertFalse(paths.contains { $0.contains("/blobs/") }, "no blobs after a config 4xx") + } + + func testWarmRelaunchServes304And204() throws { + self.install() + + // Priming launch populates etags, offerings disk cache, and the persisted config. + let priming = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "warm-user") + priming.clearAllDiskState() + priming.refreshRemoteConfigIfWired() + _ = try self.fetchOfferings(priming, appUserID: "warm-user") + _ = SimulatedTransportURLProtocol.drainEvents() + + // Simulated relaunch: fresh in-memory stack, retained disk state. + let relaunch = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "warm-user") + relaunch.refreshRemoteConfigIfWired() + _ = try self.fetchOfferings(relaunch, appUserID: "warm-user") + + let events = SimulatedTransportURLProtocol.drainEvents() + let statusesByPath = Dictionary( + events.map { ($0.path, $0.statusCode) }, + uniquingKeysWith: { first, _ in first } + ) + + XCTAssertTrue( + statusesByPath.contains { $0.key.hasSuffix("/offerings") && $0.value == 304 }, + "warm offerings must be revalidated via 304, got \(statusesByPath)" + ) + XCTAssertTrue( + statusesByPath.contains { $0.key.hasSuffix("/config/app") && $0.value == 204 }, + "warm config must be revalidated via manifest 204, got \(statusesByPath)" + ) + XCTAssertFalse( + statusesByPath.contains { $0.key.contains("/blobs/") }, + "warm relaunch must not re-download blobs, got \(statusesByPath)" + ) + } + +} + +private final class LockedResult: @unchecked Sendable { + + private let lock = NSLock() + private var value: Value? + + func set(_ value: Value) { + self.lock.withLock { self.value = value } + } + + func get() -> Value? { + return self.lock.withLock { self.value } + } + +} From 7dda70809c5683bc7d7fb4f1c648e18113460ab4 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:27:12 +0200 Subject: [PATCH 06/34] other(benchmarks): add benchmark runner with phase metrics and jsonl output Embeds an info-plist section in the CLI binary so the SDK's bundle-id-derived disk caches (etags, offerings, remote config) work outside an app container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmark/Project.swift | 7 +- .../Sources/BenchmarkMain.swift | 24 ++- .../Sources/BenchmarkMetrics.swift | 138 +++++++++++++++++ .../Sources/BenchmarkRunner.swift | 145 ++++++++++++++++++ .../Sources/BenchmarkSDKStack.swift | 2 +- .../Tests/BenchmarkMetricsTests.swift | 103 +++++++++++++ 6 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift diff --git a/Projects/SDKConfigBenchmark/Project.swift b/Projects/SDKConfigBenchmark/Project.swift index e257a09a0b..9fdaf0b938 100644 --- a/Projects/SDKConfigBenchmark/Project.swift +++ b/Projects/SDKConfigBenchmark/Project.swift @@ -52,7 +52,12 @@ let project = Project( .target(name: "SDKConfigBenchmarkCore") ], settings: .settings( - base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + // The SDK's disk caches derive their directory from Bundle.main.bundleIdentifier, + // which a bare command-line binary does not have. Embedding the Info.plist into + // the binary gives it one, so etags/offerings/remote-config persistence works. + base: benchmarkSwiftConditions + .merging(["CREATE_INFOPLIST_SECTION_IN_BINARY": "YES"]) + .appendingTuistSwiftConditions() ) ), .target( diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift index a1f8fa7848..f77eb6910d 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift @@ -10,17 +10,31 @@ public struct BenchmarkMain { /// Parses `CommandLine.arguments`, runs the requested benchmark, prints one JSONL row /// to stdout, and terminates the process (0 on success, 1 on error). + /// + /// The benchmark itself runs on a background queue while the main thread sits in + /// `dispatchMain()`: `OfferingsManager` delivers its completion on the main queue, so + /// blocking the main thread on the run would deadlock every iteration. public static func run() -> Never { + let command: BenchmarkCommand do { - let command = try BenchmarkCommand.parse(Array(CommandLine.arguments.dropFirst())) - // Placeholder output until the runner lands; proves parsing and target wiring. - print("parsed mode=\(command.mode.rawValue) scenario=\(command.scenario.rawValue) " + - "profile=\(command.profileName) iterations=\(command.iterations)") - exit(0) + command = try BenchmarkCommand.parse(Array(CommandLine.arguments.dropFirst())) } catch { FileHandle.standardError.write(Data("\(error)\n".utf8)) exit(1) } + + DispatchQueue.global(qos: .userInitiated).async { + do { + let row = try BenchmarkRunner(command: command).run() + print(row) + exit(0) + } catch { + FileHandle.standardError.write(Data("\(error)\n".utf8)) + exit(1) + } + } + + dispatchMain() } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift new file mode 100644 index 0000000000..dc68f3d1b9 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -0,0 +1,138 @@ +import Foundation + +/// One measured simulated launch, with phases attributed from the transport event log. +struct IterationMeasurement { + + /// Start of the iteration to delivery of the offerings completion. + let totalMs: Double + /// Span of the offerings request(s), nil if none were made. + let offeringsMs: Double? + /// Span of the `/v1/config` request(s), nil for legacy mode. + let configMs: Double? + /// First blob request start to last blob request end, nil if no blobs were fetched. + let blobMs: Double? + let requestCount: Int + let bytesReceived: Int + let failedRequestCount: Int + let fallbackHostRequestCount: Int + /// Statuses seen, for warm-scenario verification (304/204). + let statusCodes: [Int] + + init(totalMs: Double, events: [TransportEvent]) { + self.totalMs = totalMs + self.offeringsMs = Self.span(of: events.filter { $0.path.hasSuffix("/offerings") }) + self.configMs = Self.span(of: events.filter { $0.path.hasSuffix("/config/app") }) + self.blobMs = Self.span(of: events.filter { $0.path.contains("/blobs/") }) + self.requestCount = events.count + self.bytesReceived = events.reduce(0) { $0 + $1.bytesReceived } + self.failedRequestCount = events.filter(\.failed).count + self.fallbackHostRequestCount = events + .filter { $0.host.contains("8-lives-cat") || $0.host.contains("rc-backup") } + .count + self.statusCodes = events.map(\.statusCode) + } + + private static func span(of events: [TransportEvent]) -> Double? { + guard let first = events.map(\.startedAt.uptimeNanoseconds).min(), + let last = events.map(\.endedAt.uptimeNanoseconds).max() else { + return nil + } + return Double(last - first) / 1_000_000 + } + +} + +/// Aggregates measured iterations into one JSONL row. The first `warmupIterations` recorded +/// measurements are excluded from the statistics but counted in `warmup_discarded`, so one-time +/// process costs (lazy statics, first JSON decoder use) don't pollute the distribution. +struct BenchmarkMetrics { + + private var measurements: [IterationMeasurement] = [] + private var errors: [String] = [] + + mutating func record(_ measurement: IterationMeasurement) { + self.measurements.append(measurement) + } + + mutating func record(error: Error) { + self.errors.append("\(error)") + } + + var allStatusCodes: [Int] { + return self.measurements.flatMap(\.statusCodes) + } + + var errorCount: Int { + return self.errors.count + } + + func jsonlRow(for command: BenchmarkCommand) -> String { + let measured = Array(self.measurements.dropFirst(command.warmupIterations)) + let totals = measured.map(\.totalMs).sorted() + + var row: [String: Any] = [ + "mode": command.mode.rawValue, + "scenario": command.scenario.rawValue, + "profile": command.profileName, + "loss_percent": command.lossPercent, + "paywalls": command.paywallCount, + "workflows": command.workflowCount, + "seed": command.seed, + "iterations": command.iterations, + "warmup_discarded": min(command.warmupIterations, self.measurements.count), + "error_count": self.errors.count + ] + + if !totals.isEmpty { + row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) + row["min_ms"] = Self.rounded(totals[0]) + row["max_ms"] = Self.rounded(totals[totals.count - 1]) + for percentile in [50, 90, 95, 99] { + row["p\(percentile)_ms"] = Self.rounded(Self.percentile(percentile, of: totals)) + } + } + + if !measured.isEmpty { + let count = Double(measured.count) + row["request_count_mean"] = Self.rounded(Double(measured.reduce(0) { $0 + $1.requestCount }) / count) + row["bytes_received_mean"] = Self.rounded(Double(measured.reduce(0) { $0 + $1.bytesReceived }) / count) + row["failed_requests_total"] = measured.reduce(0) { $0 + $1.failedRequestCount } + row["fallback_host_requests_total"] = measured.reduce(0) { $0 + $1.fallbackHostRequestCount } + + for (key, values) in [ + ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), + ("config_ms_mean", measured.compactMap(\.configMs)), + ("blob_ms_mean", measured.compactMap(\.blobMs)) + ] where !values.isEmpty { + row[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) + } + } + + if !self.errors.isEmpty { + row["first_error"] = self.errors[0] + } + + for (key, value) in command.annotations { + row[key] = value + } + + do { + let data = try JSONSerialization.data(withJSONObject: row, options: [.sortedKeys]) + return String(bytes: data, encoding: .utf8) ?? "{}" + } catch { + preconditionFailure("Could not serialize benchmark row: \(error)") + } + } + + /// Nearest-rank percentile over an already-sorted array. + static func percentile(_ percentile: Int, of sorted: [Double]) -> Double { + precondition(!sorted.isEmpty) + let rank = Int((Double(percentile) / 100 * Double(sorted.count)).rounded(.up)) + return sorted[max(0, min(sorted.count - 1, rank - 1))] + } + + private static func rounded(_ value: Double) -> Double { + return (value * 1_000).rounded() / 1_000 + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift new file mode 100644 index 0000000000..a12a7739db --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -0,0 +1,145 @@ +import Foundation + +/// Drives one benchmark configuration: N simulated app launches through the real SDK stack, +/// against the simulated transport, producing a single JSONL row. +/// +/// An iteration is one launch: build a fresh stack (fresh in-memory state, like a new process), +/// kick the remote config refresh when wired, fetch offerings, and stop the clock when the +/// offerings completion fires. `cold` wipes disk state before every iteration and uses a +/// per-iteration app user ID; `warm` primes disk once, then relaunches against retained disk +/// state, which must revalidate via 304 (offerings) and 204 (config) or the run fails loudly +/// rather than silently measuring cold behavior. +final class BenchmarkRunner { + + private let command: BenchmarkCommand + + init(command: BenchmarkCommand) { + self.command = command + } + + func run() throws -> String { + guard let profile = NetworkProfile.named(self.command.profileName) else { + throw BenchmarkError.invalidArgument("unknown profile \(self.command.profileName)") + } + + let factory = BenchmarkPayloadFactory( + paywallCount: self.command.paywallCount, + workflowCount: self.command.workflowCount + ) + let server = FixtureServer( + factory: factory, + killSwitchConfig: self.command.mode == .configKillswitch + ) + SimulatedTransportURLProtocol.install( + server: server, + profile: profile, + loss: LossModel(lossPercent: self.command.lossPercent), + seed: self.command.seed + ) + defer { SimulatedTransportURLProtocol.uninstall() } + + var metrics = BenchmarkMetrics() + + if self.command.scenario == .warm { + try self.primeDiskState() + } + + for iteration in 0.. String { + switch self.command.scenario { + case .cold: + return "\(self.command.appUserID)-\(iteration)" + case .warm: + return self.command.appUserID + } + } + + /// Uncounted launch that fills the disk caches the warm iterations relaunch against. + func primeDiskState() throws { + let stack = BenchmarkSDKStack( + mode: self.command.mode, + apiKey: self.command.apiKey, + appUserID: self.command.appUserID + ) + stack.clearAllDiskState() + _ = try self.launch(stack, appUserID: self.command.appUserID) + _ = SimulatedTransportURLProtocol.drainEvents() + } + + func runIteration(_ iteration: Int) throws -> IterationMeasurement { + let appUserID = self.appUserID(forIteration: iteration) + let stack = BenchmarkSDKStack( + mode: self.command.mode, + apiKey: self.command.apiKey, + appUserID: appUserID + ) + if self.command.scenario == .cold { + stack.clearAllDiskState() + } + _ = SimulatedTransportURLProtocol.drainEvents() + + let totalMs = try self.launch(stack, appUserID: appUserID) + + return IterationMeasurement( + totalMs: totalMs, + events: SimulatedTransportURLProtocol.drainEvents() + ) + } + + /// One simulated launch; returns start-to-offerings-delivered wall time in milliseconds. + /// Runs off the main thread; the offerings completion is delivered on the main queue, which + /// `BenchmarkMain` keeps pumping via `dispatchMain()`. + func launch(_ stack: BenchmarkSDKStack, appUserID: String) throws -> Double { + let start = DispatchTime.now() + stack.refreshRemoteConfigIfWired() + + let semaphore = DispatchSemaphore(value: 0) + let failure = Atomic(nil) + stack.offeringsManager.offerings(appUserID: appUserID) { result in + if case let .failure(error) = result { + failure.value = error + } + semaphore.signal() + } + + guard semaphore.wait(timeout: .now() + 120) == .success else { + throw BenchmarkError.timeout("offerings fetch") + } + if let error = failure.value { + throw BenchmarkError.backendFailure("offerings fetch failed: \(error)") + } + + let end = DispatchTime.now() + return Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000 + } + + /// Warm runs must prove they actually hit the revalidation paths. + func verifyScenario(_ metrics: BenchmarkMetrics) throws { + guard self.command.scenario == .warm, self.command.lossPercent == 0 else { return } + + let statuses = Set(metrics.allStatusCodes) + guard statuses.contains(304) else { + throw BenchmarkError.scenarioViolation("warm run never revalidated offerings via 304") + } + if self.command.mode == .config, !statuses.contains(204) { + throw BenchmarkError.scenarioViolation("warm config run never revalidated via manifest 204") + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift index 7c2b9017e8..4dfb8d2052 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift @@ -49,7 +49,7 @@ final class BenchmarkSDKStack { ) ) - guard let userDefaults = UserDefaults(suiteName: "com.revenuecat.SDKConfigBenchmark") else { + guard let userDefaults = UserDefaults(suiteName: "com.revenuecat.SDKConfigBenchmark.scratch") else { preconditionFailure("Could not create benchmark UserDefaults suite") } let deviceCache = DeviceCache(systemInfo: systemInfo, userDefaults: userDefaults) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift new file mode 100644 index 0000000000..491b303fb9 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -0,0 +1,103 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkMetricsTests: BenchmarkTestCase { + + private func measurement(totalMs: Double) -> IterationMeasurement { + return IterationMeasurement(totalMs: totalMs, events: []) + } + + func testNearestRankPercentiles() { + let sorted: [Double] = (1...100).map(Double.init) + + XCTAssertEqual(BenchmarkMetrics.percentile(50, of: sorted), 50) + XCTAssertEqual(BenchmarkMetrics.percentile(90, of: sorted), 90) + XCTAssertEqual(BenchmarkMetrics.percentile(95, of: sorted), 95) + XCTAssertEqual(BenchmarkMetrics.percentile(99, of: sorted), 99) + XCTAssertEqual(BenchmarkMetrics.percentile(99, of: [7]), 7) + } + + private func decodeRow(_ metrics: BenchmarkMetrics, command: BenchmarkCommand) throws -> [String: Any] { + let row = metrics.jsonlRow(for: command) + let object = try JSONSerialization.jsonObject(with: Data(row.utf8)) + return try XCTUnwrap(object as? [String: Any]) + } + + func testJSONLRowDiscardsWarmupIterations() throws { + var command = BenchmarkCommand() + command.iterations = 5 + command.warmupIterations = 2 + + var metrics = BenchmarkMetrics() + // Two slow warmup iterations that must not pollute the stats. + [1_000.0, 900.0, 10.0, 20.0, 30.0].forEach { metrics.record(self.measurement(totalMs: $0)) } + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["warmup_discarded"] as? Int, 2) + XCTAssertEqual(row["mean_ms"] as? Double, 20) + XCTAssertEqual(row["max_ms"] as? Double, 30) + } + + func testJSONLRowCarriesConfigurationAndAnnotations() throws { + var command = BenchmarkCommand() + command.mode = .configKillswitch + command.scenario = .warm + command.profileName = "lte" + command.lossPercent = 20 + command.annotations = ["sdk_commit": "abc123"] + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 42)) + metrics.record(error: BenchmarkError.timeout("offerings fetch")) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["mode"] as? String, "config-killswitch") + XCTAssertEqual(row["scenario"] as? String, "warm") + XCTAssertEqual(row["profile"] as? String, "lte") + XCTAssertEqual(row["loss_percent"] as? Int, 20) + XCTAssertEqual(row["sdk_commit"] as? String, "abc123") + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertNotNil(row["first_error"]) + XCTAssertEqual(row["p50_ms"] as? Double, 42) + } + + func testMeasurementDerivesPhasesFromEvents() { + let start = DispatchTime.now() + func time(_ offsetMs: UInt64) -> DispatchTime { + return DispatchTime(uptimeNanoseconds: start.uptimeNanoseconds + offsetMs * 1_000_000) + } + func event( + path: String, + host: String = "api.revenuecat.com", + from startMs: UInt64, + until endMs: UInt64, + status: Int = 200, + failed: Bool = false + ) -> TransportEvent { + return TransportEvent(host: host, path: path, statusCode: status, bytesReceived: 100, + startedAt: time(startMs), endedAt: time(endMs), failed: failed) + } + + let measurement = IterationMeasurement(totalMs: 100, events: [ + event(path: "/v1/config/app", from: 0, until: 10), + event(path: "/blobs/aaa", host: "cdn.revenuecat.local", from: 10, until: 20), + event(path: "/blobs/bbb", host: "cdn.revenuecat.local", from: 12, until: 30), + event(path: "/v1/subscribers/u/offerings", from: 5, until: 45), + event(path: "/v1/offerings", host: "api-production.8-lives-cat.io", + from: 46, until: 50, status: 0, failed: true) + ]) + + XCTAssertEqual(measurement.requestCount, 5) + XCTAssertEqual(measurement.bytesReceived, 500) + XCTAssertEqual(measurement.failedRequestCount, 1) + XCTAssertEqual(measurement.fallbackHostRequestCount, 1) + XCTAssertEqual(measurement.configMs ?? 0, 10, accuracy: 0.001) + XCTAssertEqual(measurement.blobMs ?? 0, 20, accuracy: 0.001) + XCTAssertEqual(measurement.offeringsMs ?? 0, 45, accuracy: 0.001) + } + +} From fb61655d679ac13dc0118b61c96a9f3910ae0f86 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:33:53 +0200 Subject: [PATCH 07/34] other(benchmarks): add matrix runner, compare script and benchmark docs Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 172 ++++++++++++++++++ Tests/Benchmarks/SDKConfigBenchmark/README.md | 73 ++++++++ .../Benchmarks/SDKConfigBenchmark/compare.py | 98 ++++++++++ .../SDKConfigBenchmark/run-matrix.sh | 89 +++++++++ 4 files changed, 432 insertions(+) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/README.md create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/compare.py create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md new file mode 100644 index 0000000000..8fce8fd7ba --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -0,0 +1,172 @@ +# SDK Config Endpoint Benchmarks + +Last updated: 2026-07-08 + +## Goal + +Produce reproducible, SDK-side numbers comparing the **current offerings system** with the +**config endpoint system** (`/v1/config/{domain}` + content-addressed blobs), so the config +endpoint plus workflows can be released with data instead of guesses. The harness also measures +the **kill switch**: what a launch costs when `/v1/config` is force-failed with a 4xx and the +SDK falls back to the legacy path. + +This is not a backend stress test. It measures the iOS SDK's fetch, cache, revalidation, and +decode behavior under controlled network conditions. + +## What the benchmark actually drives + +Each iteration is one simulated app launch through the **real manager-level SDK code**: + +- **`legacy`**: `OfferingsManager.offerings(appUserID:)` with `remoteConfigManager: nil`, the + exact wiring `Purchases` uses when the remote config gate is off. One + `GET /v1/subscribers/{id}/offerings` request. +- **`config`**: the full production stack: `RemoteConfigManager` (with `RemoteConfigDiskCache`, + `RemoteConfigBlobStore`, `RemoteConfigSourceProvider`, `RemoteConfigBlobFetcher`), refreshed + at launch like `updateAllCaches` does, plus the same `OfferingsManager` fetch. Offerings + delivery goes through `deliverWhenConfigReady`, so the measured time includes waiting for the + `workflows` topic and its prefetch-marked blobs, exactly as production would. +- **`config-killswitch`**: identical wiring to `config`, but the fixture returns `400` for + `/v1/config`, tripping `RemoteConfigManager`'s session kill switch. Measures what the switch + costs users: the wasted config round trip plus the legacy fetch. + +Timing stops when the offerings completion fires. That is the product-shaped metric ("time +until offerings are usable"), not "time for N HTTP requests". + +Payloads are generated by `BenchmarkPayloadFactory` and are validated by unit tests to decode +into the real SDK models (`OfferingsResponse`, `RemoteConfiguration` inside a binary +RC-Container, `PublishedWorkflow` blobs, `ui_config` blobs) with content-addressed blob refs +that pass the SDK's checksum validation. StoreKit is replaced by a `ProductsManagerType` fake; +everything else is production code. + +## Network model + +All requests are intercepted in-process by `SimulatedTransportURLProtocol` (every host, +including the SDK's fallback hosts, so nothing can leak to the real network). Responses are +delivered asynchronously with: + +- **Per-class round-trip times**: dynamic API endpoints and CDN blob endpoints get separate RTT + ranges, because the config system's premise is that blobs are cheap CDN fetches while + `/offerings` and `/v1/config` are dynamic backend calls. A flat per-request latency would + structurally bias against whichever mode makes more requests. +- **Bandwidth-paced chunked bodies**, so payload size costs transfer time. +- **A seeded loss model**: per body chunk, `loss%` chance of one retransmission delay (1x-2x + RTT); per request, `(loss/100)^3` chance of failing with a timeout after one RTO, which + exercises the SDK's retry and fallback-host logic. + +Profiles: `ideal` (0ms, unlimited), `wifi` (25-40ms API / 10-20ms CDN, ~50 Mbit/s), `lte` +(55-110ms API / 30-70ms CDN, ~12 Mbit/s). All randomness is seeded (`--seed`), so runs are +reproducible. + +**Honest caveat:** the loss model approximates TCP behavior; it does not simulate real packet +loss (retransmission dynamics, congestion control, TLS handshakes). It is good enough to +compare the two systems under identical degraded conditions. For absolute numbers under real +loss, use a local HTTP server plus the OS Network Link Conditioner. + +## Scenarios + +- **`cold`**: every iteration wipes ETags, the offerings disk cache, and the persisted remote + config + blob store, and uses a fresh app user ID. First-launch behavior. +- **`warm`**: disk state is primed once, then every iteration builds a fresh in-memory stack + against the retained disk state, simulating an app relaunch. The fixture honors the SDK's + actual revalidation mechanisms: offerings revalidate via `X-RevenueCat-ETag` -> `304`, config + revalidates via its manifest token -> `204`, and blobs are content-addressed so they are never + re-downloaded. The runner **fails loudly** if a warm run never sees a 304/204, so the scenario + can't silently degrade into measuring cold behavior. + +The first `--warmup-iterations` (default 3) measured iterations are discarded from statistics +so one-time process costs don't pollute the distribution. + +## Running + +```sh +bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py results.jsonl +``` + +Run the same command on a baseline branch and a candidate branch, then: + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py baseline.jsonl candidate.jsonl +``` + +Every JSONL row carries the full configuration (mode, scenario, profile, loss, counts, seed) +plus `sdk_commit`, so result files are self-describing. + +## Sample numbers + +Default matrix (25 iterations, 3 warmup discarded, 50 paywalls, 100 workflows, seed 42), +macOS Release build, commit `c1958bf64`: + +| mode | scenario | profile | loss % | p50 ms | p95 ms | requests (mean) | bytes (mean) | +|---|---|---|---:|---:|---:|---:|---:| +| legacy | cold | ideal | 0 | 6.8 | 7.6 | 1 | 64,972 | +| config | cold | ideal | 0 | 25.2 | 28.7 | 102 | 132,647 | +| config-killswitch | cold | ideal | 0 | 7.3 | 7.6 | 2 | 65,022 | +| legacy | cold | lte | 0 | 146.5 | 166.4 | 1 | 64,972 | +| config | cold | lte | 0 | 1,493.0 | 1,589.2 | 103 | 132,704 | +| config-killswitch | cold | lte | 0 | 232.1 | 274.8 | 2 | 65,022 | +| legacy | warm | ideal | 0 | 7.2 | 7.4 | 1 | 0 | +| config | warm | ideal | 0 | 10.4 | 10.9 | 2 | 0 | +| config-killswitch | warm | ideal | 0 | 8.3 | 11.0 | 2 | 50 | +| legacy | warm | lte | 0 | 101.0 | 127.8 | 1 | 0 | +| config | warm | lte | 0 | 212.4 | 231.6 | 2 | 0 | +| config-killswitch | warm | lte | 0 | 191.1 | 229.4 | 2 | 50 | +| legacy | cold | lte | 10 | 163.1 | 385.1 | 1 | 64,972 | +| config | cold | lte | 10 | 1,712.9 | 1,829.5 | 102 | 131,882 | +| legacy | cold | lte | 20 | 284.2 | 448.0 | 1 | 64,972 | +| config | cold | lte | 20 | 1,963.5 | 2,466.3 | 97 | 129,178 | +| legacy | cold | lte | 30 | 293.1 | 585.1 | 1 | 64,972 | +| config | cold | lte | 30 | 2,154.8 | 3,038.0 | 70 | 114,617 | + +## Interpretation + +- **Prefetch gating dominates config cold starts.** With 100 workflows all marked + `prefetch: true`, offerings delivery waits for ~100 blob downloads through the fetcher's + 4-concurrent-download cap: ~25 sequential batches at LTE CDN RTTs is ~1.3-1.5s, which is + exactly the gap to legacy (1,493ms vs 146ms p50). This is a **fixture policy worst case**, + not an inherent cost of the config system; the real product decision is prefetch scope + (current offering only? top N?), and this harness can now measure each option. +- **Warm launches are where the config design pays off.** A warm config relaunch is 2 tiny + revalidation requests (manifest 204 + offerings 304) and zero content bytes; blobs are + content-addressed and never re-downloaded. The remaining LTE gap (212ms vs 101ms p50) is one + extra API round trip, serialized behind the offerings call by `HTTPClient`'s + single-connection-per-host policy; request parallelism is a measurable lever here. +- **The kill switch is cheap.** Force-failing `/v1/config` with a 4xx costs roughly one API + round trip over pure legacy (232ms vs 146ms cold LTE p50), after which the session behaves + like legacy. No cascading retries, no blob traffic. +- **Loss hurts config more in absolute terms** because it multiplies across ~100 requests + (and at 30% loss, failed blob downloads start shrinking the request/byte counts as fetches + give up), while legacy's single request degrades only its own tail. Per request the two + systems degrade the same way; the difference is request count, which again points at + prefetch scope and blob layout as the levers that matter. +- **Bytes double on cold config** (133KB vs 65KB) in this fixture because offerings still + carries full `paywall_components` while the config path also downloads workflow and + `ui_config` blobs. The "stripped offerings" lever below is the measurement that matters for + deciding whether to remove paywall data from `/offerings`. + +## Known limitations + +- **macOS CPU, not device CPU.** Decode and CPU costs are measured on desktop hardware. The + comparison between modes is valid; absolute numbers on device will differ. If absolute + launch-time claims are needed, wrap the same runner in an iOS app target. +- **Loss is approximated**, see the network model section above. +- **Fixture payloads are simpler than production.** Paywall component trees are minimal (one + stack + one text component per paywall). Byte counts scale with `--paywalls`/`--workflows`, + but production component trees are larger and heavier to decode. Grow the factory shapes + before making final decisions that hinge on decode cost. +- **No identity or purchase flows.** The benchmark measures the launch-path fetch and cache + system only. + +## Future levers to measure + +These slot in as new fixture/factory knobs rather than new machinery: + +- **Blob layout**: bundled vs topic-split vs per-paywall vs selected-paywalls blobs. +- **Prefetch scope**: which workflow blobs are marked `prefetch: true` (today the fixture marks + all of them, which is the worst case for `config` cold starts, since offerings delivery waits + on every prefetch-marked blob). +- **Stripped offerings**: offerings payloads without `paywall_components`, with paywalls served + from config blobs instead. +- **Parallelism**: `HTTPClient` pins `httpMaximumConnectionsPerHost = 1` and the blob fetcher + caps at 4 concurrent downloads; both are worth sweeping once the transport models connection + reuse. diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md new file mode 100644 index 0000000000..0f20182e66 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -0,0 +1,73 @@ +# SDKConfigBenchmark + +A macOS command-line benchmark that measures the SDK's **legacy offerings flow** against the +**remote config endpoint flow** (and its kill switch) through the real manager-level code paths, +under simulated network conditions. + +See `CONFIG_ENDPOINT_BENCHMARKS.md` for methodology, scenario definitions, sample numbers, and +known limitations. + +## Setup + +```sh +tuist install +tuist generate SDKConfigBenchmark SDKConfigBenchmarkTests +``` + +## Run the matrix + +```sh +bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl +``` + +Compare two runs (e.g. baseline branch vs candidate branch): + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py baseline.jsonl candidate.jsonl +``` + +Render one run as a table: + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py results.jsonl +``` + +## Run a single configuration + +```sh +xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmark \ + -configuration Release -destination platform=macOS build + +SDKConfigBenchmark \ + --mode config \ # legacy | config | config-killswitch + --scenario cold \ # cold | warm + --profile lte \ # ideal | wifi | lte + --loss-percent 20 \ + --iterations 25 \ + --warmup-iterations 3 \ + --paywalls 50 \ + --workflows 100 \ + --seed 42 \ + --annotation sdk_commit=$(git rev-parse --short HEAD) +``` + +Output is one JSON object (a JSONL row) on stdout. + +Run the benchmark with a scratch `HOME` (the matrix script does this automatically) so the +SDK's disk caches don't touch your real user directory: + +```sh +HOME=$(mktemp -d) SDKConfigBenchmark --mode legacy --scenario cold --profile ideal +``` + +## Unit tests + +```sh +xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmarkTests \ + -destination platform=macOS test +``` + +The tests validate that fixtures decode into the real SDK response models, that the RC-Container +encoder round-trips through the SDK parser, that the transport is deterministic per seed, and +that each benchmark mode drives the expected SDK flow end to end (including warm 304/204 +revalidation and the kill-switch fallback). diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py new file mode 100644 index 0000000000..3c047ad3c5 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Render SDK config benchmark JSONL results as a markdown table. + +Usage: + python3 compare.py results.jsonl # one file: summary table + python3 compare.py baseline.jsonl candidate.jsonl # two files: side-by-side with deltas + +Rows are grouped by (mode, scenario, profile, loss_percent). When comparing, groups present +in only one file are shown with the other side empty. +""" + +import json +import sys + + +KEY_FIELDS = ("mode", "scenario", "profile", "loss_percent") +METRICS = ("p50_ms", "p95_ms", "request_count_mean", "bytes_received_mean") + + +def load(path): + rows = {} + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + row = json.loads(line) + key = tuple(row.get(field) for field in KEY_FIELDS) + rows[key] = row + return rows + + +def fmt(value): + if value is None: + return "-" + if isinstance(value, float): + return f"{value:,.1f}" + return f"{value:,}" + + +def delta(base, cand): + if base in (None, 0) or cand is None: + return "-" + change = (cand - base) / base * 100 + return f"{change:+.0f}%" + + +def print_single(rows): + header = list(KEY_FIELDS) + list(METRICS) + ["errors"] + print("| " + " | ".join(header) + " |") + print("|" + "---|" * len(header)) + for key in sorted(rows, key=str): + row = rows[key] + cells = [str(part) for part in key] + cells += [fmt(row.get(metric)) for metric in METRICS] + cells.append(str(row.get("error_count", 0))) + print("| " + " | ".join(cells) + " |") + + +def print_comparison(baseline, candidate): + header = list(KEY_FIELDS) + for metric in ("p50_ms", "p95_ms"): + header += [f"{metric} base", f"{metric} cand", "Δ"] + header += ["req base", "req cand", "bytes base", "bytes cand"] + print("| " + " | ".join(header) + " |") + print("|" + "---|" * len(header)) + + for key in sorted(set(baseline) | set(candidate), key=str): + base, cand = baseline.get(key, {}), candidate.get(key, {}) + cells = [str(part) for part in key] + for metric in ("p50_ms", "p95_ms"): + cells += [ + fmt(base.get(metric)), + fmt(cand.get(metric)), + delta(base.get(metric), cand.get(metric)), + ] + cells += [ + fmt(base.get("request_count_mean")), + fmt(cand.get("request_count_mean")), + fmt(base.get("bytes_received_mean")), + fmt(cand.get("bytes_received_mean")), + ] + print("| " + " | ".join(cells) + " |") + + +def main(argv): + if len(argv) == 2: + print_single(load(argv[1])) + elif len(argv) == 3: + print_comparison(load(argv[1]), load(argv[2])) + else: + print(__doc__, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh new file mode 100644 index 0000000000..bddc118e0c --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Runs the SDK config benchmark matrix and emits one JSONL row per configuration to stdout. +# Build logs and progress go to stderr, so redirecting stdout captures clean JSONL: +# +# bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > baseline.jsonl +# +# Requires the Tuist workspace: tuist install && tuist generate SDKConfigBenchmark +# +# Override the matrix through environment variables: +# MODES="legacy config config-killswitch" +# SCENARIOS="cold warm" +# PROFILES="ideal lte" +# LOSSES="0" # extra loss sweep applies to config/cold/lte below +# LOSS_SWEEP="10 20 30" +# ITERATIONS=25 WARMUP=3 PAYWALLS=50 WORKFLOWS=100 SEED=42 +# SKIP_BUILD=1 # reuse the previously built binary + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DERIVED_DATA="${DERIVED_DATA:-$REPO_ROOT/.build/sdk-config-benchmark-derived-data}" +BINARY="$DERIVED_DATA/Build/Products/Release/SDKConfigBenchmark" + +MODES="${MODES:-legacy config config-killswitch}" +SCENARIOS="${SCENARIOS:-cold warm}" +PROFILES="${PROFILES:-ideal lte}" +LOSSES="${LOSSES:-0}" +LOSS_SWEEP="${LOSS_SWEEP:-10 20 30}" +ITERATIONS="${ITERATIONS:-25}" +WARMUP="${WARMUP:-3}" +PAYWALLS="${PAYWALLS:-50}" +WORKFLOWS="${WORKFLOWS:-100}" +SEED="${SEED:-42}" + +if [[ "${SKIP_BUILD:-0}" != "1" ]]; then + echo "Building SDKConfigBenchmark (Release)..." >&2 + xcodebuild -workspace "$REPO_ROOT/RevenueCat-Tuist.xcworkspace" \ + -scheme SDKConfigBenchmark \ + -configuration Release \ + -destination platform=macOS \ + -derivedDataPath "$DERIVED_DATA" \ + build >&2 +fi + +if [[ ! -x "$BINARY" ]]; then + echo "Benchmark binary not found at $BINARY" >&2 + exit 1 +fi + +SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +BENCH_HOME="$(mktemp -d)" +trap 'rm -rf "$BENCH_HOME"' EXIT + +run_row() { + local mode="$1" scenario="$2" profile="$3" loss="$4" + echo "Running mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 + HOME="$BENCH_HOME" "$BINARY" \ + --mode "$mode" \ + --scenario "$scenario" \ + --profile "$profile" \ + --loss-percent "$loss" \ + --iterations "$ITERATIONS" \ + --warmup-iterations "$WARMUP" \ + --paywalls "$PAYWALLS" \ + --workflows "$WORKFLOWS" \ + --seed "$SEED" \ + --annotation "sdk_commit=$SDK_COMMIT" +} + +for mode in $MODES; do + for scenario in $SCENARIOS; do + for profile in $PROFILES; do + for loss in $LOSSES; do + run_row "$mode" "$scenario" "$profile" "$loss" + done + done + done +done + +# Loss sweep: degraded LTE for the two systems' cold starts. Warm scenarios are skipped here +# because loss-induced request failures make the 304/204 verification nondeterministic. +if [[ -n "$LOSS_SWEEP" ]]; then + for loss in $LOSS_SWEEP; do + for mode in legacy config; do + run_row "$mode" cold lte "$loss" + done + done +fi From c20edc13c3b40ffefcca4dde14e2e98aa8fb66ff Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 18:50:42 +0200 Subject: [PATCH 08/34] feat(benchmarks): add live transport pinned to the stress-test project --transport live re-issues requests against the real backend through a recording passthrough (same per-request metrics as simulated runs), defaults to the Stress Test Config Endpoint project's public test store key, and rejects the simulation-only knobs (profiles, loss, forced kill-switch 4xx). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 36 ++++ Tests/Benchmarks/SDKConfigBenchmark/README.md | 24 ++- .../Sources/BenchmarkCommand.swift | 64 ++++++- .../Sources/BenchmarkMetrics.swift | 12 +- .../Sources/BenchmarkRunner.swift | 61 +++--- .../SimulatedTransportURLProtocol.swift | 174 +++++++++++++++--- .../Tests/BenchmarkCommandTests.swift | 25 +++ .../Tests/PassthroughTransportTests.swift | 107 +++++++++++ .../SDKConfigBenchmark/run-matrix.sh | 22 ++- 9 files changed, 460 insertions(+), 65 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index 8fce8fd7ba..bd5cfa08ac 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -38,6 +38,20 @@ RC-Container, `PublishedWorkflow` blobs, `ui_config` blobs) with content-address that pass the SDK's checksum validation. StoreKit is replaced by a `ProductsManagerType` fake; everything else is production code. +## Transports + +- **`--transport simulated`** (default): the in-process network model below. Deterministic, + seedable, and the only way to measure loss, degraded profiles, and the forced kill-switch + 4xx. Use it for controlled A/B comparisons of SDK behavior and for CI regression tracking. +- **`--transport live`**: real requests against production, pinned to the prepared stress-test + project [`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3) ("Stress Test Config + Endpoint"; keys hardcoded in `BenchmarkProject`, Test Store key by default because the + project's packages live on its Test Store app). Requests flow through a recording + passthrough, so live rows carry the same per-request metrics: API traffic re-issues through a + single-connection pool mirroring `HTTPClient`, blob traffic through a default pool mirroring + the production blob downloader. Use it for real-world numbers (CDN, TLS, actual backend + latency) and for monitoring the two systems against real project content. + ## Network model All requests are intercepted in-process by `SimulatedTransportURLProtocol` (every host, @@ -118,6 +132,28 @@ macOS Release build, commit `c1958bf64`: | legacy | cold | lte | 30 | 293.1 | 585.1 | 1 | 64,972 | | config | cold | lte | 30 | 2,154.8 | 3,038.0 | 70 | 114,617 | +### Live sample numbers + +`TRANSPORT=live` matrix (25 iterations, 3 warmup discarded) against the stress-test project, +run 2026-07-08 from a residential connection: + +| mode | scenario | p50 ms | p95 ms | requests (mean) | bytes (mean) | +|---|---|---:|---:|---:|---:| +| legacy | cold | 163.6 | 334.9 | 1 | 32,124 | +| config | cold | 334.1 | 525.4 | 2 | 36,052 | +| legacy | warm | 122.7 | 321.1 | 1 | 0 | +| config | warm | 316.4 | 635.0 | 2 | 0 | + +Live observations (as of this run): + +- The project's `/v1/config` response currently drives **no blob downloads** (2 requests + total: config + offerings), so live config cost is simply one extra API round trip, + serialized behind offerings by `HTTPClient`'s single-connection queue. As the backend starts + serving workflow/paywall blobs for this project, live runs will pick that up automatically. +- Warm revalidation works end to end against production: offerings answers `304` to + `X-RevenueCat-ETag` and config answers `204` to a matching manifest, both with zero content + bytes. The runner verifies this and fails the run if it stops happening. + ## Interpretation - **Prefetch gating dominates config cold starts.** With 100 workflows all marked diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 0f20182e66..1c47afe5fd 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -1,8 +1,18 @@ # SDKConfigBenchmark A macOS command-line benchmark that measures the SDK's **legacy offerings flow** against the -**remote config endpoint flow** (and its kill switch) through the real manager-level code paths, -under simulated network conditions. +**remote config endpoint flow** (and its kill switch) through the real manager-level code paths. + +Two transports: + +- **`--transport simulated`** (default): in-process fixtures with seeded network profiles + (ideal/wifi/lte), packet-loss modeling, and a forced kill-switch 4xx. Deterministic and + CI-friendly. +- **`--transport live`**: real requests against the production backend, pinned to the + prepared stress-test project + ([`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3), hardcoded key). Same recorded + per-request metrics; real CDN/TLS/latency behavior. Kill-switch, profiles, and loss are + simulated-only (you cannot force a 4xx or packet loss on production). See `CONFIG_ENDPOINT_BENCHMARKS.md` for methodology, scenario definitions, sample numbers, and known limitations. @@ -17,7 +27,8 @@ tuist generate SDKConfigBenchmark SDKConfigBenchmarkTests ## Run the matrix ```sh -bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl +bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl # simulated +TRANSPORT=live bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > live.jsonl ``` Compare two runs (e.g. baseline branch vs candidate branch): @@ -39,10 +50,11 @@ xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmark \ -configuration Release -destination platform=macOS build SDKConfigBenchmark \ - --mode config \ # legacy | config | config-killswitch + --transport simulated \ # simulated | live + --mode config \ # legacy | config | config-killswitch (killswitch: simulated only) --scenario cold \ # cold | warm - --profile lte \ # ideal | wifi | lte - --loss-percent 20 \ + --profile lte \ # ideal | wifi | lte (simulated only) + --loss-percent 20 \ # simulated only --iterations 25 \ --warmup-iterations 3 \ --paywalls 50 \ diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 696cf31287..3167aa9a46 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -16,6 +16,32 @@ enum BenchmarkMode: String, CaseIterable { } +enum BenchmarkTransport: String, CaseIterable { + + /// In-process fixture transport: deterministic, seedable, supports profiles, loss, and the + /// forced kill-switch 4xx. + case simulated + + /// Real requests against the production backend, recorded for metrics. Pinned to the + /// prepared stress-test project (`BenchmarkProject`). + case live + +} + +/// The canonical RevenueCat project live runs measure against: +/// https://app.revenuecat.com/projects/5f07e7e3 ("Stress Test Config Endpoint"), prepared with +/// a large number of paywalls and workflows. Keys are the project's client-side public SDK +/// keys, hardcoded on purpose so every live run measures the same content. The Test Store key +/// is the default because the project's packages live on its Test Store app; the App Store app +/// has no products registered, which makes `OfferingsManager` fail with a configuration error. +enum BenchmarkProject { + + static let dashboardURL = "https://app.revenuecat.com/projects/5f07e7e3" + static let testStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" + static let appStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" + +} + enum BenchmarkScenario: String, CaseIterable { /// Every iteration starts with empty disk caches and a fresh app user ID. @@ -30,7 +56,10 @@ enum BenchmarkScenario: String, CaseIterable { struct BenchmarkCommand { + static let defaultSimulatedAPIKey = "appl_benchmark" + var mode: BenchmarkMode = .legacy + var transport: BenchmarkTransport = .simulated var scenario: BenchmarkScenario = .cold var profileName: String = "ideal" var lossPercent: Int = 0 @@ -40,7 +69,7 @@ struct BenchmarkCommand { var workflowCount: Int = 100 var seed: UInt64 = 42 var appUserID: String = "benchmark-user" - var apiKey: String = "appl_benchmark" + var apiKey: String = BenchmarkCommand.defaultSimulatedAPIKey /// Extra key=value pairs echoed verbatim into the JSONL row (e.g. sdk_commit=abc123). var annotations: [String: String] = [:] @@ -74,6 +103,12 @@ struct BenchmarkCommand { throw BenchmarkError.invalidArgument("unknown mode \(raw)") } command.mode = mode + case "--transport": + let raw = try value(for: flag) + guard let transport = BenchmarkTransport(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown transport \(raw)") + } + command.transport = transport case "--scenario": let raw = try value(for: flag) guard let scenario = BenchmarkScenario(rawValue: raw) else { @@ -121,7 +156,34 @@ struct BenchmarkCommand { ) } + try command.validateAndDefaultTransport() + return command } + /// Live runs hit the real backend: the knobs that shape the simulated network (and the + /// forced kill-switch 4xx) cannot apply there, and the API key defaults to the pinned + /// stress-test project. + private mutating func validateAndDefaultTransport() throws { + guard self.transport == .live else { return } + + guard self.lossPercent == 0 else { + throw BenchmarkError.invalidArgument("--loss-percent requires --transport simulated") + } + guard self.profileName == "ideal" else { + throw BenchmarkError.invalidArgument( + "--profile requires --transport simulated; live runs use the real network" + ) + } + guard self.mode != .configKillswitch else { + throw BenchmarkError.invalidArgument( + "config-killswitch cannot force a 4xx on the real backend; use --transport simulated" + ) + } + + if self.apiKey == Self.defaultSimulatedAPIKey { + self.apiKey = BenchmarkProject.testStoreAPIKey + } + } + } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index dc68f3d1b9..d9f1b929b1 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -20,9 +20,14 @@ struct IterationMeasurement { init(totalMs: Double, events: [TransportEvent]) { self.totalMs = totalMs - self.offeringsMs = Self.span(of: events.filter { $0.path.hasSuffix("/offerings") }) - self.configMs = Self.span(of: events.filter { $0.path.hasSuffix("/config/app") }) - self.blobMs = Self.span(of: events.filter { $0.path.contains("/blobs/") }) + let offerings = events.filter { $0.path.hasSuffix("/offerings") } + let config = events.filter { $0.path.hasSuffix("/config/app") } + // Blob URLs come from the (real or fixture) config's url_format, so classify them as + // everything that is neither an offerings nor a config request. + let blobs = events.filter { !$0.path.hasSuffix("/offerings") && !$0.path.hasSuffix("/config/app") } + self.offeringsMs = Self.span(of: offerings) + self.configMs = Self.span(of: config) + self.blobMs = Self.span(of: blobs) self.requestCount = events.count self.bytesReceived = events.reduce(0) { $0 + $1.bytesReceived } self.failedRequestCount = events.filter(\.failed).count @@ -72,6 +77,7 @@ struct BenchmarkMetrics { var row: [String: Any] = [ "mode": command.mode.rawValue, + "transport": command.transport.rawValue, "scenario": command.scenario.rawValue, "profile": command.profileName, "loss_percent": command.lossPercent, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index a12a7739db..d1e6d319c0 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -18,24 +18,30 @@ final class BenchmarkRunner { } func run() throws -> String { - guard let profile = NetworkProfile.named(self.command.profileName) else { - throw BenchmarkError.invalidArgument("unknown profile \(self.command.profileName)") + switch self.command.transport { + case .simulated: + guard let profile = NetworkProfile.named(self.command.profileName) else { + throw BenchmarkError.invalidArgument("unknown profile \(self.command.profileName)") + } + let factory = BenchmarkPayloadFactory( + paywallCount: self.command.paywallCount, + workflowCount: self.command.workflowCount + ) + let server = FixtureServer( + factory: factory, + killSwitchConfig: self.command.mode == .configKillswitch + ) + SimulatedTransportURLProtocol.install( + server: server, + profile: profile, + loss: LossModel(lossPercent: self.command.lossPercent), + seed: self.command.seed + ) + case .live: + // Requests hit the real backend (the pinned stress-test project) through a + // recording passthrough, so live rows carry the same per-request metrics. + SimulatedTransportURLProtocol.installPassthrough() } - - let factory = BenchmarkPayloadFactory( - paywallCount: self.command.paywallCount, - workflowCount: self.command.workflowCount - ) - let server = FixtureServer( - factory: factory, - killSwitchConfig: self.command.mode == .configKillswitch - ) - SimulatedTransportURLProtocol.install( - server: server, - profile: profile, - loss: LossModel(lossPercent: self.command.lossPercent), - seed: self.command.seed - ) defer { SimulatedTransportURLProtocol.uninstall() } var metrics = BenchmarkMetrics() @@ -61,12 +67,25 @@ final class BenchmarkRunner { private extension BenchmarkRunner { + /// Live runs get a per-run nonce so reruns never share server-side per-user state with a + /// previous run; simulated runs stay fully deterministic. + private static let liveRunNonce = String(Int(Date().timeIntervalSince1970)) + + var baseAppUserID: String { + switch self.command.transport { + case .simulated: + return self.command.appUserID + case .live: + return "\(self.command.appUserID)-\(Self.liveRunNonce)" + } + } + func appUserID(forIteration iteration: Int) -> String { switch self.command.scenario { case .cold: - return "\(self.command.appUserID)-\(iteration)" + return "\(self.baseAppUserID)-\(iteration)" case .warm: - return self.command.appUserID + return self.baseAppUserID } } @@ -75,10 +94,10 @@ private extension BenchmarkRunner { let stack = BenchmarkSDKStack( mode: self.command.mode, apiKey: self.command.apiKey, - appUserID: self.command.appUserID + appUserID: self.baseAppUserID ) stack.clearAllDiskState() - _ = try self.launch(stack, appUserID: self.command.appUserID) + _ = try self.launch(stack, appUserID: self.baseAppUserID) _ = SimulatedTransportURLProtocol.drainEvents() } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index a10043a808..4b2b8897a6 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -13,24 +13,28 @@ struct TransportEvent { } -/// In-process transport used by every URLSession in the benchmark. +/// The transport used by every URLSession in the benchmark. Two modes: /// -/// `canInit` claims EVERY http(s) request, so no benchmark run can ever touch the real network; -/// fallback-host retries (`api-production.8-lives-cat.io`) resolve against the same fixture -/// server as the primary host. Responses are delivered asynchronously on a private queue: -/// headers after one sampled RTT, then the body in chunks paced by the profile's bandwidth and -/// the loss model's retransmission delays. Nothing here ever blocks a thread, so concurrent -/// requests overlap the way they would on a real connection pool. +/// **Simulated**: requests resolve against an in-process `FixtureServer`, so no run can ever +/// touch the real network; fallback-host retries (`api-production.8-lives-cat.io`) resolve +/// against the same fixtures as the primary host. Responses are delivered asynchronously on a +/// private queue: headers after one sampled RTT, then the body in chunks paced by the profile's +/// bandwidth and the loss model's retransmission delays. Nothing here ever blocks a thread, so +/// concurrent requests overlap the way they would on a real connection pool. +/// +/// **Passthrough** (live runs): requests are re-issued unmodified against the real network and +/// recorded, so live runs get the same per-request metrics as simulated ones. API hosts go +/// through a single-connection session mirroring `HTTPClient`'s pool; every other host (blob +/// CDNs) goes through a default pool like the production blob downloader's `URLSession.shared`. final class SimulatedTransportURLProtocol: URLProtocol { - private struct Installation { - let server: FixtureServer - let profile: NetworkProfile - let loss: LossModel + private enum Mode { + case simulated(server: FixtureServer, profile: NetworkProfile, loss: LossModel) + case passthrough } private static let lock = NSLock() - private static var installation: Installation? + private static var mode: Mode? private static var rng = SeededRandom(seed: 0) private static var events: [TransportEvent] = [] @@ -40,19 +44,27 @@ final class SimulatedTransportURLProtocol: URLProtocol { /// each get their own queue and overlap freely. private let deliveryQueue = DispatchQueue(label: "com.revenuecat.benchmark.transport.request") private var pendingWorkItems: [DispatchWorkItem] = [] + private var passthroughTask: URLSessionDataTask? private let stateLock = NSLock() static func install(server: FixtureServer, profile: NetworkProfile, loss: LossModel, seed: UInt64) { self.lock.withLock { - self.installation = Installation(server: server, profile: profile, loss: loss) + self.mode = .simulated(server: server, profile: profile, loss: loss) self.rng = SeededRandom(seed: seed) self.events = [] } } + static func installPassthrough() { + self.lock.withLock { + self.mode = .passthrough + self.events = [] + } + } + static func uninstall() { self.lock.withLock { - self.installation = nil + self.mode = nil self.events = [] } } @@ -78,7 +90,10 @@ final class SimulatedTransportURLProtocol: URLProtocol { // MARK: - URLProtocol override class func canInit(with request: URLRequest) -> Bool { - guard let scheme = request.url?.scheme else { return false } + guard self.lock.withLock({ self.mode }) != nil, + let scheme = request.url?.scheme else { + return false + } return scheme == "http" || scheme == "https" } @@ -89,29 +104,61 @@ final class SimulatedTransportURLProtocol: URLProtocol { override func startLoading() { let started = DispatchTime.now() guard let url = self.request.url, - let installation = Self.lock.withLock({ Self.installation }) else { + let mode = Self.lock.withLock({ Self.mode }) else { self.client?.urlProtocol( self, - didFailWithError: BenchmarkError.invalidFixture("Simulated transport not installed") + didFailWithError: BenchmarkError.invalidFixture("Benchmark transport not installed") ) return } + switch mode { + case let .simulated(server, profile, loss): + self.startSimulated(url: url, server: server, profile: profile, loss: loss, startedAt: started) + case .passthrough: + self.startPassthrough(url: url, startedAt: started) + } + } + + override func stopLoading() { + self.stateLock.withLock { + for item in self.pendingWorkItems { + item.cancel() + } + self.pendingWorkItems = [] + self.passthroughTask?.cancel() + self.passthroughTask = nil + } + } + +} + +// MARK: - Simulated delivery + +private extension SimulatedTransportURLProtocol { + + func startSimulated( + url: URL, + server: FixtureServer, + profile: NetworkProfile, + loss: LossModel, + startedAt: DispatchTime + ) { let host = url.host ?? "" let bodyData = Self.bodyData(of: self.request) - let response = installation.server.response(for: self.request, bodyData: bodyData) + let response = server.response(for: self.request, bodyData: bodyData) // Sample every random decision up front, under one lock, so the request's timeline is // fixed at start time regardless of delivery interleaving. let plan: (rttMs: Double, fails: Bool, chunkDelaysMs: [Double]) = Self.lock.withLock { - let rttMs = installation.profile.rttMs(forHost: host, rng: &Self.rng) - let fails = installation.loss.shouldFailRequest(rng: &Self.rng) + let rttMs = profile.rttMs(forHost: host, rng: &Self.rng) + let fails = loss.shouldFailRequest(rng: &Self.rng) var chunkDelaysMs: [Double] = [] if !fails { var offset = 0 while offset < max(response.body.count, 1) { chunkDelaysMs.append( - installation.loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &Self.rng) + loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &Self.rng) ) offset += Self.chunkSize } @@ -120,25 +167,94 @@ final class SimulatedTransportURLProtocol: URLProtocol { } if plan.fails { - self.deliverFailure(host: host, path: url.path, rttMs: plan.rttMs, startedAt: started) + self.deliverFailure(host: host, path: url.path, rttMs: plan.rttMs, startedAt: startedAt) } else { self.deliverResponse( response, url: url, - profile: installation.profile, + profile: profile, plan: (plan.rttMs, plan.chunkDelaysMs), - startedAt: started + startedAt: startedAt ) } } - override func stopLoading() { - self.stateLock.withLock { - for item in self.pendingWorkItems { - item.cancel() +} + +// MARK: - Passthrough recording (live runs) + +extension SimulatedTransportURLProtocol { + + /// Sessions the passthrough re-issues requests on. API traffic mirrors `HTTPClient`'s + /// single-connection-per-host pool; everything else (blob CDNs) gets a default pool like + /// the production blob downloader's `URLSession.shared`. Injectable for tests. + static var passthroughAPISession: URLSession = makePassthroughSession(maxConnectionsPerHost: 1) + static var passthroughBlobSession: URLSession = makePassthroughSession(maxConnectionsPerHost: nil) + + private static func makePassthroughSession(maxConnectionsPerHost: Int?) -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.urlCache = nil + if let maxConnectionsPerHost { + configuration.httpMaximumConnectionsPerHost = maxConnectionsPerHost + } + return URLSession(configuration: configuration) + } + + static func isAPIHost(_ host: String) -> Bool { + return host.contains("revenuecat.com") && host.hasPrefix("api.") + || host.contains("8-lives-cat") + || host.contains("rc-backup") + } + + private func startPassthrough(url: URL, startedAt: DispatchTime) { + let host = url.host ?? "" + let session = Self.isAPIHost(host) ? Self.passthroughAPISession : Self.passthroughBlobSession + + let task = session.dataTask(with: self.request) { [weak self] data, response, error in + guard let self else { return } + let ended = DispatchTime.now() + + if let error { + Self.record(TransportEvent( + host: host, + path: url.path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: ended, + failed: true + )) + self.client?.urlProtocol(self, didFailWithError: error) + return } - self.pendingWorkItems = [] + + guard let httpResponse = response as? HTTPURLResponse else { + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(host)") + ) + return + } + + Self.record(TransportEvent( + host: host, + path: url.path, + statusCode: httpResponse.statusCode, + bytesReceived: data?.count ?? 0, + startedAt: startedAt, + endedAt: ended, + failed: false + )) + self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + if let data, !data.isEmpty { + self.client?.urlProtocol(self, didLoad: data) + } + self.client?.urlProtocolDidFinishLoading(self) + } + self.stateLock.withLock { + self.passthroughTask = task } + task.resume() } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index 526713ab18..db9c5fb894 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -70,4 +70,29 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertThrowsError(try BenchmarkCommand.parse(["--loss-percent", "101"])) } + // MARK: - Transport + + func testTransportDefaultsToSimulated() throws { + XCTAssertEqual(try BenchmarkCommand.parse([]).transport, .simulated) + } + + func testLiveTransportDefaultsToPinnedProjectAPIKey() throws { + let command = try BenchmarkCommand.parse(["--transport", "live"]) + + XCTAssertEqual(command.transport, .live) + XCTAssertEqual(command.apiKey, BenchmarkProject.testStoreAPIKey) + } + + func testLiveTransportKeepsExplicitAPIKey() throws { + let command = try BenchmarkCommand.parse(["--transport", "live", "--api-key", "appl_other"]) + + XCTAssertEqual(command.apiKey, "appl_other") + } + + func testLiveTransportRejectsSimulationOnlyKnobs() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--loss-percent", "10"])) + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--profile", "lte"])) + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--mode", "config-killswitch"])) + } + } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift new file mode 100644 index 0000000000..c3a807570d --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift @@ -0,0 +1,107 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +/// Passthrough (live) mode, verified against an in-test stub backend instead of the real +/// network: requests re-issue through the passthrough sessions and get recorded as events. +final class PassthroughTransportTests: BenchmarkTestCase { + + private var originalAPISession: URLSession! + private var originalBlobSession: URLSession! + + override func setUp() { + super.setUp() + self.originalAPISession = SimulatedTransportURLProtocol.passthroughAPISession + self.originalBlobSession = SimulatedTransportURLProtocol.passthroughBlobSession + + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubBackendURLProtocol.self] + let stubSession = URLSession(configuration: configuration) + SimulatedTransportURLProtocol.passthroughAPISession = stubSession + SimulatedTransportURLProtocol.passthroughBlobSession = stubSession + } + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + SimulatedTransportURLProtocol.passthroughAPISession = self.originalAPISession + SimulatedTransportURLProtocol.passthroughBlobSession = self.originalBlobSession + StubBackendURLProtocol.requestCount = 0 + super.tearDown() + } + + func testPassthroughForwardsResponseAndRecordsEvent() throws { + SimulatedTransportURLProtocol.installPassthrough() + let session = SimulatedTransportURLProtocol.makeSession() + let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + + let expectation = self.expectation(description: "request completes") + var received: (data: Data?, statusCode: Int?) + session.dataTask(with: url) { data, response, _ in + received = (data, (response as? HTTPURLResponse)?.statusCode) + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: 5) + + XCTAssertEqual(received.statusCode, 200) + XCTAssertEqual(received.data, StubBackendURLProtocol.stubBody) + XCTAssertEqual(StubBackendURLProtocol.requestCount, 1, "request must reach the (stub) backend") + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.statusCode, 200) + XCTAssertEqual(events.first?.bytesReceived, StubBackendURLProtocol.stubBody.count) + XCTAssertEqual(events.first?.failed, false) + } + + func testAPIHostClassification() { + XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api.revenuecat.com")) + XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api-production.8-lives-cat.io")) + XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api.rc-backup.com")) + XCTAssertFalse(SimulatedTransportURLProtocol.isAPIHost("blobs.revenuecat.com")) + XCTAssertFalse(SimulatedTransportURLProtocol.isAPIHost("d1234.cloudfront.net")) + } + + func testUninstalledTransportDoesNotClaimRequests() { + SimulatedTransportURLProtocol.uninstall() + + let request = URLRequest(url: URL(fileURLWithPath: "/dev/null")) + XCTAssertFalse(SimulatedTransportURLProtocol.canInit(with: request)) + } + +} + +/// Terminal stub for passthrough tests: answers every request on its session with a canned +/// 200 so no test traffic ever leaves the process. +private final class StubBackendURLProtocol: URLProtocol { + + static let stubBody = Data(#"{"stub":true}"#.utf8) + static var requestCount = 0 + + override class func canInit(with request: URLRequest) -> Bool { + return true + } + + override class func canonicalRequest(for request: URLRequest) -> URLRequest { + return request + } + + override func startLoading() { + Self.requestCount += 1 + guard let url = self.request.url, + let response = HTTPURLResponse( + url: url, + statusCode: 200, + httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + ) else { + self.client?.urlProtocol(self, didFailWithError: URLError(.badURL)) + return + } + self.client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + self.client?.urlProtocol(self, didLoad: Self.stubBody) + self.client?.urlProtocolDidFinishLoading(self) + } + + override func stopLoading() {} + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index bddc118e0c..305d9035f4 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -8,6 +8,9 @@ # Requires the Tuist workspace: tuist install && tuist generate SDKConfigBenchmark # # Override the matrix through environment variables: +# TRANSPORT=live # hit the real backend (pinned stress-test project) instead of +# # the simulated transport; forces ideal profile, no loss, and +# # drops the kill-switch mode (cannot force 4xx on production) # MODES="legacy config config-killswitch" # SCENARIOS="cold warm" # PROFILES="ideal lte" @@ -22,11 +25,19 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" DERIVED_DATA="${DERIVED_DATA:-$REPO_ROOT/.build/sdk-config-benchmark-derived-data}" BINARY="$DERIVED_DATA/Build/Products/Release/SDKConfigBenchmark" -MODES="${MODES:-legacy config config-killswitch}" +TRANSPORT="${TRANSPORT:-simulated}" +if [[ "$TRANSPORT" == "live" ]]; then + MODES="${MODES:-legacy config}" + PROFILES="ideal" + LOSSES="0" + LOSS_SWEEP="" +else + MODES="${MODES:-legacy config config-killswitch}" + PROFILES="${PROFILES:-ideal lte}" + LOSSES="${LOSSES:-0}" + LOSS_SWEEP="${LOSS_SWEEP:-10 20 30}" +fi SCENARIOS="${SCENARIOS:-cold warm}" -PROFILES="${PROFILES:-ideal lte}" -LOSSES="${LOSSES:-0}" -LOSS_SWEEP="${LOSS_SWEEP:-10 20 30}" ITERATIONS="${ITERATIONS:-25}" WARMUP="${WARMUP:-3}" PAYWALLS="${PAYWALLS:-50}" @@ -54,8 +65,9 @@ trap 'rm -rf "$BENCH_HOME"' EXIT run_row() { local mode="$1" scenario="$2" profile="$3" loss="$4" - echo "Running mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 + echo "Running transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 HOME="$BENCH_HOME" "$BINARY" \ + --transport "$TRANSPORT" \ --mode "$mode" \ --scenario "$scenario" \ --profile "$profile" \ From 8702995757dc1c4368f8e9b408ebb57da9912e9a Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 19:37:51 +0200 Subject: [PATCH 09/34] other(benchmarks): apply review-loop fixes for measurement validity - request network plans keyed by (seed, iteration, url, attempt) so sampling is independent of scheduler-driven arrival order; same-seed loss-free rows are byte-identical across processes - ui_config blobs no longer prefetched, so no downloads straddle the measured offerings completion and per-iteration accounting stays exact - warmup discard is by iteration index, rows carry measured_iterations and post_warmup_error_count, and compare.py flags rows with post-warmup errors - every measured warm iteration must revalidate (offerings 304, config 204, zero blob traffic) instead of one loose whole-run check - each process isolates all SDK disk caches under a temp root via a gated DirectoryHelper override (container dirs ignore HOME), fixing concurrent-run corruption and real-Library pollution; tests are hermetic the same way - benchmark swift conditions no longer clobbered by TUIST_SWIFT_CONDITIONS Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmark/Project.swift | 13 +- Sources/Caching/DirectoryHelper.swift | 21 +++ .../CONFIG_ENDPOINT_BENCHMARKS.md | 67 ++++++--- Tests/Benchmarks/SDKConfigBenchmark/README.md | 9 +- .../Sources/BenchmarkMain.swift | 16 +- .../Sources/BenchmarkMetrics.swift | 92 +++++++----- .../Sources/BenchmarkPayloadFactory.swift | 10 +- .../Sources/BenchmarkRunner.swift | 60 ++++++-- ...atedTransportURLProtocol+Passthrough.swift | 79 ++++++++++ .../SimulatedTransportURLProtocol+Plans.swift | 63 ++++++++ .../SimulatedTransportURLProtocol.swift | 137 +++++------------- .../Tests/BenchmarkMetricsTests.swift | 49 ++++++- .../Tests/BenchmarkPayloadFactoryTests.swift | 14 +- .../BenchmarkRunnerValidationTests.swift | 86 +++++++++++ .../Tests/BenchmarkSDKStackTests.swift | 4 +- .../Tests/BenchmarkTestCase.swift | 26 +++- .../Tests/NetworkProfileTests.swift | 27 ++++ .../Benchmarks/SDKConfigBenchmark/compare.py | 21 ++- .../SDKConfigBenchmark/run-matrix.sh | 4 +- 19 files changed, 598 insertions(+), 200 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift diff --git a/Projects/SDKConfigBenchmark/Project.swift b/Projects/SDKConfigBenchmark/Project.swift index 9fdaf0b938..c74165b1c2 100644 --- a/Projects/SDKConfigBenchmark/Project.swift +++ b/Projects/SDKConfigBenchmark/Project.swift @@ -6,8 +6,14 @@ import ProjectDescriptionHelpers // local-source mode) so the benchmark can drive internal manager-level APIs without // exposing new public SDK API. `SDK_CONFIG_BENCHMARK` installs a simulated transport // in `HTTPClient`; `ENABLE_REMOTE_CONFIG` turns on the remote config gate. +// +// TUIST_SWIFT_CONDITIONS is folded in by hand instead of via +// `appendingTuistSwiftConditions()`, because that helper replaces the whole +// SWIFT_ACTIVE_COMPILATION_CONDITIONS value and would silently drop the benchmark flags. let benchmarkSwiftConditions: SettingsDictionary = [ - "SWIFT_ACTIVE_COMPILATION_CONDITIONS": "$(inherited) SDK_CONFIG_BENCHMARK ENABLE_REMOTE_CONFIG" + "SWIFT_ACTIVE_COMPILATION_CONDITIONS": SettingValue(stringLiteral: ( + ["$(inherited)", "SDK_CONFIG_BENCHMARK", "ENABLE_REMOTE_CONFIG"] + Environment.extraSwiftConditions + ).joined(separator: " ")) ] let project = Project( @@ -35,7 +41,7 @@ let project = Project( .storeKit ], settings: .settings( - base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + base: benchmarkSwiftConditions ) ), .target( @@ -57,7 +63,6 @@ let project = Project( // the binary gives it one, so etags/offerings/remote-config persistence works. base: benchmarkSwiftConditions .merging(["CREATE_INFOPLIST_SECTION_IN_BINARY": "YES"]) - .appendingTuistSwiftConditions() ) ), .target( @@ -74,7 +79,7 @@ let project = Project( .target(name: "SDKConfigBenchmarkCore") ], settings: .settings( - base: benchmarkSwiftConditions.appendingTuistSwiftConditions() + base: benchmarkSwiftConditions ) ) ], diff --git a/Sources/Caching/DirectoryHelper.swift b/Sources/Caching/DirectoryHelper.swift index 2b4a0981c9..c88dac826b 100644 --- a/Sources/Caching/DirectoryHelper.swift +++ b/Sources/Caching/DirectoryHelper.swift @@ -37,7 +37,28 @@ enum DirectoryHelper { return Self.baseUrl(for: directoryType, inAppSpecificDirectory: false) } + #if SDK_CONFIG_BENCHMARK + /// Benchmark-only: when set, every SDK disk cache resolves under this root. The standard + /// container directories ignore $HOME, so without this override concurrent benchmark + /// processes would share (and corrupt) one another's caches in the real user Library. + /// Set once at process startup, before any disk access. + static var benchmarkBaseDirectoryOverride: URL? + #endif + static func baseUrl(for type: DirectoryType, inAppSpecificDirectory: Bool = true) -> URL? { + #if SDK_CONFIG_BENCHMARK + if let overrideRoot = Self.benchmarkBaseDirectoryOverride { + switch type { + case .cache: + return overrideRoot.appendingPathComponent("caches") + #if !os(tvOS) + case .applicationSupport: + return overrideRoot.appendingPathComponent("application-support") + #endif + } + } + #endif + guard let baseDirectory = type.url else { return nil } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index bd5cfa08ac..c912c5b563 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -68,8 +68,25 @@ delivered asynchronously with: exercises the SDK's retry and fallback-host logic. Profiles: `ideal` (0ms, unlimited), `wifi` (25-40ms API / 10-20ms CDN, ~50 Mbit/s), `lte` -(55-110ms API / 30-70ms CDN, ~12 Mbit/s). All randomness is seeded (`--seed`), so runs are -reproducible. +(55-110ms API / 30-70ms CDN, ~12 Mbit/s). + +**Determinism contract.** Every request's network plan (RTT, failure, chunk delays) is derived +from a stable key: seed, iteration, URL, and per-URL attempt. Request arrival order, which is +scheduler-dependent, cannot influence sampling. Measured on this machine: + +- Loss-free rows are exactly reproducible across processes for the same seed: identical + request counts, bytes, and failure counts; timings differ only by wall-clock jitter + (a few percent, which the warmup discard and percentiles absorb). +- Rows with loss keep deterministic per-request plans (identical failure and fallback-host + counts across processes), but the SDK's concurrent reaction to failures (blob-fetcher + failover and retry scheduling) is real thread-scheduling behavior, so request counts can + drift by about one request and timings by a few percent. That residual variation is the + system under test, not the harness, and faking it away would distort the measurement. + +Each benchmark process isolates all SDK disk caches under its own temporary root (removed on +exit); concurrent runs cannot corrupt each other and nothing touches the real user Library. +One caveat: the scratch `UserDefaults` suite is per-user, not per-process, but it only carries +cache timestamps and identity bookkeeping, never response bodies or ETags. **Honest caveat:** the loss model approximates TCP behavior; it does not simulate real packet loss (retransmission dynamics, congestion control, TLS handshakes). It is good enough to @@ -84,8 +101,9 @@ loss, use a local HTTP server plus the OS Network Link Conditioner. against the retained disk state, simulating an app relaunch. The fixture honors the SDK's actual revalidation mechanisms: offerings revalidate via `X-RevenueCat-ETag` -> `304`, config revalidates via its manifest token -> `204`, and blobs are content-addressed so they are never - re-downloaded. The runner **fails loudly** if a warm run never sees a 304/204, so the scenario - can't silently degrade into measuring cold behavior. + re-downloaded. The runner **fails loudly** unless every measured warm iteration revalidates + (offerings all-304, config all-204 in config mode, zero blob re-downloads), so the scenario + can't silently mix cold behavior into the warm distribution. The first `--warmup-iterations` (default 3) measured iterations are discarded from statistics so one-time process costs don't pollute the distribution. @@ -109,28 +127,31 @@ plus `sdk_commit`, so result files are self-describing. ## Sample numbers Default matrix (25 iterations, 3 warmup discarded, 50 paywalls, 100 workflows, seed 42), -macOS Release build, commit `c1958bf64`: +macOS Release build: | mode | scenario | profile | loss % | p50 ms | p95 ms | requests (mean) | bytes (mean) | |---|---|---|---:|---:|---:|---:|---:| -| legacy | cold | ideal | 0 | 6.8 | 7.6 | 1 | 64,972 | -| config | cold | ideal | 0 | 25.2 | 28.7 | 102 | 132,647 | -| config-killswitch | cold | ideal | 0 | 7.3 | 7.6 | 2 | 65,022 | -| legacy | cold | lte | 0 | 146.5 | 166.4 | 1 | 64,972 | -| config | cold | lte | 0 | 1,493.0 | 1,589.2 | 103 | 132,704 | -| config-killswitch | cold | lte | 0 | 232.1 | 274.8 | 2 | 65,022 | -| legacy | warm | ideal | 0 | 7.2 | 7.4 | 1 | 0 | -| config | warm | ideal | 0 | 10.4 | 10.9 | 2 | 0 | -| config-killswitch | warm | ideal | 0 | 8.3 | 11.0 | 2 | 50 | -| legacy | warm | lte | 0 | 101.0 | 127.8 | 1 | 0 | -| config | warm | lte | 0 | 212.4 | 231.6 | 2 | 0 | -| config-killswitch | warm | lte | 0 | 191.1 | 229.4 | 2 | 50 | -| legacy | cold | lte | 10 | 163.1 | 385.1 | 1 | 64,972 | -| config | cold | lte | 10 | 1,712.9 | 1,829.5 | 102 | 131,882 | -| legacy | cold | lte | 20 | 284.2 | 448.0 | 1 | 64,972 | -| config | cold | lte | 20 | 1,963.5 | 2,466.3 | 97 | 129,178 | -| legacy | cold | lte | 30 | 293.1 | 585.1 | 1 | 64,972 | -| config | cold | lte | 30 | 2,154.8 | 3,038.0 | 70 | 114,617 | +| legacy | cold | ideal | 0 | 7.2 | 7.5 | 1 | 64,972 | +| config | cold | ideal | 0 | 24.8 | 27.1 | 102 | 132,550 | +| config-killswitch | cold | ideal | 0 | 7.6 | 7.9 | 2 | 65,022 | +| legacy | cold | lte | 0 | 138.2 | 174.2 | 1 | 64,972 | +| config | cold | lte | 0 | 1,457.7 | 1,522.3 | 102 | 132,550 | +| config-killswitch | cold | lte | 0 | 234.7 | 273.8 | 2 | 65,022 | +| legacy | warm | ideal | 0 | 7.6 | 7.8 | 1 | 0 | +| config | warm | ideal | 0 | 10.4 | 11.2 | 2 | 0 | +| config-killswitch | warm | ideal | 0 | 8.2 | 8.5 | 2 | 50 | +| legacy | warm | lte | 0 | 95.5 | 123.0 | 1 | 0 | +| config | warm | lte | 0 | 184.2 | 233.0 | 2 | 0 | +| config-killswitch | warm | lte | 0 | 185.3 | 227.7 | 2 | 50 | +| legacy | cold | lte | 10 | 154.9 | 288.9 | 1 | 64,972 | +| config | cold | lte | 10 | 1,764.6 | 2,152.9 | 102 | 132,503 | +| legacy | cold | lte | 20 | 162.2 | 297.3 | 1 | 64,972 | +| config | cold | lte | 20 | 2,108.9 | 3,011.0 | 94 | 127,878 | +| legacy | cold | lte | 30 | 219.1 | 626.0 | 1 | 64,972 | +| config | cold | lte | 30 | 1,967.8 | 2,847.1 | 62 | 110,680 | + +Every warm row above is verified per iteration: offerings revalidated via 304, config via +manifest 204 (or the expected 4xx in kill-switch mode), and zero blob re-downloads. ### Live sample numbers diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 1c47afe5fd..507fd96d79 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -65,12 +65,9 @@ SDKConfigBenchmark \ Output is one JSON object (a JSONL row) on stdout. -Run the benchmark with a scratch `HOME` (the matrix script does this automatically) so the -SDK's disk caches don't touch your real user directory: - -```sh -HOME=$(mktemp -d) SDKConfigBenchmark --mode legacy --scenario cold --profile ideal -``` +Every run isolates the SDK's disk caches (ETags, offerings, remote config, blobs) under a +fresh temporary directory that is removed on exit, so runs never touch the real user Library +and concurrent runs cannot corrupt each other. ## Unit tests diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift index f77eb6910d..d7236be212 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift @@ -23,14 +23,26 @@ public struct BenchmarkMain { exit(1) } + // Every run gets its own disk-cache root: the SDK's container directories ignore + // $HOME, so this is the only way concurrent runs stay isolated (and the user's real + // Library stays clean). + let diskRoot = FileManager.default.temporaryDirectory + .appendingPathComponent("SDKConfigBenchmark-\(UUID().uuidString)", isDirectory: true) + DirectoryHelper.benchmarkBaseDirectoryOverride = diskRoot + + func finish(exitCode: Int32) -> Never { + try? FileManager.default.removeItem(at: diskRoot) + exit(exitCode) + } + DispatchQueue.global(qos: .userInitiated).async { do { let row = try BenchmarkRunner(command: command).run() print(row) - exit(0) + finish(exitCode: 0) } catch { FileHandle.standardError.write(Data("\(error)\n".utf8)) - exit(1) + finish(exitCode: 1) } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index d9f1b929b1..fe4ef7857a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -15,8 +15,10 @@ struct IterationMeasurement { let bytesReceived: Int let failedRequestCount: Int let fallbackHostRequestCount: Int - /// Statuses seen, for warm-scenario verification (304/204). - let statusCodes: [Int] + /// Per-kind statuses and blob traffic, for strict warm-scenario verification. + let offeringsStatusCodes: [Int] + let configStatusCodes: [Int] + let blobRequestCount: Int init(totalMs: Double, events: [TransportEvent]) { self.totalMs = totalMs @@ -34,7 +36,9 @@ struct IterationMeasurement { self.fallbackHostRequestCount = events .filter { $0.host.contains("8-lives-cat") || $0.host.contains("rc-backup") } .count - self.statusCodes = events.map(\.statusCode) + self.offeringsStatusCodes = offerings.map(\.statusCode) + self.configStatusCodes = config.map(\.statusCode) + self.blobRequestCount = blobs.count } private static func span(of events: [TransportEvent]) -> Double? { @@ -47,32 +51,30 @@ struct IterationMeasurement { } -/// Aggregates measured iterations into one JSONL row. The first `warmupIterations` recorded -/// measurements are excluded from the statistics but counted in `warmup_discarded`, so one-time -/// process costs (lazy statics, first JSON decoder use) don't pollute the distribution. +/// Aggregates measured iterations into one JSONL row. Iterations with index below +/// `warmupIterations` are excluded from the statistics (by index, so a failed warmup iteration +/// never shifts a slow measured one into the discarded window), and errors are reported both in +/// total and for the post-warmup window: a run whose candidate "wins" by erroring out of its +/// slowest iterations must be visibly invalid, not quietly faster. struct BenchmarkMetrics { - private var measurements: [IterationMeasurement] = [] - private var errors: [String] = [] + private var measurementsByIteration: [Int: IterationMeasurement] = [:] + private var errorsByIteration: [Int: String] = [:] - mutating func record(_ measurement: IterationMeasurement) { - self.measurements.append(measurement) + mutating func record(_ measurement: IterationMeasurement, iteration: Int) { + self.measurementsByIteration[iteration] = measurement } - mutating func record(error: Error) { - self.errors.append("\(error)") - } - - var allStatusCodes: [Int] { - return self.measurements.flatMap(\.statusCodes) - } - - var errorCount: Int { - return self.errors.count + mutating func record(error: Error, iteration: Int) { + self.errorsByIteration[iteration] = "\(error)" } func jsonlRow(for command: BenchmarkCommand) -> String { - let measured = Array(self.measurements.dropFirst(command.warmupIterations)) + let measured = self.measurementsByIteration + .filter { $0.key >= command.warmupIterations } + .sorted { $0.key < $1.key } + .map(\.value) + let postWarmupErrors = self.errorsByIteration.filter { $0.key >= command.warmupIterations } let totals = measured.map(\.totalMs).sorted() var row: [String: Any] = [ @@ -85,8 +87,10 @@ struct BenchmarkMetrics { "workflows": command.workflowCount, "seed": command.seed, "iterations": command.iterations, - "warmup_discarded": min(command.warmupIterations, self.measurements.count), - "error_count": self.errors.count + "warmup_discarded": command.warmupIterations, + "measured_iterations": measured.count, + "error_count": self.errorsByIteration.count, + "post_warmup_error_count": postWarmupErrors.count ] if !totals.isEmpty { @@ -98,24 +102,12 @@ struct BenchmarkMetrics { } } - if !measured.isEmpty { - let count = Double(measured.count) - row["request_count_mean"] = Self.rounded(Double(measured.reduce(0) { $0 + $1.requestCount }) / count) - row["bytes_received_mean"] = Self.rounded(Double(measured.reduce(0) { $0 + $1.bytesReceived }) / count) - row["failed_requests_total"] = measured.reduce(0) { $0 + $1.failedRequestCount } - row["fallback_host_requests_total"] = measured.reduce(0) { $0 + $1.fallbackHostRequestCount } - - for (key, values) in [ - ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), - ("config_ms_mean", measured.compactMap(\.configMs)), - ("blob_ms_mean", measured.compactMap(\.blobMs)) - ] where !values.isEmpty { - row[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) - } + for (key, value) in Self.aggregates(of: measured) { + row[key] = value } - if !self.errors.isEmpty { - row["first_error"] = self.errors[0] + if let firstError = self.errorsByIteration.min(by: { $0.key < $1.key }) { + row["first_error"] = "iteration \(firstError.key): \(firstError.value)" } for (key, value) in command.annotations { @@ -130,6 +122,28 @@ struct BenchmarkMetrics { } } + private static func aggregates(of measured: [IterationMeasurement]) -> [String: Any] { + guard !measured.isEmpty else { return [:] } + + let count = Double(measured.count) + var aggregates: [String: Any] = [ + "request_count_mean": Self.rounded(Double(measured.reduce(0) { $0 + $1.requestCount }) / count), + "bytes_received_mean": Self.rounded(Double(measured.reduce(0) { $0 + $1.bytesReceived }) / count), + "failed_requests_total": measured.reduce(0) { $0 + $1.failedRequestCount }, + "fallback_host_requests_total": measured.reduce(0) { $0 + $1.fallbackHostRequestCount } + ] + + for (key, values) in [ + ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), + ("config_ms_mean", measured.compactMap(\.configMs)), + ("blob_ms_mean", measured.compactMap(\.blobMs)) + ] where !values.isEmpty { + aggregates[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) + } + + return aggregates + } + /// Nearest-rank percentile over an already-sorted array. static func percentile(_ percentile: Int, of sorted: [Double]) -> Double { precondition(!sorted.isEmpty) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift index 1425125eb8..708f4a327c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift @@ -140,8 +140,12 @@ private struct PayloadBuilder { ] } + // Only workflow blobs are prefetched: those are what offerings delivery awaits + // (`awaitTopicAndPrefetchBlobsReady(.workflows)`). Prefetching ui_config blobs too + // would leave downloads running past the measured completion, corrupting the + // per-iteration request/byte accounting; they stay fetchable on demand via blob_ref. // Sorted so the payload bytes are stable across processes (JSON arrays keep their order). - let prefetchBlobs = workflowRefsById.values.sorted() + [uiConfigAppRef, uiConfigLocalizationsRef] + let prefetchBlobs = workflowRefsById.values.sorted() return self.data([ "domain": "app", @@ -163,8 +167,8 @@ private struct PayloadBuilder { ], "workflows": workflowsTopic, "ui_config": [ - "app": ["blob_ref": uiConfigAppRef, "prefetch": true], - "localizations": ["blob_ref": uiConfigLocalizationsRef, "prefetch": true] + "app": ["blob_ref": uiConfigAppRef], + "localizations": ["blob_ref": uiConfigLocalizationsRef] ] ] ]) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index d1e6d319c0..d2ee040519 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -51,15 +51,21 @@ final class BenchmarkRunner { } for iteration in 0..= self.command.warmupIterations { + try self.validateScenario(measurement, iteration: iteration) + } + metrics.record(measurement, iteration: iteration) + } catch let error as BenchmarkError { + if case .scenarioViolation = error { throw error } + metrics.record(error: error, iteration: iteration) } catch { - metrics.record(error: error) + metrics.record(error: error, iteration: iteration) } } - try self.verifyScenario(metrics) - return metrics.jsonlRow(for: self.command) } @@ -91,6 +97,7 @@ private extension BenchmarkRunner { /// Uncounted launch that fills the disk caches the warm iterations relaunch against. func primeDiskState() throws { + SimulatedTransportURLProtocol.beginIteration(-1) let stack = BenchmarkSDKStack( mode: self.command.mode, apiKey: self.command.apiKey, @@ -148,16 +155,47 @@ private extension BenchmarkRunner { return Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000 } - /// Warm runs must prove they actually hit the revalidation paths. - func verifyScenario(_ metrics: BenchmarkMetrics) throws { + /// Warm runs must prove that EVERY measured iteration hit the revalidation paths; a single + /// iteration silently falling back to full 200 responses or re-downloading blobs would mix + /// cold behavior into the warm distribution. + func validateScenario(_ measurement: IterationMeasurement, iteration: Int) throws { guard self.command.scenario == .warm, self.command.lossPercent == 0 else { return } + try Self.validateWarmMeasurement(measurement, mode: self.command.mode, iteration: iteration) + } + +} + +extension BenchmarkRunner { + + static func validateWarmMeasurement( + _ measurement: IterationMeasurement, + mode: BenchmarkMode, + iteration: Int + ) throws { + guard !measurement.offeringsStatusCodes.isEmpty, + measurement.offeringsStatusCodes.allSatisfy({ $0 == 304 }) else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) did not revalidate offerings via 304 " + + "(statuses: \(measurement.offeringsStatusCodes))" + ) + } - let statuses = Set(metrics.allStatusCodes) - guard statuses.contains(304) else { - throw BenchmarkError.scenarioViolation("warm run never revalidated offerings via 304") + // Kill-switch mode pays the config 4xx on every launch by design, so only plain config + // mode must see manifest 204s. + if mode == .config { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ $0 == 204 }) else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) did not revalidate config via manifest 204 " + + "(statuses: \(measurement.configStatusCodes))" + ) + } } - if self.command.mode == .config, !statuses.contains(204) { - throw BenchmarkError.scenarioViolation("warm config run never revalidated via manifest 204") + + guard measurement.blobRequestCount == 0 else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) re-downloaded \(measurement.blobRequestCount) blob(s)" + ) } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift new file mode 100644 index 0000000000..135006815c --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -0,0 +1,79 @@ +import Foundation + +// MARK: - Passthrough recording (live runs) + +extension SimulatedTransportURLProtocol { + + /// Sessions the passthrough re-issues requests on. API traffic mirrors `HTTPClient`'s + /// single-connection-per-host pool; everything else (blob CDNs) gets a default pool like + /// the production blob downloader's `URLSession.shared`. Injectable for tests. + static var passthroughAPISession: URLSession = makePassthroughSession(maxConnectionsPerHost: 1) + static var passthroughBlobSession: URLSession = makePassthroughSession(maxConnectionsPerHost: nil) + + private static func makePassthroughSession(maxConnectionsPerHost: Int?) -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.urlCache = nil + if let maxConnectionsPerHost { + configuration.httpMaximumConnectionsPerHost = maxConnectionsPerHost + } + return URLSession(configuration: configuration) + } + + static func isAPIHost(_ host: String) -> Bool { + return host.contains("revenuecat.com") && host.hasPrefix("api.") + || host.contains("8-lives-cat") + || host.contains("rc-backup") + } + + func startPassthrough(url: URL, startedAt: DispatchTime) { + let host = url.host ?? "" + let session = Self.isAPIHost(host) ? Self.passthroughAPISession : Self.passthroughBlobSession + + let task = session.dataTask(with: self.request) { [weak self] data, response, error in + guard let self else { return } + let ended = DispatchTime.now() + + if let error { + Self.record(TransportEvent( + host: host, + path: url.path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: ended, + failed: true + )) + self.client?.urlProtocol(self, didFailWithError: error) + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(host)") + ) + return + } + + Self.record(TransportEvent( + host: host, + path: url.path, + statusCode: httpResponse.statusCode, + bytesReceived: data?.count ?? 0, + startedAt: startedAt, + endedAt: ended, + failed: false + )) + self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + if let data, !data.isEmpty { + self.client?.urlProtocol(self, didLoad: data) + } + self.client?.urlProtocolDidFinishLoading(self) + } + self.stateLock.withLock { + self.passthroughTask = task + } + task.resume() + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift new file mode 100644 index 0000000000..f9a9ae9362 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift @@ -0,0 +1,63 @@ +import Foundation + +// MARK: - Deterministic request plans + +extension SimulatedTransportURLProtocol { + + /// Identity of one simulated request. Plans derive from this key alone, so the same key + /// always produces the same plan, in any process, in any request order. + struct PlanKey { + + let seed: UInt64 + let iteration: Int + let url: URL + let attempt: Int + + /// FNV-1a over the request identity, mixed with the run seed. + var hashValue64: UInt64 { + var hash: UInt64 = 0xCBF29CE484222325 + func mix(_ bytes: [UInt8]) { + for byte in bytes { + hash ^= UInt64(byte) + hash = hash &* 0x100000001B3 + } + } + mix(Array(withUnsafeBytes(of: self.seed.littleEndian) { Data($0) })) + mix(Array(withUnsafeBytes(of: UInt64(bitPattern: Int64(self.iteration)).littleEndian) { Data($0) })) + mix(Array(self.url.absoluteString.utf8)) + mix(Array(withUnsafeBytes(of: UInt64(bitPattern: Int64(self.attempt)).littleEndian) { Data($0) })) + return hash + } + + } + + struct RequestPlan { + let rttMs: Double + let fails: Bool + let chunkDelaysMs: [Double] + } + + /// Computes the full network timeline of one request from its stable key. + static func requestPlan( + key: PlanKey, + bodyCount: Int, + profile: NetworkProfile, + loss: LossModel + ) -> RequestPlan { + var rng = SeededRandom(seed: key.hashValue64) + let host = key.url.host ?? "" + + let rttMs = profile.rttMs(forHost: host, rng: &rng) + let fails = loss.shouldFailRequest(rng: &rng) + var chunkDelaysMs: [Double] = [] + if !fails { + var offset = 0 + while offset < max(bodyCount, 1) { + chunkDelaysMs.append(loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &rng)) + offset += Self.chunkSize + } + } + return RequestPlan(rttMs: rttMs, fails: fails, chunkDelaysMs: chunkDelaysMs) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index 4b2b8897a6..18bb599e58 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -35,26 +35,42 @@ final class SimulatedTransportURLProtocol: URLProtocol { private static let lock = NSLock() private static var mode: Mode? - private static var rng = SeededRandom(seed: 0) + private static var seed: UInt64 = 0 + private static var iterationIndex: Int = 0 + /// Requests already planned this iteration, keyed by URL, so retries of the same URL get + /// distinct (but stable) plans. + private static var attemptCountsByURL: [String: Int] = [:] private static var events: [TransportEvent] = [] - private static let chunkSize = 16 * 1024 + static let chunkSize = 16 * 1024 /// Serial per request so header, chunks, and completion arrive in order; separate requests /// each get their own queue and overlap freely. private let deliveryQueue = DispatchQueue(label: "com.revenuecat.benchmark.transport.request") private var pendingWorkItems: [DispatchWorkItem] = [] - private var passthroughTask: URLSessionDataTask? - private let stateLock = NSLock() + var passthroughTask: URLSessionDataTask? + let stateLock = NSLock() static func install(server: FixtureServer, profile: NetworkProfile, loss: LossModel, seed: UInt64) { self.lock.withLock { self.mode = .simulated(server: server, profile: profile, loss: loss) - self.rng = SeededRandom(seed: seed) + self.seed = seed + self.iterationIndex = 0 + self.attemptCountsByURL = [:] self.events = [] } } + /// Marks the start of an iteration. Request plans are keyed by (seed, iteration, URL, + /// attempt), so samples stay stable across processes regardless of the order concurrent + /// requests happen to reach the transport, while still varying between iterations. + static func beginIteration(_ index: Int) { + self.lock.withLock { + self.iterationIndex = index + self.attemptCountsByURL = [:] + } + } + static func installPassthrough() { self.lock.withLock { self.mode = .passthrough @@ -148,22 +164,19 @@ private extension SimulatedTransportURLProtocol { let bodyData = Self.bodyData(of: self.request) let response = server.response(for: self.request, bodyData: bodyData) - // Sample every random decision up front, under one lock, so the request's timeline is - // fixed at start time regardless of delivery interleaving. - let plan: (rttMs: Double, fails: Bool, chunkDelaysMs: [Double]) = Self.lock.withLock { - let rttMs = profile.rttMs(forHost: host, rng: &Self.rng) - let fails = loss.shouldFailRequest(rng: &Self.rng) - var chunkDelaysMs: [Double] = [] - if !fails { - var offset = 0 - while offset < max(response.body.count, 1) { - chunkDelaysMs.append( - loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &Self.rng) - ) - offset += Self.chunkSize - } - } - return (rttMs, fails, chunkDelaysMs) + // The plan is derived from a stable per-request key, not drawn from a shared RNG in + // arrival order: concurrent requests reach the transport in a scheduler-dependent + // order, and order-dependent sampling would make same-seed runs disagree across + // processes. + let plan = Self.lock.withLock { () -> RequestPlan in + let attempt = Self.attemptCountsByURL[url.absoluteString, default: 0] + Self.attemptCountsByURL[url.absoluteString] = attempt + 1 + return Self.requestPlan( + key: PlanKey(seed: Self.seed, iteration: Self.iterationIndex, url: url, attempt: attempt), + bodyCount: response.body.count, + profile: profile, + loss: loss + ) } if plan.fails { @@ -181,83 +194,7 @@ private extension SimulatedTransportURLProtocol { } -// MARK: - Passthrough recording (live runs) - -extension SimulatedTransportURLProtocol { - - /// Sessions the passthrough re-issues requests on. API traffic mirrors `HTTPClient`'s - /// single-connection-per-host pool; everything else (blob CDNs) gets a default pool like - /// the production blob downloader's `URLSession.shared`. Injectable for tests. - static var passthroughAPISession: URLSession = makePassthroughSession(maxConnectionsPerHost: 1) - static var passthroughBlobSession: URLSession = makePassthroughSession(maxConnectionsPerHost: nil) - - private static func makePassthroughSession(maxConnectionsPerHost: Int?) -> URLSession { - let configuration = URLSessionConfiguration.ephemeral - configuration.urlCache = nil - if let maxConnectionsPerHost { - configuration.httpMaximumConnectionsPerHost = maxConnectionsPerHost - } - return URLSession(configuration: configuration) - } - - static func isAPIHost(_ host: String) -> Bool { - return host.contains("revenuecat.com") && host.hasPrefix("api.") - || host.contains("8-lives-cat") - || host.contains("rc-backup") - } - - private func startPassthrough(url: URL, startedAt: DispatchTime) { - let host = url.host ?? "" - let session = Self.isAPIHost(host) ? Self.passthroughAPISession : Self.passthroughBlobSession - - let task = session.dataTask(with: self.request) { [weak self] data, response, error in - guard let self else { return } - let ended = DispatchTime.now() - - if let error { - Self.record(TransportEvent( - host: host, - path: url.path, - statusCode: 0, - bytesReceived: 0, - startedAt: startedAt, - endedAt: ended, - failed: true - )) - self.client?.urlProtocol(self, didFailWithError: error) - return - } - - guard let httpResponse = response as? HTTPURLResponse else { - self.client?.urlProtocol( - self, - didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(host)") - ) - return - } - - Self.record(TransportEvent( - host: host, - path: url.path, - statusCode: httpResponse.statusCode, - bytesReceived: data?.count ?? 0, - startedAt: startedAt, - endedAt: ended, - failed: false - )) - self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) - if let data, !data.isEmpty { - self.client?.urlProtocol(self, didLoad: data) - } - self.client?.urlProtocolDidFinishLoading(self) - } - self.stateLock.withLock { - self.passthroughTask = task - } - task.resume() - } - -} +// MARK: - Delivery helpers private extension SimulatedTransportURLProtocol { @@ -345,6 +282,10 @@ private extension SimulatedTransportURLProtocol { self.deliveryQueue.asyncAfter(deadline: .now() + delayMs / 1_000, execute: item) } +} + +extension SimulatedTransportURLProtocol { + static func record(_ event: TransportEvent) { self.lock.withLock { self.events.append(event) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift index 491b303fb9..7176718841 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -24,22 +24,63 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { return try XCTUnwrap(object as? [String: Any]) } - func testJSONLRowDiscardsWarmupIterations() throws { + func testJSONLRowDiscardsWarmupIterationsByIndex() throws { var command = BenchmarkCommand() command.iterations = 5 command.warmupIterations = 2 var metrics = BenchmarkMetrics() // Two slow warmup iterations that must not pollute the stats. - [1_000.0, 900.0, 10.0, 20.0, 30.0].forEach { metrics.record(self.measurement(totalMs: $0)) } + for (index, totalMs) in [1_000.0, 900.0, 10.0, 20.0, 30.0].enumerated() { + metrics.record(self.measurement(totalMs: totalMs), iteration: index) + } let row = try self.decodeRow(metrics, command: command) XCTAssertEqual(row["warmup_discarded"] as? Int, 2) + XCTAssertEqual(row["measured_iterations"] as? Int, 3) XCTAssertEqual(row["mean_ms"] as? Double, 20) XCTAssertEqual(row["max_ms"] as? Double, 30) } + func testFailedWarmupIterationDoesNotShiftDiscardWindow() throws { + var command = BenchmarkCommand() + command.iterations = 4 + command.warmupIterations = 2 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 1_000), iteration: 0) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 1) + metrics.record(self.measurement(totalMs: 10), iteration: 2) + metrics.record(self.measurement(totalMs: 30), iteration: 3) + + let row = try self.decodeRow(metrics, command: command) + + // A failed warmup iteration must not shift a measured iteration into the discard + // window: iterations 2 and 3 are measured, the warmup error is visible but separate. + XCTAssertEqual(row["measured_iterations"] as? Int, 2) + XCTAssertEqual(row["mean_ms"] as? Double, 20) + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + } + + func testPostWarmupErrorsAreCalledOut() throws { + var command = BenchmarkCommand() + command.iterations = 3 + command.warmupIterations = 1 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10), iteration: 0) + metrics.record(self.measurement(totalMs: 20), iteration: 1) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 2) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 1) + let firstError = try XCTUnwrap(row["first_error"] as? String) + XCTAssertTrue(firstError.hasPrefix("iteration 2:")) + } + func testJSONLRowCarriesConfigurationAndAnnotations() throws { var command = BenchmarkCommand() command.mode = .configKillswitch @@ -50,8 +91,8 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { command.warmupIterations = 0 var metrics = BenchmarkMetrics() - metrics.record(self.measurement(totalMs: 42)) - metrics.record(error: BenchmarkError.timeout("offerings fetch")) + metrics.record(self.measurement(totalMs: 42), iteration: 0) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 1) let row = try self.decodeRow(metrics, command: command) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift index adddede45a..dc0ec5aac2 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift @@ -46,10 +46,18 @@ final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) - let referencedRefs = Set(configuration.prefetchBlobs) - XCTAssertEqual(referencedRefs.count, 7 + 2) + // Prefetch covers exactly the workflow blobs (what offerings delivery awaits); + // ui_config blobs are referenced by their topic but intentionally not prefetched. + let prefetchRefs = Set(configuration.prefetchBlobs) + XCTAssertEqual(prefetchRefs.count, 7) - for ref in referencedRefs { + let topicRefs = configuration.topics.entries.values.flatMap { topic in + topic.values.compactMap(\.blobRef) + } + XCTAssertEqual(Set(topicRefs).count, 7 + 2) + XCTAssertTrue(prefetchRefs.isSubset(of: Set(topicRefs))) + + for ref in Set(topicRefs) { let blob = try XCTUnwrap(self.factory.blobData(forRef: ref), "missing blob for ref \(ref)") XCTAssertEqual(RCContainerEncoder.blobRef(for: blob), ref, "blob refs must be content-addressed") } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift new file mode 100644 index 0000000000..dbf240db29 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -0,0 +1,86 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkRunnerValidationTests: BenchmarkTestCase { + + private func measurement( + offeringsStatuses: [Int], + configStatuses: [Int] = [], + blobPaths: [String] = [] + ) -> IterationMeasurement { + let start = DispatchTime.now() + var events: [TransportEvent] = [] + for status in offeringsStatuses { + events.append(TransportEvent(host: "api.revenuecat.com", path: "/v1/subscribers/u/offerings", + statusCode: status, bytesReceived: 0, + startedAt: start, endedAt: start, failed: false)) + } + for status in configStatuses { + events.append(TransportEvent(host: "api.revenuecat.com", path: "/v1/config/app", + statusCode: status, bytesReceived: 0, + startedAt: start, endedAt: start, failed: false)) + } + for path in blobPaths { + events.append(TransportEvent(host: "cdn.revenuecat.local", path: path, + statusCode: 200, bytesReceived: 100, + startedAt: start, endedAt: start, failed: false)) + } + return IterationMeasurement(totalMs: 1, events: events) + } + + func testWarmLegacyIterationWithOnly304Passes() throws { + try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304]), + mode: .legacy, + iteration: 3 + ) + } + + func testWarmIterationWithFull200OfferingsFails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [200]), + mode: .legacy, + iteration: 3 + )) + } + + func testWarmIterationMixing304And200Fails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304, 200]), + mode: .legacy, + iteration: 3 + )) + } + + func testWarmConfigIterationRequires204() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [200]), + mode: .config, + iteration: 3 + )) + + XCTAssertNoThrow(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [204]), + mode: .config, + iteration: 3 + )) + } + + func testWarmKillSwitchIterationAllowsConfig4xx() throws { + try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [400]), + mode: .configKillswitch, + iteration: 3 + ) + } + + func testWarmIterationReDownloadingBlobsFails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [204], blobPaths: ["/blobs/aaa"]), + mode: .config, + iteration: 3 + )) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift index b3f76fdabf..332c818998 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift @@ -70,9 +70,9 @@ final class BenchmarkSDKStackTests: BenchmarkTestCase { XCTAssertTrue(paths.contains { $0.hasSuffix("/offerings") }, "expected an offerings fetch in \(paths)") // The workflows topic marks every workflow blob prefetch, so offerings delivery waits - // for all of them plus the two ui_config blobs. + // for all of them. ui_config blobs are not prefetched and must not be fetched here. let blobRequests = paths.filter { $0.contains("/blobs/") } - XCTAssertEqual(Set(blobRequests).count, 4 + 2) + XCTAssertEqual(Set(blobRequests).count, 4) } func testKillSwitchModeStillDeliversOfferings() throws { diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift index d07d496af4..d1b9f52845 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift @@ -1,7 +1,31 @@ import XCTest +@testable import SDKConfigBenchmarkCore + /// Minimal base class for benchmark harness tests. The benchmark test target does not link /// the main UnitTests infrastructure, so it carries its own `XCTestCase` subclass to satisfy /// the repo convention that test classes never inherit `XCTestCase` directly. +/// +/// Every test gets a fresh disk-cache root so tests never touch the real user Library and +/// never see each other's (or a previous run's) cached state. // swiftlint:disable:next xctestcase_superclass -class BenchmarkTestCase: XCTestCase {} +class BenchmarkTestCase: XCTestCase { + + private var diskRoot: URL! + + override func setUp() { + super.setUp() + self.diskRoot = FileManager.default.temporaryDirectory + .appendingPathComponent("SDKConfigBenchmarkTests-\(UUID().uuidString)", isDirectory: true) + DirectoryHelper.benchmarkBaseDirectoryOverride = self.diskRoot + } + + override func tearDown() { + DirectoryHelper.benchmarkBaseDirectoryOverride = nil + if let diskRoot = self.diskRoot { + try? FileManager.default.removeItem(at: diskRoot) + } + super.tearDown() + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift index 4eb29b102c..0904235dde 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift @@ -78,6 +78,33 @@ final class NetworkProfileTests: BenchmarkTestCase { XCTAssertEqual(rate, 0.027, accuracy: 0.008) } + func testRequestPlansAreStableRegardlessOfComputationOrder() throws { + let url = try XCTUnwrap(URL(string: "https://cdn.revenuecat.local/blobs/abc")) + let otherURL = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/config/app")) + let loss = LossModel(lossPercent: 20) + + func plan(_ target: URL, iteration: Int = 1, attempt: Int = 0) -> SimulatedTransportURLProtocol.RequestPlan { + return SimulatedTransportURLProtocol.requestPlan( + key: .init(seed: 42, iteration: iteration, url: target, attempt: attempt), + bodyCount: 100_000, profile: .lte, loss: loss + ) + } + + // Same key produces the identical plan no matter what was computed before it: request + // arrival order (which is scheduler-dependent) must not influence sampling. + let before = plan(url) + _ = plan(otherURL) + _ = plan(otherURL, attempt: 1) + let after = plan(url) + XCTAssertEqual(before.rttMs, after.rttMs) + XCTAssertEqual(before.fails, after.fails) + XCTAssertEqual(before.chunkDelaysMs, after.chunkDelaysMs) + + // Different iterations and attempts get different samples, so distributions stay real. + XCTAssertNotEqual(plan(url, iteration: 2).rttMs, before.rttMs) + XCTAssertNotEqual(plan(url, attempt: 1).rttMs, before.rttMs) + } + func testLossChunkDelaysAreDeterministicForSameSeed() { let loss = LossModel(lossPercent: 20) var first = SeededRandom(seed: 5) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index 3c047ad3c5..5f7db7fab5 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -57,14 +57,23 @@ def print_single(rows): print("| " + " | ".join(cells) + " |") +def errors_cell(row): + """Rows with post-warmup errors are not valid comparison input; make that loud.""" + errors = row.get("post_warmup_error_count", row.get("error_count")) + if errors is None: + return "-" + return f"⚠️ {errors}" if errors else "0" + + def print_comparison(baseline, candidate): header = list(KEY_FIELDS) for metric in ("p50_ms", "p95_ms"): header += [f"{metric} base", f"{metric} cand", "Δ"] - header += ["req base", "req cand", "bytes base", "bytes cand"] + header += ["req base", "req cand", "bytes base", "bytes cand", "err base", "err cand"] print("| " + " | ".join(header) + " |") print("|" + "---|" * len(header)) + invalid = 0 for key in sorted(set(baseline) | set(candidate), key=str): base, cand = baseline.get(key, {}), candidate.get(key, {}) cells = [str(part) for part in key] @@ -79,9 +88,19 @@ def print_comparison(baseline, candidate): fmt(cand.get("request_count_mean")), fmt(base.get("bytes_received_mean")), fmt(cand.get("bytes_received_mean")), + errors_cell(base), + errors_cell(cand), ] + if any(row.get("post_warmup_error_count", row.get("error_count", 0)) for row in (base, cand) if row): + invalid += 1 print("| " + " | ".join(cells) + " |") + if invalid: + print( + f"\n**⚠️ {invalid} row(s) have post-warmup errors; " + "their timing deltas are not valid comparison input.**" + ) + def main(argv): if len(argv) == 2: diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index 305d9035f4..267de5fe7b 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -60,13 +60,11 @@ if [[ ! -x "$BINARY" ]]; then fi SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" -BENCH_HOME="$(mktemp -d)" -trap 'rm -rf "$BENCH_HOME"' EXIT run_row() { local mode="$1" scenario="$2" profile="$3" loss="$4" echo "Running transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 - HOME="$BENCH_HOME" "$BINARY" \ + "$BINARY" \ --transport "$TRANSPORT" \ --mode "$mode" \ --scenario "$scenario" \ From cfc5fb87ea44799b65199239ebc20e318a2c8f14 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 19:49:06 +0200 Subject: [PATCH 10/34] other(benchmarks): simplify pass: shared request classifier and sdk helper reuse One RequestKind classifier stamped on every TransportEvent now drives fixture routing, RTT class, live connection pooling, phase attribution, and warm validation, so those rules can no longer drift apart. Reuses the SDK's RemoteConfigBlobRefHelpers, littleEndianData, and ETag header constants; precomputes fixture responses; computes request plans outside the global lock with no-copy body slices; drops dead code and dedups test scaffolding. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkCommand.swift | 13 +- .../Sources/BenchmarkError.swift | 2 - .../Sources/BenchmarkMetrics.swift | 17 +-- .../Sources/BenchmarkPayloadFactory.swift | 9 -- .../Sources/BenchmarkRunner.swift | 8 +- .../Sources/FixtureServer.swift | 140 +++++++----------- .../Sources/NetworkProfile.swift | 14 +- .../Sources/RCContainerEncoder.swift | 23 +-- ...atedTransportURLProtocol+Passthrough.swift | 39 ++--- .../SimulatedTransportURLProtocol+Plans.swift | 30 ++-- .../SimulatedTransportURLProtocol.swift | 100 +++++++++---- .../Tests/BenchmarkMetricsTests.swift | 20 +-- .../Tests/BenchmarkPayloadFactoryTests.swift | 18 +-- .../BenchmarkRunnerValidationTests.swift | 30 ++-- .../Tests/FixtureServerTests.swift | 84 ++++++----- .../Tests/NetworkProfileTests.swift | 22 ++- .../Tests/PassthroughTransportTests.swift | 8 - .../Benchmarks/SDKConfigBenchmark/compare.py | 8 +- 18 files changed, 293 insertions(+), 292 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 3167aa9a46..bd363bddb6 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -14,6 +14,17 @@ enum BenchmarkMode: String, CaseIterable { } } + /// Whether the fixture forces `/v1/config` to fail with a 4xx (the kill switch). + var forcesConfigFailure: Bool { + return self == .configKillswitch + } + + /// Whether warm iterations must revalidate config via manifest 204s. Kill-switch mode + /// pays a 4xx on every launch instead, by design. + var expectsWarmConfigRevalidation: Bool { + return self == .config + } + } enum BenchmarkTransport: String, CaseIterable { @@ -175,7 +186,7 @@ struct BenchmarkCommand { "--profile requires --transport simulated; live runs use the real network" ) } - guard self.mode != .configKillswitch else { + guard !self.mode.forcesConfigFailure else { throw BenchmarkError.invalidArgument( "config-killswitch cannot force a 4xx on the real backend; use --transport simulated" ) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift index 94d0823d75..5fbf6b149c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift @@ -4,7 +4,6 @@ enum BenchmarkError: Error, CustomStringConvertible { case invalidArgument(String) case invalidFixture(String) - case unsupportedPath(String) case timeout(String) case backendFailure(String) case scenarioViolation(String) @@ -13,7 +12,6 @@ enum BenchmarkError: Error, CustomStringConvertible { switch self { case let .invalidArgument(message): return "Invalid argument: \(message)" case let .invalidFixture(message): return "Invalid fixture: \(message)" - case let .unsupportedPath(path): return "Unsupported fixture path: \(path)" case let .timeout(operation): return "Timed out waiting for \(operation)" case let .backendFailure(message): return "Backend failure: \(message)" case let .scenarioViolation(message): return "Scenario violation: \(message)" diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index fe4ef7857a..8b8a6d260e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -22,20 +22,17 @@ struct IterationMeasurement { init(totalMs: Double, events: [TransportEvent]) { self.totalMs = totalMs - let offerings = events.filter { $0.path.hasSuffix("/offerings") } - let config = events.filter { $0.path.hasSuffix("/config/app") } - // Blob URLs come from the (real or fixture) config's url_format, so classify them as - // everything that is neither an offerings nor a config request. - let blobs = events.filter { !$0.path.hasSuffix("/offerings") && !$0.path.hasSuffix("/config/app") } + let byKind = Dictionary(grouping: events, by: \.kind) + let offerings = byKind[.offerings] ?? [] + let config = byKind[.config] ?? [] + let blobs = byKind[.blob] ?? [] self.offeringsMs = Self.span(of: offerings) self.configMs = Self.span(of: config) self.blobMs = Self.span(of: blobs) self.requestCount = events.count self.bytesReceived = events.reduce(0) { $0 + $1.bytesReceived } self.failedRequestCount = events.filter(\.failed).count - self.fallbackHostRequestCount = events - .filter { $0.host.contains("8-lives-cat") || $0.host.contains("rc-backup") } - .count + self.fallbackHostRequestCount = events.filter(\.isFallbackHostRequest).count self.offeringsStatusCodes = offerings.map(\.statusCode) self.configStatusCodes = config.map(\.statusCode) self.blobRequestCount = blobs.count @@ -102,9 +99,7 @@ struct BenchmarkMetrics { } } - for (key, value) in Self.aggregates(of: measured) { - row[key] = value - } + row += Self.aggregates(of: measured) if let firstError = self.errorsByIteration.min(by: { $0.key < $1.key }) { row["first_error"] = "iteration \(firstError.key): \(firstError.value)" diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift index 708f4a327c..8805ad8384 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift @@ -8,8 +8,6 @@ struct BenchmarkPayloadFactory { static let blobURLFormat = "https://cdn.revenuecat.local/blobs/{blob_ref}" - let paywallCount: Int - let workflowCount: Int let configManifest = "benchmark-manifest-v1" let offeringsData: Data @@ -18,9 +16,6 @@ struct BenchmarkPayloadFactory { private let blobsByRef: [String: Data] init(paywallCount: Int, workflowCount: Int) { - self.paywallCount = paywallCount - self.workflowCount = workflowCount - let builder = PayloadBuilder(paywallCount: paywallCount, workflowCount: workflowCount) self.offeringsData = builder.offeringsData() @@ -59,10 +54,6 @@ struct BenchmarkPayloadFactory { return Array(self.blobsByRef.keys) } - var productIdentifiers: Set { - return Set((0.. Response { - guard let url = request.url else { - return Self.notFound() - } - let path = url.path - - if path.hasSuffix("/offerings"), path.contains("/subscribers/") || path.hasSuffix("/v1/offerings") { - return self.offeringsResponse(for: request) - } - - if path.hasSuffix("/config/app") { - return self.configResponse(bodyData: bodyData) - } - - if let blobRange = path.range(of: "/blobs/") { - let ref = String(path[blobRange.upperBound...]) - return self.blobResponse(forRef: ref) - } - - return Self.notFound() - } - -} - -private extension FixtureServer { - - func offeringsResponse(for request: URLRequest) -> Response { - let requestETag = request.value(forHTTPHeaderField: "X-RevenueCat-ETag") - if requestETag == Self.offeringsETag { - return Response( - statusCode: 304, - headers: self.jsonHeaders(eTag: Self.offeringsETag), - body: Data() - ) - } - return Response( - statusCode: 200, - headers: self.jsonHeaders(eTag: Self.offeringsETag), - body: self.factory.offeringsData - ) - } - - func configResponse(bodyData: Data?) -> Response { - if self.killSwitchConfig { - let errorBody = Data(#"{"code": 7000, "message": "benchmark kill switch"}"#.utf8) - return Response(statusCode: 400, headers: self.jsonHeaders(eTag: nil), body: errorBody) - } - if let bodyData, - let body = try? JSONSerialization.jsonObject(with: bodyData) as? [String: Any], - body["manifest"] as? String == self.factory.configManifest { - return Response(statusCode: 204, headers: [:], body: Data()) - } - - return Response( + let jsonHeadersWithETag = [ + "Content-Type": "application/json", + HTTPClient.ResponseHeader.eTag.rawValue: Self.offeringsETag + ] + self.offeringsResponse = Response(statusCode: 200, headers: jsonHeadersWithETag, body: factory.offeringsData) + self.offeringsNotModifiedResponse = Response(statusCode: 304, headers: jsonHeadersWithETag, body: Data()) + self.configResponse = Response( statusCode: 200, headers: ["Content-Type": "application/octet-stream"], - body: self.factory.configContainerData + body: factory.configContainerData ) - } - - func blobResponse(forRef ref: String) -> Response { - guard let blob = self.factory.blobData(forRef: ref) else { - return Self.notFound() - } - return Response( - statusCode: 200, + self.killSwitchResponse = Response( + statusCode: 400, headers: ["Content-Type": "application/json"], - body: blob + body: Data(#"{"code": 7000, "message": "benchmark kill switch"}"#.utf8) ) - } - - func jsonHeaders(eTag: String?) -> [String: String] { - var headers = ["Content-Type": "application/json"] - if let eTag { - headers["X-RevenueCat-ETag"] = eTag - } - return headers - } - - static func notFound() -> Response { - return Response( + self.notFoundResponse = Response( statusCode: 404, headers: ["Content-Type": "application/json"], body: Data(#"{"code": 7259, "message": "benchmark fixture not found"}"#.utf8) ) } + func response(for request: URLRequest, bodyData: Data?) -> Response { + guard let url = request.url else { + return self.notFoundResponse + } + + switch RequestKind(url: url) { + case .offerings: + let requestETag = request.value(forHTTPHeaderField: HTTPClient.RequestHeader.eTag.rawValue) + return requestETag == Self.offeringsETag ? self.offeringsNotModifiedResponse : self.offeringsResponse + + case .config: + if self.killSwitchConfig { + return self.killSwitchResponse + } + if let bodyData, + let body = try? JSONSerialization.jsonObject(with: bodyData) as? [String: Any], + body["manifest"] as? String == self.factory.configManifest { + return self.configNotModifiedResponse + } + return self.configResponse + + case .blob: + guard let blobRange = url.path.range(of: "/blobs/"), + let blob = self.factory.blobData(forRef: String(url.path[blobRange.upperBound...])) else { + return self.notFoundResponse + } + return Response(statusCode: 200, headers: ["Content-Type": "application/json"], body: blob) + } + } + } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift index 4ff90d320b..79330a4382 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift @@ -49,8 +49,14 @@ struct NetworkProfile { } } - func rttMs(forHost host: String, rng: inout SeededRandom) -> Double { - let range = Self.isCDNHost(host) ? self.cdnRTTMs : self.apiRTTMs + /// RTT class follows the request kind, the same classification everything else uses: + /// offerings and config are dynamic API calls, blobs are CDN downloads. + func rttMs(for kind: RequestKind, rng: inout SeededRandom) -> Double { + let range: ClosedRange + switch kind { + case .offerings, .config: range = self.apiRTTMs + case .blob: range = self.cdnRTTMs + } guard range.lowerBound < range.upperBound else { return range.lowerBound } return Double.random(in: range, using: &rng) } @@ -61,10 +67,6 @@ struct NetworkProfile { return Double(byteCount) / bandwidth * 1_000 } - static func isCDNHost(_ host: String) -> Bool { - return host.contains("cdn") - } - } /// Approximates packet loss for the simulated transport. diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift index 614f1e7c9d..8e403ff6de 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift @@ -24,14 +24,10 @@ enum RCContainerEncoder { return data } - /// The externally-referenced blob ref for a payload: SHA-256 truncated to 24 bytes, - /// URL-safe base64 without padding (32 characters). + /// The externally-referenced blob ref for a payload, computed by the same helper the SDK + /// uses to validate refs, so fixtures can never drift from the production format. static func blobRef(for data: Data) -> String { - return self.checksum(for: data) - .base64EncodedString() - .replacingOccurrences(of: "+", with: "-") - .replacingOccurrences(of: "/", with: "_") - .replacingOccurrences(of: "=", with: "") + return data.withUnsafeBytes { RemoteConfigBlobRefHelpers.ref(for: $0) } } } @@ -44,7 +40,7 @@ private extension RCContainerEncoder { static func appendElement(_ payload: Data, to data: inout Data) { data.append(self.checksum(for: payload)) - data.appendLittleEndianUInt32(UInt32(payload.count)) + data.append(UInt32(payload.count).littleEndianData) data.append(0) // ContentEncoding.none data.append(contentsOf: [0, 0, 0]) data.append(payload) @@ -56,14 +52,3 @@ private extension RCContainerEncoder { } } - -private extension Data { - - mutating func appendLittleEndianUInt32(_ value: UInt32) { - self.append(UInt8(value & 0xff)) - self.append(UInt8((value >> 8) & 0xff)) - self.append(UInt8((value >> 16) & 0xff)) - self.append(UInt8((value >> 24) & 0xff)) - } - -} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift index 135006815c..c4f5cfe869 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -19,50 +19,35 @@ extension SimulatedTransportURLProtocol { return URLSession(configuration: configuration) } - static func isAPIHost(_ host: String) -> Bool { - return host.contains("revenuecat.com") && host.hasPrefix("api.") - || host.contains("8-lives-cat") - || host.contains("rc-backup") - } - func startPassthrough(url: URL, startedAt: DispatchTime) { - let host = url.host ?? "" - let session = Self.isAPIHost(host) ? Self.passthroughAPISession : Self.passthroughBlobSession + let session: URLSession + switch RequestKind(url: url) { + case .offerings, .config: session = Self.passthroughAPISession + case .blob: session = Self.passthroughBlobSession + } let task = session.dataTask(with: self.request) { [weak self] data, response, error in guard let self else { return } - let ended = DispatchTime.now() - if let error { - Self.record(TransportEvent( - host: host, - path: url.path, - statusCode: 0, - bytesReceived: 0, - startedAt: startedAt, - endedAt: ended, - failed: true - )) - self.client?.urlProtocol(self, didFailWithError: error) + if error != nil { + Self.record(.failure(url: url, startedAt: startedAt)) + self.client?.urlProtocol(self, didFailWithError: error ?? URLError(.unknown)) return } guard let httpResponse = response as? HTTPURLResponse else { self.client?.urlProtocol( self, - didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(host)") + didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(url.host ?? "")") ) return } - Self.record(TransportEvent( - host: host, - path: url.path, + Self.record(.success( + url: url, statusCode: httpResponse.statusCode, bytesReceived: data?.count ?? 0, - startedAt: startedAt, - endedAt: ended, - failed: false + startedAt: startedAt )) self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) if let data, !data.isEmpty { diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift index f9a9ae9362..50acaba199 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift @@ -16,16 +16,21 @@ extension SimulatedTransportURLProtocol { /// FNV-1a over the request identity, mixed with the run seed. var hashValue64: UInt64 { var hash: UInt64 = 0xCBF29CE484222325 - func mix(_ bytes: [UInt8]) { - for byte in bytes { - hash ^= UInt64(byte) - hash = hash &* 0x100000001B3 + func mix(_ byte: UInt8) { + hash ^= UInt64(byte) + hash = hash &* 0x100000001B3 + } + func mix(_ value: UInt64) { + for shift in stride(from: 0, to: 64, by: 8) { + mix(UInt8(truncatingIfNeeded: value >> shift)) } } - mix(Array(withUnsafeBytes(of: self.seed.littleEndian) { Data($0) })) - mix(Array(withUnsafeBytes(of: UInt64(bitPattern: Int64(self.iteration)).littleEndian) { Data($0) })) - mix(Array(self.url.absoluteString.utf8)) - mix(Array(withUnsafeBytes(of: UInt64(bitPattern: Int64(self.attempt)).littleEndian) { Data($0) })) + mix(self.seed) + mix(UInt64(bitPattern: Int64(self.iteration))) + for byte in self.url.absoluteString.utf8 { + mix(byte) + } + mix(UInt64(bitPattern: Int64(self.attempt))) return hash } @@ -45,14 +50,15 @@ extension SimulatedTransportURLProtocol { loss: LossModel ) -> RequestPlan { var rng = SeededRandom(seed: key.hashValue64) - let host = key.url.host ?? "" - let rttMs = profile.rttMs(forHost: host, rng: &rng) + let rttMs = profile.rttMs(for: RequestKind(url: key.url), rng: &rng) let fails = loss.shouldFailRequest(rng: &rng) var chunkDelaysMs: [Double] = [] - if !fails { + // Loss-free plans skip the per-chunk delay array entirely (delivery treats an empty + // array as zero delay everywhere). + if !fails && loss.lossPercent > 0 { var offset = 0 - while offset < max(bodyCount, 1) { + while offset < bodyCount { chunkDelaysMs.append(loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &rng)) offset += Self.chunkSize } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index 18bb599e58..338230e2a8 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -1,8 +1,33 @@ import Foundation +/// What a benchmark request is for. Classified once, from the URL, and stamped on every +/// `TransportEvent`, so routing, RTT class, connection pooling, phase attribution, and warm +/// validation all share one rule instead of re-deriving it from path strings. +enum RequestKind { + + case offerings + case config + /// Anything that is neither offerings nor config: blob downloads, whose URLs come from the + /// (real or fixture) config's `url_format` and so have no fixed shape. + case blob + + init(url: URL) { + let path = url.path + if path.hasSuffix("/offerings") { + self = .offerings + } else if path.hasSuffix("/config/app") { + self = .config + } else { + self = .blob + } + } + +} + /// One completed (or failed) simulated request, for phase attribution and byte accounting. struct TransportEvent { + let kind: RequestKind let host: String let path: String let statusCode: Int @@ -11,6 +36,37 @@ struct TransportEvent { let endedAt: DispatchTime let failed: Bool + /// The SDK's backup hosts; requests here mean the primary host failed over. + var isFallbackHostRequest: Bool { + return self.host.contains("8-lives-cat") || self.host.contains("rc-backup") + } + + static func failure(url: URL, startedAt: DispatchTime) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + host: url.host ?? "", + path: url.path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: true + ) + } + + static func success(url: URL, statusCode: Int, bytesReceived: Int, startedAt: DispatchTime) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + host: url.host ?? "", + path: url.path, + statusCode: statusCode, + bytesReceived: bytesReceived, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: false + ) + } + } /// The transport used by every URLSession in the benchmark. Two modes: @@ -160,27 +216,22 @@ private extension SimulatedTransportURLProtocol { loss: LossModel, startedAt: DispatchTime ) { - let host = url.host ?? "" let bodyData = Self.bodyData(of: self.request) let response = server.response(for: self.request, bodyData: bodyData) // The plan is derived from a stable per-request key, not drawn from a shared RNG in // arrival order: concurrent requests reach the transport in a scheduler-dependent // order, and order-dependent sampling would make same-seed runs disagree across - // processes. - let plan = Self.lock.withLock { () -> RequestPlan in + // processes. Only the attempt counter needs the lock; the plan itself is pure. + let key = Self.lock.withLock { () -> PlanKey in let attempt = Self.attemptCountsByURL[url.absoluteString, default: 0] Self.attemptCountsByURL[url.absoluteString] = attempt + 1 - return Self.requestPlan( - key: PlanKey(seed: Self.seed, iteration: Self.iterationIndex, url: url, attempt: attempt), - bodyCount: response.body.count, - profile: profile, - loss: loss - ) + return PlanKey(seed: Self.seed, iteration: Self.iterationIndex, url: url, attempt: attempt) } + let plan = Self.requestPlan(key: key, bodyCount: response.body.count, profile: profile, loss: loss) if plan.fails { - self.deliverFailure(host: host, path: url.path, rttMs: plan.rttMs, startedAt: startedAt) + self.deliverFailure(url: url, rttMs: plan.rttMs, startedAt: startedAt) } else { self.deliverResponse( response, @@ -200,19 +251,11 @@ private extension SimulatedTransportURLProtocol { /// One retransmission-timeout's worth of waiting before the failure surfaces, so failed /// attempts cost time like they do on a real link. - func deliverFailure(host: String, path: String, rttMs: Double, startedAt: DispatchTime) { + func deliverFailure(url: URL, rttMs: Double, startedAt: DispatchTime) { let rtoMs = max(1_000, rttMs * 2) self.schedule(afterMs: rtoMs) { [weak self] in guard let self else { return } - Self.record(TransportEvent( - host: host, - path: path, - statusCode: 0, - bytesReceived: 0, - startedAt: startedAt, - endedAt: DispatchTime.now(), - failed: true - )) + Self.record(.failure(url: url, startedAt: startedAt)) self.client?.urlProtocol(self, didFailWithError: URLError(.timedOut)) } } @@ -224,7 +267,6 @@ private extension SimulatedTransportURLProtocol { plan: (rttMs: Double, chunkDelaysMs: [Double]), startedAt: DispatchTime ) { - let host = url.host ?? "" guard let httpResponse = HTTPURLResponse( url: url, statusCode: response.statusCode, @@ -248,9 +290,12 @@ private extension SimulatedTransportURLProtocol { var chunkIndex = 0 while offset < response.body.count { let end = min(offset + Self.chunkSize, response.body.count) - let chunk = response.body.subdata(in: offset.. DispatchTime { return DispatchTime(uptimeNanoseconds: start.uptimeNanoseconds + offsetMs * 1_000_000) @@ -118,18 +118,20 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { until endMs: UInt64, status: Int = 200, failed: Bool = false - ) -> TransportEvent { - return TransportEvent(host: host, path: path, statusCode: status, bytesReceived: 100, + ) throws -> TransportEvent { + let url = try XCTUnwrap(URL(string: "https://\(host)\(path)")) + return TransportEvent(kind: RequestKind(url: url), host: host, path: path, + statusCode: status, bytesReceived: 100, startedAt: time(startMs), endedAt: time(endMs), failed: failed) } let measurement = IterationMeasurement(totalMs: 100, events: [ - event(path: "/v1/config/app", from: 0, until: 10), - event(path: "/blobs/aaa", host: "cdn.revenuecat.local", from: 10, until: 20), - event(path: "/blobs/bbb", host: "cdn.revenuecat.local", from: 12, until: 30), - event(path: "/v1/subscribers/u/offerings", from: 5, until: 45), - event(path: "/v1/offerings", host: "api-production.8-lives-cat.io", - from: 46, until: 50, status: 0, failed: true) + try event(path: "/v1/config/app", from: 0, until: 10), + try event(path: "/blobs/aaa", host: "cdn.revenuecat.local", from: 10, until: 20), + try event(path: "/blobs/bbb", host: "cdn.revenuecat.local", from: 12, until: 30), + try event(path: "/v1/subscribers/u/offerings", from: 5, until: 45), + try event(path: "/v1/offerings", host: "api-production.8-lives-cat.io", + from: 46, until: 50, status: 0, failed: true) ]) XCTAssertEqual(measurement.requestCount, 5) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift index dc0ec5aac2..2a7d904083 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift @@ -6,6 +6,12 @@ final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { private let factory = BenchmarkPayloadFactory(paywallCount: 5, workflowCount: 7) + private func decodedConfiguration() throws -> RemoteConfiguration { + let container = try RemoteConfigContainer(data: self.factory.configContainerData) + let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } + return try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + } + func testOfferingsDataDecodesIntoSDKModel() throws { let response = try JSONDecoder.default.decode(OfferingsResponse.self, from: self.factory.offeringsData) @@ -21,9 +27,7 @@ final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { } func testConfigContainerDecodesIntoRemoteConfiguration() throws { - let container = try RemoteConfigContainer(data: self.factory.configContainerData) - let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } - let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + let configuration = try self.decodedConfiguration() XCTAssertEqual(configuration.domain, "app") XCTAssertEqual(configuration.manifest, self.factory.configManifest) @@ -42,9 +46,7 @@ final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { } func testAllReferencedBlobsResolveAndValidate() throws { - let container = try RemoteConfigContainer(data: self.factory.configContainerData) - let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } - let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + let configuration = try self.decodedConfiguration() // Prefetch covers exactly the workflow blobs (what offerings delivery awaits); // ui_config blobs are referenced by their topic but intentionally not prefetched. @@ -64,9 +66,7 @@ final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { } func testWorkflowBlobDecodesIntoPublishedWorkflow() throws { - let container = try RemoteConfigContainer(data: self.factory.configContainerData) - let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } - let configuration = try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + let configuration = try self.decodedConfiguration() let workflowsTopic = try XCTUnwrap(configuration.topics.entries["workflows"]) let ref = try XCTUnwrap(workflowsTopic.values.first?.blobRef) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift index dbf240db29..288f1b716c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -4,27 +4,27 @@ import XCTest final class BenchmarkRunnerValidationTests: BenchmarkTestCase { + private func events(path: String, host: String, statuses: [Int]) -> [TransportEvent] { + let start = DispatchTime.now() + return statuses.compactMap { status in + guard let url = URL(string: "https://\(host)\(path)") else { return nil } + return TransportEvent(kind: RequestKind(url: url), host: host, path: path, + statusCode: status, bytesReceived: 0, + startedAt: start, endedAt: start, failed: false) + } + } + private func measurement( offeringsStatuses: [Int], configStatuses: [Int] = [], blobPaths: [String] = [] ) -> IterationMeasurement { - let start = DispatchTime.now() - var events: [TransportEvent] = [] - for status in offeringsStatuses { - events.append(TransportEvent(host: "api.revenuecat.com", path: "/v1/subscribers/u/offerings", - statusCode: status, bytesReceived: 0, - startedAt: start, endedAt: start, failed: false)) - } - for status in configStatuses { - events.append(TransportEvent(host: "api.revenuecat.com", path: "/v1/config/app", - statusCode: status, bytesReceived: 0, - startedAt: start, endedAt: start, failed: false)) - } + var events = self.events(path: "/v1/subscribers/u/offerings", + host: "api.revenuecat.com", + statuses: offeringsStatuses) + events += self.events(path: "/v1/config/app", host: "api.revenuecat.com", statuses: configStatuses) for path in blobPaths { - events.append(TransportEvent(host: "cdn.revenuecat.local", path: path, - statusCode: 200, bytesReceived: 100, - startedAt: start, endedAt: start, failed: false)) + events += self.events(path: path, host: "cdn.revenuecat.local", statuses: [200]) } return IterationMeasurement(totalMs: 1, events: events) } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift index f4c8d3aaa1..34bb7cfb00 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift @@ -104,76 +104,65 @@ final class FixtureServerTests: BenchmarkTestCase { // MARK: - Transport integration - func testTransportDeliversFixtureBodyAndRecordsEvent() throws { + private func installTransport(lossPercent: Int = 0) { SimulatedTransportURLProtocol.install( server: self.server, profile: .ideal, - loss: LossModel(lossPercent: 0), + loss: LossModel(lossPercent: lossPercent), seed: 42 ) + } + + private func fetch(_ urlString: String, timeout: TimeInterval = 10) throws -> (data: Data?, + statusCode: Int?, + error: Error?) { + let url = try XCTUnwrap(URL(string: urlString)) let session = SimulatedTransportURLProtocol.makeSession() - let ref = try XCTUnwrap(self.factory.allBlobRefs.first) - let url = try XCTUnwrap(URL(string: "https://cdn.revenuecat.local/blobs/\(ref)")) let expectation = self.expectation(description: "request completes") - var received: (data: Data?, statusCode: Int?) + let result = LockedValue<(Data?, Int?, Error?)>() session.dataTask(with: url) { data, response, error in - XCTAssertNil(error) - received = (data, (response as? HTTPURLResponse)?.statusCode) + result.set((data, (response as? HTTPURLResponse)?.statusCode, error)) expectation.fulfill() }.resume() - self.wait(for: [expectation], timeout: 5) + self.wait(for: [expectation], timeout: timeout) + + let (data, statusCode, error) = result.get() ?? (nil, nil, nil) + return (data, statusCode, error) + } + + func testTransportDeliversFixtureBodyAndRecordsEvent() throws { + self.installTransport() + let ref = try XCTUnwrap(self.factory.allBlobRefs.first) + let received = try self.fetch("https://cdn.revenuecat.local/blobs/\(ref)") + + XCTAssertNil(received.error) XCTAssertEqual(received.statusCode, 200) XCTAssertEqual(received.data, self.factory.blobData(forRef: ref)) let events = SimulatedTransportURLProtocol.drainEvents() XCTAssertEqual(events.count, 1) XCTAssertEqual(events.first?.host, "cdn.revenuecat.local") + XCTAssertEqual(events.first?.kind, .blob) XCTAssertEqual(events.first?.bytesReceived, self.factory.blobData(forRef: ref)?.count) XCTAssertTrue(SimulatedTransportURLProtocol.drainEvents().isEmpty, "drain must clear events") } func testTransportInterceptsRealAPIHostsSoNothingLeaks() throws { - SimulatedTransportURLProtocol.install( - server: self.server, - profile: .ideal, - loss: LossModel(lossPercent: 0), - seed: 42 - ) - let session = SimulatedTransportURLProtocol.makeSession() - let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + self.installTransport() - let expectation = self.expectation(description: "request completes") - var received: Data? - session.dataTask(with: url) { data, _, _ in - received = data - expectation.fulfill() - }.resume() - self.wait(for: [expectation], timeout: 5) + let received = try self.fetch("https://api.revenuecat.com/v1/subscribers/u/offerings") - XCTAssertEqual(received, self.factory.offeringsData, "real API hosts must resolve to fixtures") + XCTAssertEqual(received.data, self.factory.offeringsData, "real API hosts must resolve to fixtures") } func testTransportModelsLossAsTimedOutFailure() throws { - SimulatedTransportURLProtocol.install( - server: self.server, - profile: .ideal, - loss: LossModel(lossPercent: 100), - seed: 42 - ) - let session = SimulatedTransportURLProtocol.makeSession() - let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + self.installTransport(lossPercent: 100) - let expectation = self.expectation(description: "request fails") - var receivedError: Error? - session.dataTask(with: url) { _, _, error in - receivedError = error - expectation.fulfill() - }.resume() - self.wait(for: [expectation], timeout: 10) + let received = try self.fetch("https://api.revenuecat.com/v1/subscribers/u/offerings") - XCTAssertEqual((receivedError as? URLError)?.code, .timedOut) + XCTAssertEqual((received.error as? URLError)?.code, .timedOut) let events = SimulatedTransportURLProtocol.drainEvents() XCTAssertEqual(events.count, 1) @@ -181,3 +170,18 @@ final class FixtureServerTests: BenchmarkTestCase { } } + +private final class LockedValue: @unchecked Sendable { + + private let lock = NSLock() + private var value: Value? + + func set(_ value: Value) { + self.lock.withLock { self.value = value } + } + + func get() -> Value? { + return self.lock.withLock { self.value } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift index 0904235dde..a2c84b26d2 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift @@ -21,8 +21,8 @@ final class NetworkProfileTests: BenchmarkTestCase { var first = SeededRandom(seed: 9) var second = SeededRandom(seed: 9) - let firstSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(forHost: "api.revenuecat.com", rng: &first) } - let secondSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(forHost: "api.revenuecat.com", rng: &second) } + let firstSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(for: .offerings, rng: &first) } + let secondSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(for: .offerings, rng: &second) } XCTAssertEqual(firstSamples, secondSamples) for sample in firstSamples { @@ -30,19 +30,31 @@ final class NetworkProfileTests: BenchmarkTestCase { } } - func testCDNHostsSampleFromCDNRange() { + func testBlobRequestsSampleFromCDNRange() { var rng = SeededRandom(seed: 9) for _ in 0..<32 { - let sample = NetworkProfile.lte.rttMs(forHost: "cdn.revenuecat.local", rng: &rng) + let sample = NetworkProfile.lte.rttMs(for: .blob, rng: &rng) XCTAssertTrue(NetworkProfile.lte.cdnRTTMs.contains(sample)) } } + func testRequestKindClassification() throws { + func kind(_ urlString: String) throws -> RequestKind { + return RequestKind(url: try XCTUnwrap(URL(string: urlString))) + } + + XCTAssertEqual(try kind("https://api.revenuecat.com/v1/subscribers/u/offerings"), .offerings) + XCTAssertEqual(try kind("https://api-production.8-lives-cat.io/v1/offerings"), .offerings) + XCTAssertEqual(try kind("https://api.revenuecat.com/v1/config/app"), .config) + XCTAssertEqual(try kind("https://cdn.revenuecat.local/blobs/abc"), .blob) + XCTAssertEqual(try kind("https://d123.cloudfront.net/some/real/cdn/path"), .blob) + } + func testIdealProfileAddsNoDelay() { var rng = SeededRandom(seed: 1) - XCTAssertEqual(NetworkProfile.ideal.rttMs(forHost: "api.revenuecat.com", rng: &rng), 0) + XCTAssertEqual(NetworkProfile.ideal.rttMs(for: .config, rng: &rng), 0) XCTAssertEqual(NetworkProfile.ideal.transferTimeMs(forByteCount: 10_000_000), 0) } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift index c3a807570d..c969763610 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift @@ -53,14 +53,6 @@ final class PassthroughTransportTests: BenchmarkTestCase { XCTAssertEqual(events.first?.failed, false) } - func testAPIHostClassification() { - XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api.revenuecat.com")) - XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api-production.8-lives-cat.io")) - XCTAssertTrue(SimulatedTransportURLProtocol.isAPIHost("api.rc-backup.com")) - XCTAssertFalse(SimulatedTransportURLProtocol.isAPIHost("blobs.revenuecat.com")) - XCTAssertFalse(SimulatedTransportURLProtocol.isAPIHost("d1234.cloudfront.net")) - } - func testUninstalledTransportDoesNotClaimRequests() { SimulatedTransportURLProtocol.uninstall() diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index 5f7db7fab5..ad7e2eccd5 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -57,9 +57,13 @@ def print_single(rows): print("| " + " | ".join(cells) + " |") +def post_warmup_errors(row): + return row.get("post_warmup_error_count", row.get("error_count")) + + def errors_cell(row): """Rows with post-warmup errors are not valid comparison input; make that loud.""" - errors = row.get("post_warmup_error_count", row.get("error_count")) + errors = post_warmup_errors(row) if errors is None: return "-" return f"⚠️ {errors}" if errors else "0" @@ -91,7 +95,7 @@ def print_comparison(baseline, candidate): errors_cell(base), errors_cell(cand), ] - if any(row.get("post_warmup_error_count", row.get("error_count", 0)) for row in (base, cand) if row): + if any(post_warmup_errors(row) for row in (base, cand) if row): invalid += 1 print("| " + " | ".join(cells) + " |") From 29a083b73d37b3f10fe0d9d99759f8b428e45757 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Wed, 8 Jul 2026 20:18:45 +0200 Subject: [PATCH 11/34] other(benchmarks): apply second review pass fixes - warm kill-switch iterations must actually pay the config 4xx (with negative tests), so the mode can never silently degrade into measuring pure legacy - compare.py keys rows by transport/paywalls/workflows/seed too and rejects duplicate rows instead of silently overwriting them - live blob passthrough re-issues on URLSession.shared, matching the production blob downloader exactly - shared LockedValue test helper replaces three private copies; README benchmark command is copy-pastable again Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Tests/Benchmarks/SDKConfigBenchmark/README.md | 13 ++++++++----- .../Sources/BenchmarkRunner.swift | 14 ++++++++++++-- ...ulatedTransportURLProtocol+Passthrough.swift | 17 +++++++---------- .../Tests/BenchmarkRunnerValidationTests.swift | 15 ++++++++++++++- .../Tests/BenchmarkSDKStackTests.swift | 17 +---------------- .../Tests/BenchmarkTestCase.swift | 17 +++++++++++++++++ .../Tests/FixtureServerTests.swift | 15 --------------- .../Tests/PassthroughTransportTests.swift | 5 +++-- Tests/Benchmarks/SDKConfigBenchmark/compare.py | 5 ++++- 9 files changed, 66 insertions(+), 52 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 507fd96d79..9b12b85520 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -49,12 +49,15 @@ python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py results.jsonl xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmark \ -configuration Release -destination platform=macOS build +# --transport: simulated | live +# --mode: legacy | config | config-killswitch (kill switch is simulated-only) +# --profile (ideal | wifi | lte) and --loss-percent are simulated-only SDKConfigBenchmark \ - --transport simulated \ # simulated | live - --mode config \ # legacy | config | config-killswitch (killswitch: simulated only) - --scenario cold \ # cold | warm - --profile lte \ # ideal | wifi | lte (simulated only) - --loss-percent 20 \ # simulated only + --transport simulated \ + --mode config \ + --scenario cold \ + --profile lte \ + --loss-percent 20 \ --iterations 25 \ --warmup-iterations 3 \ --paywalls 50 \ diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index 0fa19008cf..22677036dc 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -178,8 +178,9 @@ extension BenchmarkRunner { ) } - // Kill-switch mode pays the config 4xx on every launch by design, so only plain config - // mode must see manifest 204s. + // Plain config mode must revalidate via manifest 204s; kill-switch mode must pay the + // disabling 4xx on every launch (each iteration is a fresh manager, so the switch + // trips again). Anything else means the mode isn't measuring what it claims. if mode.expectsWarmConfigRevalidation { guard !measurement.configStatusCodes.isEmpty, measurement.configStatusCodes.allSatisfy({ $0 == 204 }) else { @@ -189,6 +190,15 @@ extension BenchmarkRunner { ) } } + if mode.forcesConfigFailure { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ (400...499).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "warm kill-switch iteration \(iteration) did not hit the config 4xx " + + "(statuses: \(measurement.configStatusCodes))" + ) + } + } guard measurement.blobRequestCount == 0 else { throw BenchmarkError.scenarioViolation( diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift index c4f5cfe869..116fcba07a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -5,19 +5,16 @@ import Foundation extension SimulatedTransportURLProtocol { /// Sessions the passthrough re-issues requests on. API traffic mirrors `HTTPClient`'s - /// single-connection-per-host pool; everything else (blob CDNs) gets a default pool like - /// the production blob downloader's `URLSession.shared`. Injectable for tests. - static var passthroughAPISession: URLSession = makePassthroughSession(maxConnectionsPerHost: 1) - static var passthroughBlobSession: URLSession = makePassthroughSession(maxConnectionsPerHost: nil) - - private static func makePassthroughSession(maxConnectionsPerHost: Int?) -> URLSession { + /// pool: ephemeral, no URL cache (the SDK does its own ETag caching), one connection per + /// host. Blob traffic uses `URLSession.shared`, exactly what the production + /// `URLSessionRemoteConfigBlobDownloader` defaults to. Injectable for tests. + static var passthroughAPISession: URLSession = { let configuration = URLSessionConfiguration.ephemeral configuration.urlCache = nil - if let maxConnectionsPerHost { - configuration.httpMaximumConnectionsPerHost = maxConnectionsPerHost - } + configuration.httpMaximumConnectionsPerHost = 1 return URLSession(configuration: configuration) - } + }() + static var passthroughBlobSession: URLSession = .shared func startPassthrough(url: URL, startedAt: DispatchTime) { let session: URLSession diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift index 288f1b716c..fc7cbaef5a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -67,12 +67,25 @@ final class BenchmarkRunnerValidationTests: BenchmarkTestCase { )) } - func testWarmKillSwitchIterationAllowsConfig4xx() throws { + func testWarmKillSwitchIterationRequiresConfig4xx() throws { try BenchmarkRunner.validateWarmMeasurement( self.measurement(offeringsStatuses: [304], configStatuses: [400]), mode: .configKillswitch, iteration: 3 ) + + // Missing the config request, or getting a non-4xx, means the kill switch was not + // actually exercised and the row would measure something else. + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304]), + mode: .configKillswitch, + iteration: 3 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [200]), + mode: .configKillswitch, + iteration: 3 + )) } func testWarmIterationReDownloadingBlobsFails() { diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift index 332c818998..c6e7f68b0b 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift @@ -29,7 +29,7 @@ final class BenchmarkSDKStackTests: BenchmarkTestCase { timeout: TimeInterval = 15 ) throws -> Offerings { let expectation = self.expectation(description: "offerings delivered") - let result = LockedResult>() + let result = LockedValue>() stack.offeringsManager.offerings(appUserID: appUserID, fetchCurrent: true) { offerings in result.set(offerings) expectation.fulfill() @@ -128,18 +128,3 @@ final class BenchmarkSDKStackTests: BenchmarkTestCase { } } - -private final class LockedResult: @unchecked Sendable { - - private let lock = NSLock() - private var value: Value? - - func set(_ value: Value) { - self.lock.withLock { self.value = value } - } - - func get() -> Value? { - return self.lock.withLock { self.value } - } - -} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift index d1b9f52845..8c66ff97cb 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift @@ -8,6 +8,23 @@ import XCTest /// /// Every test gets a fresh disk-cache root so tests never touch the real user Library and /// never see each other's (or a previous run's) cached state. +/// Thread-safe box for values written from URLSession completion handlers and read after +/// `wait(for:)`, shared by every async transport test. +final class LockedValue: @unchecked Sendable { + + private let lock = NSLock() + private var value: Value? + + func set(_ value: Value) { + self.lock.withLock { self.value = value } + } + + func get() -> Value? { + return self.lock.withLock { self.value } + } + +} + // swiftlint:disable:next xctestcase_superclass class BenchmarkTestCase: XCTestCase { diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift index 34bb7cfb00..fa68cbbc79 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift @@ -170,18 +170,3 @@ final class FixtureServerTests: BenchmarkTestCase { } } - -private final class LockedValue: @unchecked Sendable { - - private let lock = NSLock() - private var value: Value? - - func set(_ value: Value) { - self.lock.withLock { self.value = value } - } - - func get() -> Value? { - return self.lock.withLock { self.value } - } - -} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift index c969763610..fa5b5e0386 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift @@ -35,13 +35,14 @@ final class PassthroughTransportTests: BenchmarkTestCase { let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) let expectation = self.expectation(description: "request completes") - var received: (data: Data?, statusCode: Int?) + let result = LockedValue<(data: Data?, statusCode: Int?)>() session.dataTask(with: url) { data, response, _ in - received = (data, (response as? HTTPURLResponse)?.statusCode) + result.set((data, (response as? HTTPURLResponse)?.statusCode)) expectation.fulfill() }.resume() self.wait(for: [expectation], timeout: 5) + let received = try XCTUnwrap(result.get()) XCTAssertEqual(received.statusCode, 200) XCTAssertEqual(received.data, StubBackendURLProtocol.stubBody) XCTAssertEqual(StubBackendURLProtocol.requestCount, 1, "request must reach the (stub) backend") diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index ad7e2eccd5..5635c73ca7 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -13,7 +13,8 @@ import sys -KEY_FIELDS = ("mode", "scenario", "profile", "loss_percent") +KEY_FIELDS = ("mode", "transport", "scenario", "profile", "loss_percent", + "paywalls", "workflows", "seed") METRICS = ("p50_ms", "p95_ms", "request_count_mean", "bytes_received_mean") @@ -26,6 +27,8 @@ def load(path): continue row = json.loads(line) key = tuple(row.get(field) for field in KEY_FIELDS) + if key in rows: + raise SystemExit(f"{path}: duplicate row for {dict(zip(KEY_FIELDS, key))}") rows[key] = row return rows From 1071e154dd2eed16d59997928f33fcaed2243d75 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 00:33:50 +0200 Subject: [PATCH 12/34] other(benchmarks): apply third review pass fixes - legacy Documents-directory migrations (ETagManager, DeviceCache) are gated off while the benchmark disk-root override is active, so runs can never touch or delete real user files outside their sandbox - rows with post-warmup iteration errors exit nonzero (matrix runner counts them and fails at the end), and compare.py exits nonzero on invalid or duplicate rows instead of just warning - comparison keys include iterations and warmup_discarded so different sample-size runs can never be compared as equivalents Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Sources/Caching/DeviceCache.swift | 3 +++ Sources/Caching/DirectoryHelper.swift | 7 +++++++ Sources/Networking/HTTPClient/ETagManager.swift | 3 +++ .../Sources/BenchmarkMain.swift | 10 ++++++++-- .../Sources/BenchmarkMetrics.swift | 9 +++++++-- .../Sources/BenchmarkRunner.swift | 15 +++++++++++++-- .../Tests/BenchmarkMetricsTests.swift | 1 + Tests/Benchmarks/SDKConfigBenchmark/compare.py | 16 +++++++++++----- .../Benchmarks/SDKConfigBenchmark/run-matrix.sh | 13 +++++++++++-- 9 files changed, 64 insertions(+), 13 deletions(-) diff --git a/Sources/Caching/DeviceCache.swift b/Sources/Caching/DeviceCache.swift index 4911da3bd8..ff1b15e7bf 100644 --- a/Sources/Caching/DeviceCache.swift +++ b/Sources/Caching/DeviceCache.swift @@ -856,6 +856,9 @@ private extension DeviceCache { // swiftlint:disable avoid_using_directory_apis_directly private func oldDocumentsDirectoryURL() -> URL? { + #if SDK_CONFIG_BENCHMARK + guard !DirectoryHelper.skipsLegacyDocumentsMigrations else { return nil } + #endif let documentsDirectoryURL: URL? if #available(iOS 16.0, macOS 13.0, tvOS 16.0, watchOS 9.0, *) { documentsDirectoryURL = URL.documentsDirectory diff --git a/Sources/Caching/DirectoryHelper.swift b/Sources/Caching/DirectoryHelper.swift index c88dac826b..b80123ffe4 100644 --- a/Sources/Caching/DirectoryHelper.swift +++ b/Sources/Caching/DirectoryHelper.swift @@ -43,6 +43,13 @@ enum DirectoryHelper { /// processes would share (and corrupt) one another's caches in the real user Library. /// Set once at process startup, before any disk access. static var benchmarkBaseDirectoryOverride: URL? + + /// While the override is active, legacy Documents-directory migrations must not run: + /// they read, move, and delete files under the real user Documents folder, outside the + /// benchmark's sandbox. + static var skipsLegacyDocumentsMigrations: Bool { + return self.benchmarkBaseDirectoryOverride != nil + } #endif static func baseUrl(for type: DirectoryType, inAppSpecificDirectory: Bool = true) -> URL? { diff --git a/Sources/Networking/HTTPClient/ETagManager.swift b/Sources/Networking/HTTPClient/ETagManager.swift index bc799a4e7c..1bfb8a52c8 100644 --- a/Sources/Networking/HTTPClient/ETagManager.swift +++ b/Sources/Networking/HTTPClient/ETagManager.swift @@ -205,6 +205,9 @@ private extension ETagManager { } private func oldETagDirectoryURL() -> URL? { + #if SDK_CONFIG_BENCHMARK + guard !DirectoryHelper.skipsLegacyDocumentsMigrations else { return nil } + #endif // swiftlint:disable:next avoid_using_directory_apis_directly guard let documentsURL = Self.fileManager.urls( for: .documentDirectory, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift index d7236be212..b583dd6d3e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift @@ -37,8 +37,14 @@ public struct BenchmarkMain { DispatchQueue.global(qos: .userInitiated).async { do { - let row = try BenchmarkRunner(command: command).run() - print(row) + let result = try BenchmarkRunner(command: command).run() + print(result.jsonlRow) + if result.postWarmupErrorCount > 0 { + let message = "\(result.postWarmupErrorCount) post-warmup iteration(s) failed; " + + "timings are not valid comparison input\n" + FileHandle.standardError.write(Data(message.utf8)) + finish(exitCode: 2) + } finish(exitCode: 0) } catch { FileHandle.standardError.write(Data("\(error)\n".utf8)) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index 8b8a6d260e..facfc779c0 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -66,12 +66,17 @@ struct BenchmarkMetrics { self.errorsByIteration[iteration] = "\(error)" } + /// Errors in the measured (post-warmup) window. Nonzero means the row's timing statistics + /// are not valid comparison input, and the process must exit nonzero so automation notices. + func postWarmupErrorCount(warmupIterations: Int) -> Int { + return self.errorsByIteration.filter { $0.key >= warmupIterations }.count + } + func jsonlRow(for command: BenchmarkCommand) -> String { let measured = self.measurementsByIteration .filter { $0.key >= command.warmupIterations } .sorted { $0.key < $1.key } .map(\.value) - let postWarmupErrors = self.errorsByIteration.filter { $0.key >= command.warmupIterations } let totals = measured.map(\.totalMs).sorted() var row: [String: Any] = [ @@ -87,7 +92,7 @@ struct BenchmarkMetrics { "warmup_discarded": command.warmupIterations, "measured_iterations": measured.count, "error_count": self.errorsByIteration.count, - "post_warmup_error_count": postWarmupErrors.count + "post_warmup_error_count": self.postWarmupErrorCount(warmupIterations: command.warmupIterations) ] if !totals.isEmpty { diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index 22677036dc..525f4f6607 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -1,5 +1,13 @@ import Foundation +struct BenchmarkRunResult { + + let jsonlRow: String + /// Nonzero means the row is visible but its timings are not valid comparison input. + let postWarmupErrorCount: Int + +} + /// Drives one benchmark configuration: N simulated app launches through the real SDK stack, /// against the simulated transport, producing a single JSONL row. /// @@ -17,7 +25,7 @@ final class BenchmarkRunner { self.command = command } - func run() throws -> String { + func run() throws -> BenchmarkRunResult { switch self.command.transport { case .simulated: guard let profile = NetworkProfile.named(self.command.profileName) else { @@ -64,7 +72,10 @@ final class BenchmarkRunner { } } - return metrics.jsonlRow(for: self.command) + return BenchmarkRunResult( + jsonlRow: metrics.jsonlRow(for: self.command), + postWarmupErrorCount: metrics.postWarmupErrorCount(warmupIterations: self.command.warmupIterations) + ) } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift index 1f138128f4..7c86663edf 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -77,6 +77,7 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { let row = try self.decodeRow(metrics, command: command) XCTAssertEqual(row["post_warmup_error_count"] as? Int, 1) + XCTAssertEqual(metrics.postWarmupErrorCount(warmupIterations: command.warmupIterations), 1) let firstError = try XCTUnwrap(row["first_error"] as? String) XCTAssertTrue(firstError.hasPrefix("iteration 2:")) } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index 5635c73ca7..72e248da3c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -14,7 +14,7 @@ KEY_FIELDS = ("mode", "transport", "scenario", "profile", "loss_percent", - "paywalls", "workflows", "seed") + "paywalls", "workflows", "seed", "iterations", "warmup_discarded") METRICS = ("p50_ms", "p95_ms", "request_count_mean", "bytes_received_mean") @@ -52,12 +52,16 @@ def print_single(rows): header = list(KEY_FIELDS) + list(METRICS) + ["errors"] print("| " + " | ".join(header) + " |") print("|" + "---|" * len(header)) + invalid = 0 for key in sorted(rows, key=str): row = rows[key] cells = [str(part) for part in key] cells += [fmt(row.get(metric)) for metric in METRICS] - cells.append(str(row.get("error_count", 0))) + cells.append(errors_cell(row)) + if post_warmup_errors(row): + invalid += 1 print("| " + " | ".join(cells) + " |") + return invalid def post_warmup_errors(row): @@ -107,17 +111,19 @@ def print_comparison(baseline, candidate): f"\n**⚠️ {invalid} row(s) have post-warmup errors; " "their timing deltas are not valid comparison input.**" ) + return invalid def main(argv): if len(argv) == 2: - print_single(load(argv[1])) + invalid = print_single(load(argv[1])) elif len(argv) == 3: - print_comparison(load(argv[1]), load(argv[2])) + invalid = print_comparison(load(argv[1]), load(argv[2])) else: print(__doc__, file=sys.stderr) return 2 - return 0 + # Invalid rows must fail automation, not just print a warning. + return 1 if invalid else 0 if __name__ == "__main__": diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index 267de5fe7b..f4fcfc9657 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -60,11 +60,12 @@ if [[ ! -x "$BINARY" ]]; then fi SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +FAILED_ROWS=0 run_row() { local mode="$1" scenario="$2" profile="$3" loss="$4" echo "Running transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 - "$BINARY" \ + if ! "$BINARY" \ --transport "$TRANSPORT" \ --mode "$mode" \ --scenario "$scenario" \ @@ -75,7 +76,10 @@ run_row() { --paywalls "$PAYWALLS" \ --workflows "$WORKFLOWS" \ --seed "$SEED" \ - --annotation "sdk_commit=$SDK_COMMIT" + --annotation "sdk_commit=$SDK_COMMIT"; then + echo "Row FAILED (transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%)" >&2 + FAILED_ROWS=$((FAILED_ROWS + 1)) + fi } for mode in $MODES; do @@ -97,3 +101,8 @@ if [[ -n "$LOSS_SWEEP" ]]; then done done fi + +if (( FAILED_ROWS > 0 )); then + echo "$FAILED_ROWS row(s) failed or produced invalid timings" >&2 + exit 1 +fi From a4d931b72698a296d68d4914b14973f1b8d6ebab Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 00:58:07 +0200 Subject: [PATCH 13/34] other(benchmarks): apply fourth review pass fixes - launch order matches Purchases.updateAllCaches exactly (offerings enqueued before the remote config refresh), so config-mode serialization measures the production path instead of an inverted one - fixture 204 additionally requires the request to replay the full prefetched blob set, so a regression in prefetched_blobs reporting surfaces as warm 200s; verified against real SDK behavior by the end-to-end warm test - compare.py fails on baseline/candidate key-set mismatches (opt out with --allow-missing), so truncated result files can't pass a regression gate - live-run user nonce is a UUID, immune to same-second or parallel launches Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkPayloadFactory.swift | 4 ++ .../Sources/BenchmarkRunner.swift | 12 ++++-- .../Sources/FixtureServer.swift | 8 +++- .../Tests/FixtureServerTests.swift | 38 +++++++++++++++++-- .../Benchmarks/SDKConfigBenchmark/compare.py | 15 ++++++-- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift index 8805ad8384..350296397e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift @@ -12,6 +12,9 @@ struct BenchmarkPayloadFactory { let offeringsData: Data let configContainerData: Data + /// The refs the config marks for prefetch; a warm relaunch must report exactly these back + /// as its locally-cached blobs. + let workflowPrefetchRefs: Set private let blobsByRef: [String: Data] @@ -43,6 +46,7 @@ struct BenchmarkPayloadFactory { uiConfigLocalizationsRef: uiConfigLocalizationsRef ) self.configContainerData = RCContainerEncoder.container(config: configJSON, contentElements: []) + self.workflowPrefetchRefs = Set(workflowRefs.values) self.blobsByRef = blobs } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index 525f4f6607..d274377f69 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -82,9 +82,9 @@ final class BenchmarkRunner { private extension BenchmarkRunner { - /// Live runs get a per-run nonce so reruns never share server-side per-user state with a - /// previous run; simulated runs stay fully deterministic. - private static let liveRunNonce = String(Int(Date().timeIntervalSince1970)) + /// Live runs get a per-run nonce so reruns (even parallel or same-second ones) never share + /// server-side per-user state; simulated runs stay fully deterministic. + private static let liveRunNonce = String(UUID().uuidString.prefix(8)) var baseAppUserID: String { switch self.command.transport { @@ -140,9 +140,12 @@ private extension BenchmarkRunner { /// One simulated launch; returns start-to-offerings-delivered wall time in milliseconds. /// Runs off the main thread; the offerings completion is delivered on the main queue, which /// `BenchmarkMain` keeps pumping via `dispatchMain()`. + /// + /// The offerings fetch is kicked BEFORE the remote config refresh, matching + /// `Purchases.updateAllCaches` exactly: both land on the same serial backend queue and the + /// same single-connection host pool, so enqueue order shapes the measured serialization. func launch(_ stack: BenchmarkSDKStack, appUserID: String) throws -> Double { let start = DispatchTime.now() - stack.refreshRemoteConfigIfWired() let semaphore = DispatchSemaphore(value: 0) let failure = Atomic(nil) @@ -152,6 +155,7 @@ private extension BenchmarkRunner { } semaphore.signal() } + stack.refreshRemoteConfigIfWired() guard semaphore.wait(timeout: .now() + 120) == .success else { throw BenchmarkError.timeout("offerings fetch") diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift index dbc7bc2151..d6641521fa 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift @@ -71,9 +71,15 @@ final class FixtureServer { if self.killSwitchConfig { return self.killSwitchResponse } + // 204 requires the client to prove BOTH that its config is current (manifest) and + // that it still holds every prefetched blob (`prefetched_blobs`). A client that + // stops replaying its cached refs gets a full response, so a regression in that + // reporting shows up as warm 200s instead of silently passing. if let bodyData, let body = try? JSONSerialization.jsonObject(with: bodyData) as? [String: Any], - body["manifest"] as? String == self.factory.configManifest { + body["manifest"] as? String == self.factory.configManifest, + let prefetchedBlobs = body["prefetched_blobs"] as? [String], + Set(prefetchedBlobs).isSuperset(of: self.factory.workflowPrefetchRefs) { return self.configNotModifiedResponse } return self.configResponse diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift index fa68cbbc79..58d8a94e26 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift @@ -52,19 +52,49 @@ final class FixtureServerTests: BenchmarkTestCase { XCTAssertEqual(fallback.body, self.factory.offeringsData) } - func testConfigServes200ContainerThen204OnMatchingManifest() throws { + private func configBody(manifest: String? = nil, prefetchedBlobs: [String]? = nil) throws -> Data { + var body: [String: Any] = ["app_user_id": "u"] + if let manifest { + body["manifest"] = manifest + } + if let prefetchedBlobs { + body["prefetched_blobs"] = prefetchedBlobs + } + return try JSONSerialization.data(withJSONObject: body) + } + + func testConfigServes200ContainerThen204OnMatchingManifestAndBlobs() throws { let url = "https://api.revenuecat.com/v1/config/app" - let cold = self.server.response(for: try self.request(url), bodyData: Data(#"{"appUserId":"u"}"#.utf8)) + let cold = self.server.response(for: try self.request(url), bodyData: try self.configBody()) XCTAssertEqual(cold.statusCode, 200) XCTAssertEqual(cold.body, self.factory.configContainerData) - let warmBody = Data(#"{"appUserId":"u","manifest":"benchmark-manifest-v1"}"#.utf8) - let warm = self.server.response(for: try self.request(url), bodyData: warmBody) + // 204 requires the client to prove its config is current AND that it still holds the + // prefetched blobs it was told to cache. + let warm = self.server.response( + for: try self.request(url), + bodyData: try self.configBody( + manifest: self.factory.configManifest, + prefetchedBlobs: Array(self.factory.workflowPrefetchRefs) + ) + ) XCTAssertEqual(warm.statusCode, 204) XCTAssertTrue(warm.body.isEmpty) } + func testConfigMatchingManifestWithoutCachedBlobProofGets200() throws { + let url = "https://api.revenuecat.com/v1/config/app" + + for prefetchedBlobs in [nil, [], Array(self.factory.workflowPrefetchRefs.dropFirst())] as [[String]?] { + let response = self.server.response( + for: try self.request(url), + bodyData: try self.configBody(manifest: self.factory.configManifest, prefetchedBlobs: prefetchedBlobs) + ) + XCTAssertEqual(response.statusCode, 200, "missing blob proof \(String(describing: prefetchedBlobs))") + } + } + func testKillSwitchConfigReturns400ButOfferingsStillServe() throws { let killServer = FixtureServer(factory: self.factory, killSwitchConfig: true) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index 72e248da3c..52bfdef20b 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -76,7 +76,7 @@ def errors_cell(row): return f"⚠️ {errors}" if errors else "0" -def print_comparison(baseline, candidate): +def print_comparison(baseline, candidate, allow_missing=False): header = list(KEY_FIELDS) for metric in ("p50_ms", "p95_ms"): header += [f"{metric} base", f"{metric} cand", "Δ"] @@ -85,6 +85,7 @@ def print_comparison(baseline, candidate): print("|" + "---|" * len(header)) invalid = 0 + missing = sorted(set(baseline) ^ set(candidate), key=str) for key in sorted(set(baseline) | set(candidate), key=str): base, cand = baseline.get(key, {}), candidate.get(key, {}) cells = [str(part) for part in key] @@ -111,18 +112,26 @@ def print_comparison(baseline, candidate): f"\n**⚠️ {invalid} row(s) have post-warmup errors; " "their timing deltas are not valid comparison input.**" ) + if missing and not allow_missing: + print(f"\n**⚠️ {len(missing)} configuration(s) present in only one file:**") + for key in missing: + side = "candidate" if key in candidate else "baseline" + print(f"- only in {side}: {dict(zip(KEY_FIELDS, key))}") + invalid += len(missing) return invalid def main(argv): + allow_missing = "--allow-missing" in argv + argv = [arg for arg in argv if arg != "--allow-missing"] if len(argv) == 2: invalid = print_single(load(argv[1])) elif len(argv) == 3: - invalid = print_comparison(load(argv[1]), load(argv[2])) + invalid = print_comparison(load(argv[1]), load(argv[2]), allow_missing=allow_missing) else: print(__doc__, file=sys.stderr) return 2 - # Invalid rows must fail automation, not just print a warning. + # Invalid or incomplete rows must fail automation, not just print a warning. return 1 if invalid else 0 From 5770b32a46083fd19a062ad1e7ef5d89356b4035 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 01:24:38 +0200 Subject: [PATCH 14/34] other(benchmarks): apply fifth review pass fixes - launch drives OfferingsManager.updateOfferingsCache directly, the exact call (and synchronous enqueue order) Purchases.updateAllCaches uses, so the config request can never race ahead of offerings through the main actor hop - transport events carry the iteration that started them and the runner waits for transport quiescence before draining, so trailing requests (the warm config 204 that legitimately finishes after offerings delivery) are counted in the right row and stragglers from failed launches are filtered out - failed launches close their remote config manager to stop residual churn - live rows label paywalls/workflows as 0 (payloads come from the pinned project, not fixture-size knobs) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkCommand.swift | 5 + .../Sources/BenchmarkRunner.swift | 34 +++-- ...atedTransportURLProtocol+Passthrough.swift | 6 +- .../SimulatedTransportURLProtocol.swift | 117 +++++++----------- .../Sources/TransportEvent.swift | 82 ++++++++++++ .../Tests/BenchmarkCommandTests.swift | 8 ++ .../Tests/BenchmarkMetricsTests.swift | 2 +- .../BenchmarkRunnerValidationTests.swift | 2 +- 8 files changed, 169 insertions(+), 87 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index bd363bddb6..18c59ad207 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -195,6 +195,11 @@ struct BenchmarkCommand { if self.apiKey == Self.defaultSimulatedAPIKey { self.apiKey = BenchmarkProject.testStoreAPIKey } + + // Fixture-size knobs don't shape live payloads (the pinned project's real content + // does), so rows carry 0 rather than a fixture size that was never measured. + self.paywallCount = 0 + self.workflowCount = 0 } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index d274377f69..7e72c90fad 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -114,6 +114,7 @@ private extension BenchmarkRunner { ) stack.clearAllDiskState() _ = try self.launch(stack, appUserID: self.baseAppUserID) + SimulatedTransportURLProtocol.waitUntilIdle() _ = SimulatedTransportURLProtocol.drainEvents() } @@ -129,27 +130,42 @@ private extension BenchmarkRunner { } _ = SimulatedTransportURLProtocol.drainEvents() - let totalMs = try self.launch(stack, appUserID: appUserID) + let totalMs: Double + do { + totalMs = try self.launch(stack, appUserID: appUserID) + } catch { + // Quiet the failed stack so its in-flight work stops churning; any straggler + // events it still produces carry this iteration's stamp and are filtered out of + // later measurements below. + stack.remoteConfigManager?.close() + throw error + } - return IterationMeasurement( - totalMs: totalMs, - events: SimulatedTransportURLProtocol.drainEvents() - ) + // The clock stopped at offerings delivery, but trailing requests from this launch + // (e.g. the warm config 204 that finishes after offerings, matching production order) + // still belong to this iteration's accounting: wait for quiescence, then keep only + // this iteration's events so stragglers from earlier launches can't inflate the row. + SimulatedTransportURLProtocol.waitUntilIdle() + let events = SimulatedTransportURLProtocol.drainEvents().filter { $0.iteration == iteration } + return IterationMeasurement(totalMs: totalMs, events: events) } /// One simulated launch; returns start-to-offerings-delivered wall time in milliseconds. /// Runs off the main thread; the offerings completion is delivered on the main queue, which /// `BenchmarkMain` keeps pumping via `dispatchMain()`. /// - /// The offerings fetch is kicked BEFORE the remote config refresh, matching - /// `Purchases.updateAllCaches` exactly: both land on the same serial backend queue and the - /// same single-connection host pool, so enqueue order shapes the measured serialization. + /// This calls the exact pair `Purchases.updateAllCaches` calls, in the same order: + /// `updateOfferingsCache` (which enqueues the offerings operation synchronously, before + /// this method moves on) and then `refreshRemoteConfig`. Both land on the same serial + /// backend queue and single-connection host pool, so enqueue order shapes the measured + /// serialization; using `offerings(appUserID:)` instead would hop through the main actor + /// first and let the config request race ahead. func launch(_ stack: BenchmarkSDKStack, appUserID: String) throws -> Double { let start = DispatchTime.now() let semaphore = DispatchSemaphore(value: 0) let failure = Atomic(nil) - stack.offeringsManager.offerings(appUserID: appUserID) { result in + stack.offeringsManager.updateOfferingsCache(appUserID: appUserID, isAppBackgrounded: false) { result in if case let .failure(error) = result { failure.value = error } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift index 116fcba07a..27998038ff 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -16,7 +16,7 @@ extension SimulatedTransportURLProtocol { }() static var passthroughBlobSession: URLSession = .shared - func startPassthrough(url: URL, startedAt: DispatchTime) { + func startPassthrough(url: URL, iteration: Int, startedAt: DispatchTime) { let session: URLSession switch RequestKind(url: url) { case .offerings, .config: session = Self.passthroughAPISession @@ -27,12 +27,13 @@ extension SimulatedTransportURLProtocol { guard let self else { return } if error != nil { - Self.record(.failure(url: url, startedAt: startedAt)) + Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol(self, didFailWithError: error ?? URLError(.unknown)) return } guard let httpResponse = response as? HTTPURLResponse else { + Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol( self, didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(url.host ?? "")") @@ -42,6 +43,7 @@ extension SimulatedTransportURLProtocol { Self.record(.success( url: url, + iteration: iteration, statusCode: httpResponse.statusCode, bytesReceived: data?.count ?? 0, startedAt: startedAt diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index 338230e2a8..02251be87e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -1,74 +1,5 @@ import Foundation -/// What a benchmark request is for. Classified once, from the URL, and stamped on every -/// `TransportEvent`, so routing, RTT class, connection pooling, phase attribution, and warm -/// validation all share one rule instead of re-deriving it from path strings. -enum RequestKind { - - case offerings - case config - /// Anything that is neither offerings nor config: blob downloads, whose URLs come from the - /// (real or fixture) config's `url_format` and so have no fixed shape. - case blob - - init(url: URL) { - let path = url.path - if path.hasSuffix("/offerings") { - self = .offerings - } else if path.hasSuffix("/config/app") { - self = .config - } else { - self = .blob - } - } - -} - -/// One completed (or failed) simulated request, for phase attribution and byte accounting. -struct TransportEvent { - - let kind: RequestKind - let host: String - let path: String - let statusCode: Int - let bytesReceived: Int - let startedAt: DispatchTime - let endedAt: DispatchTime - let failed: Bool - - /// The SDK's backup hosts; requests here mean the primary host failed over. - var isFallbackHostRequest: Bool { - return self.host.contains("8-lives-cat") || self.host.contains("rc-backup") - } - - static func failure(url: URL, startedAt: DispatchTime) -> TransportEvent { - return TransportEvent( - kind: RequestKind(url: url), - host: url.host ?? "", - path: url.path, - statusCode: 0, - bytesReceived: 0, - startedAt: startedAt, - endedAt: DispatchTime.now(), - failed: true - ) - } - - static func success(url: URL, statusCode: Int, bytesReceived: Int, startedAt: DispatchTime) -> TransportEvent { - return TransportEvent( - kind: RequestKind(url: url), - host: url.host ?? "", - path: url.path, - statusCode: statusCode, - bytesReceived: bytesReceived, - startedAt: startedAt, - endedAt: DispatchTime.now(), - failed: false - ) - } - -} - /// The transport used by every URLSession in the benchmark. Two modes: /// /// **Simulated**: requests resolve against an in-process `FixtureServer`, so no run can ever @@ -97,6 +28,9 @@ final class SimulatedTransportURLProtocol: URLProtocol { /// distinct (but stable) plans. private static var attemptCountsByURL: [String: Int] = [:] private static var events: [TransportEvent] = [] + /// Requests started but not yet recorded. Every `startLoading` is balanced by exactly one + /// `record`, so zero means the transport is quiescent. + private static var inFlightCount = 0 static let chunkSize = 16 * 1024 @@ -141,6 +75,22 @@ final class SimulatedTransportURLProtocol: URLProtocol { } } + /// Blocks until every started request has recorded its event (or the timeout passes). + /// Launch measurements stop at offerings delivery, but trailing requests from the same + /// launch (e.g. the warm config 204 finishing after offerings) still belong to the + /// iteration's accounting, so callers wait for quiescence before draining. + @discardableResult + static func waitUntilIdle(timeoutMs: Int = 10_000) -> Bool { + let deadline = DispatchTime.now() + .milliseconds(timeoutMs) + while DispatchTime.now() < deadline { + if self.lock.withLock({ self.inFlightCount }) == 0 { + return true + } + usleep(2_000) + } + return self.lock.withLock { self.inFlightCount } == 0 + } + /// Returns all events recorded since the last drain, oldest first. static func drainEvents() -> [TransportEvent] { return self.lock.withLock { @@ -184,11 +134,22 @@ final class SimulatedTransportURLProtocol: URLProtocol { return } + let iteration = Self.lock.withLock { () -> Int in + Self.inFlightCount += 1 + return Self.iterationIndex + } switch mode { case let .simulated(server, profile, loss): - self.startSimulated(url: url, server: server, profile: profile, loss: loss, startedAt: started) + self.startSimulated( + url: url, + server: server, + profile: profile, + loss: loss, + iteration: iteration, + startedAt: started + ) case .passthrough: - self.startPassthrough(url: url, startedAt: started) + self.startPassthrough(url: url, iteration: iteration, startedAt: started) } } @@ -209,11 +170,13 @@ final class SimulatedTransportURLProtocol: URLProtocol { private extension SimulatedTransportURLProtocol { + // swiftlint:disable:next function_parameter_count func startSimulated( url: URL, server: FixtureServer, profile: NetworkProfile, loss: LossModel, + iteration: Int, startedAt: DispatchTime ) { let bodyData = Self.bodyData(of: self.request) @@ -231,13 +194,14 @@ private extension SimulatedTransportURLProtocol { let plan = Self.requestPlan(key: key, bodyCount: response.body.count, profile: profile, loss: loss) if plan.fails { - self.deliverFailure(url: url, rttMs: plan.rttMs, startedAt: startedAt) + self.deliverFailure(url: url, iteration: iteration, rttMs: plan.rttMs, startedAt: startedAt) } else { self.deliverResponse( response, url: url, profile: profile, plan: (plan.rttMs, plan.chunkDelaysMs), + iteration: iteration, startedAt: startedAt ) } @@ -251,20 +215,22 @@ private extension SimulatedTransportURLProtocol { /// One retransmission-timeout's worth of waiting before the failure surfaces, so failed /// attempts cost time like they do on a real link. - func deliverFailure(url: URL, rttMs: Double, startedAt: DispatchTime) { + func deliverFailure(url: URL, iteration: Int, rttMs: Double, startedAt: DispatchTime) { let rtoMs = max(1_000, rttMs * 2) self.schedule(afterMs: rtoMs) { [weak self] in guard let self else { return } - Self.record(.failure(url: url, startedAt: startedAt)) + Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol(self, didFailWithError: URLError(.timedOut)) } } + // swiftlint:disable:next function_parameter_count func deliverResponse( _ response: FixtureServer.Response, url: URL, profile: NetworkProfile, plan: (rttMs: Double, chunkDelaysMs: [Double]), + iteration: Int, startedAt: DispatchTime ) { guard let httpResponse = HTTPURLResponse( @@ -273,6 +239,7 @@ private extension SimulatedTransportURLProtocol { httpVersion: "HTTP/1.1", headerFields: response.headers ) else { + Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol( self, didFailWithError: BenchmarkError.invalidFixture("Could not create fixture HTTP response") @@ -308,6 +275,7 @@ private extension SimulatedTransportURLProtocol { guard let self else { return } Self.record(.success( url: url, + iteration: iteration, statusCode: response.statusCode, bytesReceived: response.body.count, startedAt: startedAt @@ -331,6 +299,7 @@ extension SimulatedTransportURLProtocol { static func record(_ event: TransportEvent) { self.lock.withLock { self.events.append(event) + self.inFlightCount -= 1 } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift new file mode 100644 index 0000000000..cb2b6a5088 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift @@ -0,0 +1,82 @@ +import Foundation + +/// What a benchmark request is for. Classified once, from the URL, and stamped on every +/// `TransportEvent`, so routing, RTT class, connection pooling, phase attribution, and warm +/// validation all share one rule instead of re-deriving it from path strings. +enum RequestKind { + + case offerings + case config + /// Anything that is neither offerings nor config: blob downloads, whose URLs come from the + /// (real or fixture) config's `url_format` and so have no fixed shape. + case blob + + init(url: URL) { + let path = url.path + if path.hasSuffix("/offerings") { + self = .offerings + } else if path.hasSuffix("/config/app") { + self = .config + } else { + self = .blob + } + } + +} + +/// One completed (or failed) simulated request, for phase attribution and byte accounting. +struct TransportEvent { + + let kind: RequestKind + /// The iteration active when the request STARTED; measurements only aggregate events + /// stamped with their own iteration, so stragglers from failed or slow earlier launches + /// can never contaminate a measured row. + let iteration: Int + let host: String + let path: String + let statusCode: Int + let bytesReceived: Int + let startedAt: DispatchTime + let endedAt: DispatchTime + let failed: Bool + + /// The SDK's backup hosts; requests here mean the primary host failed over. + var isFallbackHostRequest: Bool { + return self.host.contains("8-lives-cat") || self.host.contains("rc-backup") + } + + static func failure(url: URL, iteration: Int, startedAt: DispatchTime) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + iteration: iteration, + host: url.host ?? "", + path: url.path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: true + ) + } + + static func success( + url: URL, + iteration: Int, + statusCode: Int, + bytesReceived: Int, + startedAt: DispatchTime + ) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + iteration: iteration, + host: url.host ?? "", + path: url.path, + statusCode: statusCode, + bytesReceived: bytesReceived, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: false + ) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index db9c5fb894..185fb1b768 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -83,6 +83,14 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertEqual(command.apiKey, BenchmarkProject.testStoreAPIKey) } + func testLiveTransportZeroesFixtureSizeKnobs() throws { + let command = try BenchmarkCommand.parse(["--transport", "live", "--paywalls", "500", "--workflows", "500"]) + + // Live payloads come from the pinned project, so fixture sizes must not label the row. + XCTAssertEqual(command.paywallCount, 0) + XCTAssertEqual(command.workflowCount, 0) + } + func testLiveTransportKeepsExplicitAPIKey() throws { let command = try BenchmarkCommand.parse(["--transport", "live", "--api-key", "appl_other"]) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift index 7c86663edf..d3629b423d 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -121,7 +121,7 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { failed: Bool = false ) throws -> TransportEvent { let url = try XCTUnwrap(URL(string: "https://\(host)\(path)")) - return TransportEvent(kind: RequestKind(url: url), host: host, path: path, + return TransportEvent(kind: RequestKind(url: url), iteration: 0, host: host, path: path, statusCode: status, bytesReceived: 100, startedAt: time(startMs), endedAt: time(endMs), failed: failed) } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift index fc7cbaef5a..d56fbe6eca 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -8,7 +8,7 @@ final class BenchmarkRunnerValidationTests: BenchmarkTestCase { let start = DispatchTime.now() return statuses.compactMap { status in guard let url = URL(string: "https://\(host)\(path)") else { return nil } - return TransportEvent(kind: RequestKind(url: url), host: host, path: path, + return TransportEvent(kind: RequestKind(url: url), iteration: 0, host: host, path: path, statusCode: status, bytesReceived: 0, startedAt: start, endedAt: start, failed: false) } From 7e828bac8b5ab7a7e0a9f742b688fd3842d7c405 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 01:33:22 +0200 Subject: [PATCH 15/34] other(benchmarks): wait for the iteration's config event and refresh numbers Transport idleness alone can race the gap between kicking the refresh and its request reaching the transport (seen against the real backend), so event collection also waits for the iteration's config request in config modes. Sample tables regenerated with the production-order launch semantics; warm config delivery now measures at parity with legacy, matching production. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 96 ++++++++++--------- .../Sources/BenchmarkRunner.swift | 29 +++++- 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index c912c5b563..a34bf49138 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -131,75 +131,83 @@ macOS Release build: | mode | scenario | profile | loss % | p50 ms | p95 ms | requests (mean) | bytes (mean) | |---|---|---|---:|---:|---:|---:|---:| -| legacy | cold | ideal | 0 | 7.2 | 7.5 | 1 | 64,972 | -| config | cold | ideal | 0 | 24.8 | 27.1 | 102 | 132,550 | -| config-killswitch | cold | ideal | 0 | 7.6 | 7.9 | 2 | 65,022 | -| legacy | cold | lte | 0 | 138.2 | 174.2 | 1 | 64,972 | -| config | cold | lte | 0 | 1,457.7 | 1,522.3 | 102 | 132,550 | -| config-killswitch | cold | lte | 0 | 234.7 | 273.8 | 2 | 65,022 | -| legacy | warm | ideal | 0 | 7.6 | 7.8 | 1 | 0 | -| config | warm | ideal | 0 | 10.4 | 11.2 | 2 | 0 | -| config-killswitch | warm | ideal | 0 | 8.2 | 8.5 | 2 | 50 | -| legacy | warm | lte | 0 | 95.5 | 123.0 | 1 | 0 | -| config | warm | lte | 0 | 184.2 | 233.0 | 2 | 0 | -| config-killswitch | warm | lte | 0 | 185.3 | 227.7 | 2 | 50 | -| legacy | cold | lte | 10 | 154.9 | 288.9 | 1 | 64,972 | -| config | cold | lte | 10 | 1,764.6 | 2,152.9 | 102 | 132,503 | -| legacy | cold | lte | 20 | 162.2 | 297.3 | 1 | 64,972 | -| config | cold | lte | 20 | 2,108.9 | 3,011.0 | 94 | 127,878 | -| legacy | cold | lte | 30 | 219.1 | 626.0 | 1 | 64,972 | -| config | cold | lte | 30 | 1,967.8 | 2,847.1 | 62 | 110,680 | +| legacy | cold | ideal | 0 | 6.6 | 7.3 | 1 | 64,972 | +| config | cold | ideal | 0 | 30.8 | 35.1 | 102 | 132,550 | +| config-killswitch | cold | ideal | 0 | 6.9 | 7.7 | 2 | 65,022 | +| legacy | cold | lte | 0 | 146.5 | 176.5 | 1 | 64,972 | +| config | cold | lte | 0 | 1,653.2 | 1,735.7 | 102 | 132,550 | +| config-killswitch | cold | lte | 0 | 239.8 | 280.0 | 2 | 65,022 | +| legacy | warm | ideal | 0 | 7.0 | 7.2 | 1 | 0 | +| config | warm | ideal | 0 | 8.1 | 8.5 | 2 | 0 | +| config-killswitch | warm | ideal | 0 | 7.0 | 7.8 | 2 | 50 | +| legacy | warm | lte | 0 | 102.8 | 131.4 | 1 | 0 | +| config | warm | lte | 0 | 108.4 | 133.2 | 2 | 0 | +| config-killswitch | warm | lte | 0 | 198.9 | 237.8 | 2 | 50 | +| legacy | cold | lte | 10 | 158.3 | 296.2 | 1 | 64,972 | +| config | cold | lte | 10 | 1,948.1 | 2,577.2 | 102 | 132,503 | +| legacy | cold | lte | 20 | 169.5 | 300.7 | 1 | 64,972 | +| config | cold | lte | 20 | 2,352.9 | 3,128.8 | 96 | 128,778 | +| legacy | cold | lte | 30 | 222.5 | 630.2 | 1 | 64,972 | +| config | cold | lte | 30 | 2,828.5 | 3,802.0 | 74 | 116,852 | Every warm row above is verified per iteration: offerings revalidated via 304, config via -manifest 204 (or the expected 4xx in kill-switch mode), and zero blob re-downloads. +manifest 204 with the full prefetched-blob proof (or the expected 4xx in kill-switch mode), +and zero blob re-downloads. Launches fire the exact `updateAllCaches` pair in production +order (offerings enqueued before the config refresh), and each iteration's accounting +includes trailing requests that finish after offerings delivery (like the warm config 204). ### Live sample numbers `TRANSPORT=live` matrix (25 iterations, 3 warmup discarded) against the stress-test project, -run 2026-07-08 from a residential connection: +run 2026-07-09 from a residential connection: | mode | scenario | p50 ms | p95 ms | requests (mean) | bytes (mean) | |---|---|---:|---:|---:|---:| -| legacy | cold | 163.6 | 334.9 | 1 | 32,124 | -| config | cold | 334.1 | 525.4 | 2 | 36,052 | -| legacy | warm | 122.7 | 321.1 | 1 | 0 | -| config | warm | 316.4 | 635.0 | 2 | 0 | +| legacy | cold | 133.4 | 331.4 | 1 | 32,124 | +| config | cold | 280.0 | 564.9 | 2 | 36,052 | +| legacy | warm | 135.1 | 329.0 | 1 | 0 | +| config | warm | 142.2 | 398.9 | 2 | 0 | Live observations (as of this run): - The project's `/v1/config` response currently drives **no blob downloads** (2 requests - total: config + offerings), so live config cost is simply one extra API round trip, - serialized behind offerings by `HTTPClient`'s single-connection queue. As the backend starts - serving workflow/paywall blobs for this project, live runs will pick that up automatically. + total: config + offerings), so live config cost is one extra API round trip. As the backend + starts serving workflow/paywall blobs for this project, live runs pick that up automatically. +- **Warm config launches cost almost nothing extra** (142ms vs 135ms p50): offerings delivery + does not wait for the config revalidation when the topics are already on disk, so the 204 + trails after delivery, exactly like production. Cold config pays the full extra round trip + (280ms vs 133ms) because first delivery waits for the config sync. - Warm revalidation works end to end against production: offerings answers `304` to - `X-RevenueCat-ETag` and config answers `204` to a matching manifest, both with zero content - bytes. The runner verifies this and fails the run if it stops happening. + `X-RevenueCat-ETag` and config answers `204` to a matching manifest + prefetched-blob set, + both with zero content bytes. The runner verifies every measured iteration and fails the + run if any falls back to full responses. ## Interpretation - **Prefetch gating dominates config cold starts.** With 100 workflows all marked - `prefetch: true`, offerings delivery waits for ~100 blob downloads through the fetcher's - 4-concurrent-download cap: ~25 sequential batches at LTE CDN RTTs is ~1.3-1.5s, which is - exactly the gap to legacy (1,493ms vs 146ms p50). This is a **fixture policy worst case**, - not an inherent cost of the config system; the real product decision is prefetch scope - (current offering only? top N?), and this harness can now measure each option. -- **Warm launches are where the config design pays off.** A warm config relaunch is 2 tiny - revalidation requests (manifest 204 + offerings 304) and zero content bytes; blobs are - content-addressed and never re-downloaded. The remaining LTE gap (212ms vs 101ms p50) is one - extra API round trip, serialized behind the offerings call by `HTTPClient`'s - single-connection-per-host policy; request parallelism is a measurable lever here. + `prefetch: true`, first offerings delivery waits for ~100 blob downloads through the + fetcher's 4-concurrent cap: ~25 sequential batches at LTE CDN RTTs is the entire gap to + legacy (1,653ms vs 147ms p50). This is a **fixture policy worst case**, not an inherent + cost of the config system; the real product decision is prefetch scope (current offering + only? top N?), and this harness can now measure each option. +- **Warm launches are where the config design pays off.** A warm config relaunch delivers + offerings essentially as fast as legacy (108ms vs 103ms p50 on LTE): the topics are already + on disk, so delivery doesn't wait for the manifest 204, which revalidates in the background. + Content bytes are zero and blobs are never re-downloaded. - **The kill switch is cheap.** Force-failing `/v1/config` with a 4xx costs roughly one API - round trip over pure legacy (232ms vs 146ms cold LTE p50), after which the session behaves - like legacy. No cascading retries, no blob traffic. + round trip over pure legacy on a cold LTE launch (240ms vs 147ms p50), after which the + session behaves like legacy. No cascading retries, no blob traffic. Warm kill-switch + launches keep paying that round trip each launch (199ms vs 103ms), which is the argument + for flipping the switch server-side only briefly. - **Loss hurts config more in absolute terms** because it multiplies across ~100 requests (and at 30% loss, failed blob downloads start shrinking the request/byte counts as fetches give up), while legacy's single request degrades only its own tail. Per request the two systems degrade the same way; the difference is request count, which again points at prefetch scope and blob layout as the levers that matter. - **Bytes double on cold config** (133KB vs 65KB) in this fixture because offerings still - carries full `paywall_components` while the config path also downloads workflow and - `ui_config` blobs. The "stripped offerings" lever below is the measurement that matters for - deciding whether to remove paywall data from `/offerings`. + carries full `paywall_components` while the config path also downloads workflow blobs. The + "stripped offerings" lever below is the measurement that matters for deciding whether to + remove paywall data from `/offerings`. ## Known limitations diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index 7e72c90fad..d59f576dcd 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -143,13 +143,34 @@ private extension BenchmarkRunner { // The clock stopped at offerings delivery, but trailing requests from this launch // (e.g. the warm config 204 that finishes after offerings, matching production order) - // still belong to this iteration's accounting: wait for quiescence, then keep only - // this iteration's events so stragglers from earlier launches can't inflate the row. - SimulatedTransportURLProtocol.waitUntilIdle() - let events = SimulatedTransportURLProtocol.drainEvents().filter { $0.iteration == iteration } + // still belong to this iteration's accounting: collect until the transport is idle + // AND the config request (when wired) has landed, then keep only this iteration's + // events so stragglers from earlier launches can't inflate the row. + let events = self.collectEvents(forIteration: iteration) return IterationMeasurement(totalMs: totalMs, events: events) } + /// Drains transport events until quiescence, additionally waiting (bounded) for this + /// iteration's config request in config modes: transport idleness alone can race the gap + /// between kicking the refresh and its request reaching the transport. + func collectEvents(forIteration iteration: Int, timeoutMs: Int = 15_000) -> [TransportEvent] { + var events: [TransportEvent] = [] + let deadline = DispatchTime.now() + .milliseconds(timeoutMs) + + while true { + SimulatedTransportURLProtocol.waitUntilIdle() + events += SimulatedTransportURLProtocol.drainEvents() + + let configLanded = events.contains { $0.iteration == iteration && $0.kind == .config } + if !self.command.mode.usesRemoteConfig || configLanded || DispatchTime.now() >= deadline { + break + } + usleep(5_000) + } + + return events.filter { $0.iteration == iteration } + } + /// One simulated launch; returns start-to-offerings-delivered wall time in milliseconds. /// Runs off the main thread; the offerings completion is delivered on the main queue, which /// `BenchmarkMain` keeps pumping via `dispatchMain()`. From d0e86051c1689dce9eb6f129be98ea09b1d39fca Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 02:03:24 +0200 Subject: [PATCH 16/34] other(benchmarks): apply final review pass fixes - cancelled simulated requests record their terminal event from stopLoading (record-once), so the in-flight counter can never leak and stall later iterations after an HTTP-client timeout - annotation keys that collide with reserved row fields are rejected, so a row can never be relabeled into the wrong comparison group Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkCommand.swift | 14 ++++++++ ...atedTransportURLProtocol+Passthrough.swift | 6 ++-- .../SimulatedTransportURLProtocol.swift | 36 ++++++++++++++++--- .../Tests/BenchmarkCommandTests.swift | 8 +++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 18c59ad207..d48811a4b1 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -69,6 +69,17 @@ struct BenchmarkCommand { static let defaultSimulatedAPIKey = "appl_benchmark" + /// Keys `jsonlRow` writes itself; annotations must never overwrite them, or a row could + /// lie about what was measured (e.g. `--annotation mode=legacy` on a config run). + static let reservedRowKeys: Set = [ + "mode", "transport", "scenario", "profile", "loss_percent", "paywalls", "workflows", + "seed", "iterations", "warmup_discarded", "measured_iterations", "error_count", + "post_warmup_error_count", "mean_ms", "min_ms", "max_ms", "p50_ms", "p90_ms", "p95_ms", + "p99_ms", "request_count_mean", "bytes_received_mean", "failed_requests_total", + "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", + "first_error" + ] + var mode: BenchmarkMode = .legacy var transport: BenchmarkTransport = .simulated var scenario: BenchmarkScenario = .cold @@ -154,6 +165,9 @@ struct BenchmarkCommand { guard parts.count == 2, !parts[0].isEmpty else { throw BenchmarkError.invalidArgument("--annotation expects key=value, got \(raw)") } + guard !Self.reservedRowKeys.contains(parts[0]) else { + throw BenchmarkError.invalidArgument("--annotation key \(parts[0]) is a reserved row field") + } command.annotations[parts[0]] = parts[1] default: throw BenchmarkError.invalidArgument("unknown flag \(flag)") diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift index 27998038ff..4e5721046a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -27,13 +27,13 @@ extension SimulatedTransportURLProtocol { guard let self else { return } if error != nil { - Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol(self, didFailWithError: error ?? URLError(.unknown)) return } guard let httpResponse = response as? HTTPURLResponse else { - Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol( self, didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(url.host ?? "")") @@ -41,7 +41,7 @@ extension SimulatedTransportURLProtocol { return } - Self.record(.success( + self.recordTerminal(.success( url: url, iteration: iteration, statusCode: httpResponse.statusCode, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift index 02251be87e..22fdb4acfd 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -39,8 +39,26 @@ final class SimulatedTransportURLProtocol: URLProtocol { private let deliveryQueue = DispatchQueue(label: "com.revenuecat.benchmark.transport.request") private var pendingWorkItems: [DispatchWorkItem] = [] var passthroughTask: URLSessionDataTask? + /// Captured at `startLoading` so a cancellation can still record this request's terminal + /// event (URLSession may cancel before any scheduled delivery block runs). + private var requestContext: (url: URL, iteration: Int, startedAt: DispatchTime)? + private var hasRecordedTerminalEvent = false let stateLock = NSLock() + /// Records the request's single terminal event. Every started request must record exactly + /// once (that's what balances the in-flight counter), whether it completes, fails, or is + /// cancelled mid-delivery. + func recordTerminal(_ event: TransportEvent) { + let isFirst = self.stateLock.withLock { () -> Bool in + guard !self.hasRecordedTerminalEvent else { return false } + self.hasRecordedTerminalEvent = true + return true + } + if isFirst { + Self.record(event) + } + } + static func install(server: FixtureServer, profile: NetworkProfile, loss: LossModel, seed: UInt64) { self.lock.withLock { self.mode = .simulated(server: server, profile: profile, loss: loss) @@ -138,6 +156,9 @@ final class SimulatedTransportURLProtocol: URLProtocol { Self.inFlightCount += 1 return Self.iterationIndex } + self.stateLock.withLock { + self.requestContext = (url, iteration, started) + } switch mode { case let .simulated(server, profile, loss): self.startSimulated( @@ -154,13 +175,20 @@ final class SimulatedTransportURLProtocol: URLProtocol { } override func stopLoading() { - self.stateLock.withLock { + let context = self.stateLock.withLock { () -> (url: URL, iteration: Int, startedAt: DispatchTime)? in for item in self.pendingWorkItems { item.cancel() } self.pendingWorkItems = [] self.passthroughTask?.cancel() self.passthroughTask = nil + return self.requestContext + } + // Cancellation can kill the scheduled delivery blocks before they record; backfill the + // terminal event so the in-flight counter always balances and cancelled requests show + // up as failures instead of disappearing. + if let context { + self.recordTerminal(.failure(url: context.url, iteration: context.iteration, startedAt: context.startedAt)) } } @@ -219,7 +247,7 @@ private extension SimulatedTransportURLProtocol { let rtoMs = max(1_000, rttMs * 2) self.schedule(afterMs: rtoMs) { [weak self] in guard let self else { return } - Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol(self, didFailWithError: URLError(.timedOut)) } } @@ -239,7 +267,7 @@ private extension SimulatedTransportURLProtocol { httpVersion: "HTTP/1.1", headerFields: response.headers ) else { - Self.record(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) self.client?.urlProtocol( self, didFailWithError: BenchmarkError.invalidFixture("Could not create fixture HTTP response") @@ -273,7 +301,7 @@ private extension SimulatedTransportURLProtocol { self.schedule(afterMs: deliveryOffsetMs) { [weak self] in guard let self else { return } - Self.record(.success( + self.recordTerminal(.success( url: url, iteration: iteration, statusCode: response.statusCode, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index 185fb1b768..421f68e3e4 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -70,6 +70,14 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertThrowsError(try BenchmarkCommand.parse(["--loss-percent", "101"])) } + func testParseRejectsReservedAnnotationKeys() { + // Annotations overwriting identity or metric fields would let a row lie about what + // was measured. + for key in ["mode", "p50_ms", "post_warmup_error_count"] { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--annotation", "\(key)=x"]), key) + } + } + // MARK: - Transport func testTransportDefaultsToSimulated() throws { From 2fdc3c7567445d228840593132542bf3fb451b39 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 09:14:42 +0200 Subject: [PATCH 17/34] other(benchmarks): document live inline blob delivery The stress project now publishes one workflow; khepri inlines its blob in the config container, so the full config-to-blob-store chain runs live with no CDN fetches yet. CDN fan-out starts once enough content is published to stop inlining. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index a34bf49138..11b4d990fb 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -170,9 +170,13 @@ run 2026-07-09 from a residential connection: Live observations (as of this run): -- The project's `/v1/config` response currently drives **no blob downloads** (2 requests - total: config + offerings), so live config cost is one extra API round trip. As the backend - starts serving workflow/paywall blobs for this project, live runs pick that up automatically. +- The project currently publishes **one workflow**, and khepri delivers its blob **inline in + the RC-Container** (config response ~3.9KB vs ~1.2KB of config JSON), so cold launches make + 2 requests and no CDN fetches. The full chain still runs: inline extraction -> blob store -> + offerings delivery gated on the prefetched workflow -> warm 204 with the blob proof. The + **CDN download path** (`config.revenuecat-static.com` + failover) only engages once enough + workflows/paywalls are published that the response stops inlining; live runs pick that up + automatically with no harness changes. - **Warm config launches cost almost nothing extra** (142ms vs 135ms p50): offerings delivery does not wait for the config revalidation when the topics are already on disk, so the 204 trails after delivery, exactly like production. Cold config pays the full extra round trip From 5d0ed6dce815e3d7a0374b20960729de72bd5f20 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 09:30:17 +0200 Subject: [PATCH 18/34] feat(benchmarks): swappable live target project PROJECT_ID= on the matrix resolves the project's key via mafdet (test-store app preferred); the binary takes --project-id, labels every live row with it, requires it alongside a custom --api-key, and compare.py keys on it, so results from different projects can never be compared as equivalents. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Tests/Benchmarks/SDKConfigBenchmark/README.md | 12 +++++++++ .../Sources/BenchmarkCommand.swift | 20 ++++++++++++-- .../Sources/BenchmarkMetrics.swift | 4 +++ .../Tests/BenchmarkCommandTests.swift | 24 ++++++++++++----- .../Benchmarks/SDKConfigBenchmark/compare.py | 2 +- .../SDKConfigBenchmark/run-matrix.sh | 26 ++++++++++++++++++- 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 9b12b85520..029710d965 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -31,6 +31,18 @@ bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl # TRANSPORT=live bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > live.jsonl ``` +Live runs default to the pinned stress-test project. To measure a different project, pass its +id; the key is resolved via the mafdet CLI (test-store app preferred) and every row is labeled +with `project_id`, so results from different projects can never be compared as equivalents: + +```sh +PROJECT_ID= TRANSPORT=live \ + bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > other-project.jsonl +``` + +(The target project's keyed app needs at least one package with a product attached, or the +offerings fetch fails with a configuration error.) + Compare two runs (e.g. baseline branch vs candidate branch): ```sh diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index d48811a4b1..b2f2b6fa85 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -47,6 +47,7 @@ enum BenchmarkTransport: String, CaseIterable { /// has no products registered, which makes `OfferingsManager` fail with a configuration error. enum BenchmarkProject { + static let projectID = "5f07e7e3" static let dashboardURL = "https://app.revenuecat.com/projects/5f07e7e3" static let testStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" static let appStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" @@ -77,7 +78,7 @@ struct BenchmarkCommand { "post_warmup_error_count", "mean_ms", "min_ms", "max_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "request_count_mean", "bytes_received_mean", "failed_requests_total", "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", - "first_error" + "first_error", "project_id" ] var mode: BenchmarkMode = .legacy @@ -92,6 +93,9 @@ struct BenchmarkCommand { var seed: UInt64 = 42 var appUserID: String = "benchmark-user" var apiKey: String = BenchmarkCommand.defaultSimulatedAPIKey + /// Which RevenueCat project a live run measures; labels the row so results from different + /// projects can never be compared as equivalents. Nil for simulated runs. + var projectID: String? /// Extra key=value pairs echoed verbatim into the JSONL row (e.g. sdk_commit=abc123). var annotations: [String: String] = [:] @@ -159,6 +163,8 @@ struct BenchmarkCommand { command.appUserID = try value(for: flag) case "--api-key": command.apiKey = try value(for: flag) + case "--project-id": + command.projectID = try value(for: flag) case "--annotation": let raw = try value(for: flag) let parts = raw.split(separator: "=", maxSplits: 1).map(String.init) @@ -190,7 +196,12 @@ struct BenchmarkCommand { /// forced kill-switch 4xx) cannot apply there, and the API key defaults to the pinned /// stress-test project. private mutating func validateAndDefaultTransport() throws { - guard self.transport == .live else { return } + guard self.transport == .live else { + guard self.projectID == nil else { + throw BenchmarkError.invalidArgument("--project-id only applies to --transport live") + } + return + } guard self.lossPercent == 0 else { throw BenchmarkError.invalidArgument("--loss-percent requires --transport simulated") @@ -208,6 +219,11 @@ struct BenchmarkCommand { if self.apiKey == Self.defaultSimulatedAPIKey { self.apiKey = BenchmarkProject.testStoreAPIKey + self.projectID = self.projectID ?? BenchmarkProject.projectID + } else if self.projectID == nil { + // A custom key without a project label would let rows from different projects + // collide in comparisons. + throw BenchmarkError.invalidArgument("--api-key with --transport live also requires --project-id") } // Fixture-size knobs don't shape live payloads (the pinned project's real content diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index facfc779c0..f1bdae8992 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -106,6 +106,10 @@ struct BenchmarkMetrics { row += Self.aggregates(of: measured) + if let projectID = command.projectID { + row["project_id"] = projectID + } + if let firstError = self.errorsByIteration.min(by: { $0.key < $1.key }) { row["first_error"] = "iteration \(firstError.key): \(firstError.value)" } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index 421f68e3e4..21d2fa21ff 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -91,6 +91,24 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertEqual(command.apiKey, BenchmarkProject.testStoreAPIKey) } + func testLiveTransportLabelsThePinnedProjectByDefault() throws { + XCTAssertEqual(try BenchmarkCommand.parse(["--transport", "live"]).projectID, BenchmarkProject.projectID) + } + + func testLiveTransportWithCustomKeyRequiresProjectID() throws { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--api-key", "appl_other"])) + + let labeled = try BenchmarkCommand.parse( + ["--transport", "live", "--api-key", "appl_other", "--project-id", "abc123"] + ) + XCTAssertEqual(labeled.projectID, "abc123") + XCTAssertEqual(labeled.apiKey, "appl_other") + } + + func testSimulatedTransportRejectsProjectID() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--project-id", "abc123"])) + } + func testLiveTransportZeroesFixtureSizeKnobs() throws { let command = try BenchmarkCommand.parse(["--transport", "live", "--paywalls", "500", "--workflows", "500"]) @@ -99,12 +117,6 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertEqual(command.workflowCount, 0) } - func testLiveTransportKeepsExplicitAPIKey() throws { - let command = try BenchmarkCommand.parse(["--transport", "live", "--api-key", "appl_other"]) - - XCTAssertEqual(command.apiKey, "appl_other") - } - func testLiveTransportRejectsSimulationOnlyKnobs() { XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--loss-percent", "10"])) XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--profile", "lte"])) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py index 52bfdef20b..c26eaa68a5 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/compare.py +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -14,7 +14,7 @@ KEY_FIELDS = ("mode", "transport", "scenario", "profile", "loss_percent", - "paywalls", "workflows", "seed", "iterations", "warmup_discarded") + "paywalls", "workflows", "seed", "iterations", "warmup_discarded", "project_id") METRICS = ("p50_ms", "p95_ms", "request_count_mean", "bytes_received_mean") diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index f4fcfc9657..bac12181f7 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -11,6 +11,9 @@ # TRANSPORT=live # hit the real backend (pinned stress-test project) instead of # # the simulated transport; forces ideal profile, no loss, and # # drops the kill-switch mode (cannot force 4xx on production) +# PROJECT_ID= # live only: measure a different RevenueCat project; the key is +# # resolved via mafdet (test-store app preferred) and rows are +# # labeled so cross-project comparisons never mix # MODES="legacy config config-killswitch" # SCENARIOS="cold warm" # PROFILES="ideal lte" @@ -62,6 +65,26 @@ fi SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" FAILED_ROWS=0 +PROJECT_ARGS=() +if [[ "$TRANSPORT" == "live" && -n "${PROJECT_ID:-}" ]]; then + if ! command -v mafdet >/dev/null; then + echo "PROJECT_ID requires the mafdet CLI to resolve the project's API key" >&2 + exit 1 + fi + RESOLVED_KEY="$(mafdet app api-keys --project-id "$PROJECT_ID" 2>/dev/null | python3 -c ' +import json, sys +keys = json.load(sys.stdin) +keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") +print(keys[0]["key"] if keys else "") +')" + if [[ -z "$RESOLVED_KEY" ]]; then + echo "Could not resolve an API key for project $PROJECT_ID via mafdet" >&2 + exit 1 + fi + echo "Live target: project $PROJECT_ID (key resolved via mafdet)" >&2 + PROJECT_ARGS=(--api-key "$RESOLVED_KEY" --project-id "$PROJECT_ID") +fi + run_row() { local mode="$1" scenario="$2" profile="$3" loss="$4" echo "Running transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 @@ -76,7 +99,8 @@ run_row() { --paywalls "$PAYWALLS" \ --workflows "$WORKFLOWS" \ --seed "$SEED" \ - --annotation "sdk_commit=$SDK_COMMIT"; then + --annotation "sdk_commit=$SDK_COMMIT" \ + ${PROJECT_ARGS[@]+"${PROJECT_ARGS[@]}"}; then echo "Row FAILED (transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%)" >&2 FAILED_ROWS=$((FAILED_ROWS + 1)) fi From 9597957d180fa32886cf83b3f411ea61b1acd5f0 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 09:54:33 +0200 Subject: [PATCH 19/34] other(benchmarks): resolve live API keys at run time instead of in source Live runs read the key from --api-key or SDK_CONFIG_BENCHMARK_API_KEY, and run-matrix.sh resolves it via mafdet for the (default or PROJECT_ID-swapped) target project. No keys live in the repository. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 9 ++--- Tests/Benchmarks/SDKConfigBenchmark/README.md | 10 +++--- .../Sources/BenchmarkCommand.swift | 33 +++++++++++++------ .../Tests/BenchmarkCommandTests.swift | 19 +++++++---- .../SDKConfigBenchmark/run-matrix.sh | 6 ++-- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index 11b4d990fb..d91e6bfea1 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -43,10 +43,11 @@ everything else is production code. - **`--transport simulated`** (default): the in-process network model below. Deterministic, seedable, and the only way to measure loss, degraded profiles, and the forced kill-switch 4xx. Use it for controlled A/B comparisons of SDK behavior and for CI regression tracking. -- **`--transport live`**: real requests against production, pinned to the prepared stress-test - project [`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3) ("Stress Test Config - Endpoint"; keys hardcoded in `BenchmarkProject`, Test Store key by default because the - project's packages live on its Test Store app). Requests flow through a recording +- **`--transport live`**: real requests against production, defaulting to the prepared + stress-test project [`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3) ("Stress Test + Config Endpoint"). No keys live in source: `run-matrix.sh` resolves the project's key via + mafdet (Test Store app preferred, since the project's packages live there); direct binary + runs take `--api-key` or `SDK_CONFIG_BENCHMARK_API_KEY`. Requests flow through a recording passthrough, so live rows carry the same per-request metrics: API traffic re-issues through a single-connection pool mirroring `HTTPClient`, blob traffic through a default pool mirroring the production blob downloader. Use it for real-world numbers (CDN, TLS, actual backend diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 029710d965..fde1a0638e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -8,11 +8,13 @@ Two transports: - **`--transport simulated`** (default): in-process fixtures with seeded network profiles (ideal/wifi/lte), packet-loss modeling, and a forced kill-switch 4xx. Deterministic and CI-friendly. -- **`--transport live`**: real requests against the production backend, pinned to the +- **`--transport live`**: real requests against the production backend, defaulting to the prepared stress-test project - ([`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3), hardcoded key). Same recorded - per-request metrics; real CDN/TLS/latency behavior. Kill-switch, profiles, and loss are - simulated-only (you cannot force a 4xx or packet loss on production). + ([`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3)). No keys live in source: the + matrix script resolves the key via mafdet; direct binary runs take `--api-key` or the + `SDK_CONFIG_BENCHMARK_API_KEY` environment variable. Same recorded per-request metrics; real + CDN/TLS/latency behavior. Kill-switch, profiles, and loss are simulated-only (you cannot + force a 4xx or packet loss on production). See `CONFIG_ENDPOINT_BENCHMARKS.md` for methodology, scenario definitions, sample numbers, and known limitations. diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index b2f2b6fa85..36ff1f4e4b 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -41,16 +41,16 @@ enum BenchmarkTransport: String, CaseIterable { /// The canonical RevenueCat project live runs measure against: /// https://app.revenuecat.com/projects/5f07e7e3 ("Stress Test Config Endpoint"), prepared with -/// a large number of paywalls and workflows. Keys are the project's client-side public SDK -/// keys, hardcoded on purpose so every live run measures the same content. The Test Store key -/// is the default because the project's packages live on its Test Store app; the App Store app -/// has no products registered, which makes `OfferingsManager` fail with a configuration error. +/// a large number of paywalls and workflows. No keys live in source: live runs read the key +/// from `--api-key`, the `SDK_CONFIG_BENCHMARK_API_KEY` environment variable, or (via +/// `run-matrix.sh`) mafdet resolution. Use the project's Test Store app key: its packages live +/// there; the App Store app has no products registered, which makes `OfferingsManager` fail +/// with a configuration error. enum BenchmarkProject { static let projectID = "5f07e7e3" static let dashboardURL = "https://app.revenuecat.com/projects/5f07e7e3" - static let testStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" - static let appStoreAPIKey = "REDACTED_RESOLVED_VIA_MAFDET" + static let apiKeyEnvironmentVariable = "SDK_CONFIG_BENCHMARK_API_KEY" } @@ -100,7 +100,10 @@ struct BenchmarkCommand { var annotations: [String: String] = [:] // swiftlint:disable:next cyclomatic_complexity function_body_length - static func parse(_ args: [String]) throws -> BenchmarkCommand { + static func parse( + _ args: [String], + environment: [String: String] = ProcessInfo.processInfo.environment + ) throws -> BenchmarkCommand { var command = BenchmarkCommand() var index = 0 @@ -187,7 +190,7 @@ struct BenchmarkCommand { ) } - try command.validateAndDefaultTransport() + try command.validateAndDefaultTransport(environment: environment) return command } @@ -195,7 +198,7 @@ struct BenchmarkCommand { /// Live runs hit the real backend: the knobs that shape the simulated network (and the /// forced kill-switch 4xx) cannot apply there, and the API key defaults to the pinned /// stress-test project. - private mutating func validateAndDefaultTransport() throws { + private mutating func validateAndDefaultTransport(environment: [String: String]) throws { guard self.transport == .live else { guard self.projectID == nil else { throw BenchmarkError.invalidArgument("--project-id only applies to --transport live") @@ -218,7 +221,17 @@ struct BenchmarkCommand { } if self.apiKey == Self.defaultSimulatedAPIKey { - self.apiKey = BenchmarkProject.testStoreAPIKey + // No keys live in source; resolve from the environment (run-matrix.sh populates + // it via mafdet) and assume the pinned project unless labeled otherwise. + guard let environmentKey = environment[BenchmarkProject.apiKeyEnvironmentVariable], + !environmentKey.isEmpty else { + throw BenchmarkError.invalidArgument( + "live runs need an API key: pass --api-key with --project-id, set " + + "\(BenchmarkProject.apiKeyEnvironmentVariable), or use run-matrix.sh " + + "(resolves it via mafdet)" + ) + } + self.apiKey = environmentKey self.projectID = self.projectID ?? BenchmarkProject.projectID } else if self.projectID == nil { // A custom key without a project label would let rows from different projects diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index 21d2fa21ff..d0f2a9ebb1 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -84,15 +84,19 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertEqual(try BenchmarkCommand.parse([]).transport, .simulated) } - func testLiveTransportDefaultsToPinnedProjectAPIKey() throws { - let command = try BenchmarkCommand.parse(["--transport", "live"]) + func testLiveTransportResolvesKeyFromEnvironment() throws { + let command = try BenchmarkCommand.parse( + ["--transport", "live"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + ) XCTAssertEqual(command.transport, .live) - XCTAssertEqual(command.apiKey, BenchmarkProject.testStoreAPIKey) + XCTAssertEqual(command.apiKey, "test_fromEnv") + XCTAssertEqual(command.projectID, BenchmarkProject.projectID) } - func testLiveTransportLabelsThePinnedProjectByDefault() throws { - XCTAssertEqual(try BenchmarkCommand.parse(["--transport", "live"]).projectID, BenchmarkProject.projectID) + func testLiveTransportWithoutAnyKeyFails() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live"], environment: [:])) } func testLiveTransportWithCustomKeyRequiresProjectID() throws { @@ -110,7 +114,10 @@ final class BenchmarkCommandTests: BenchmarkTestCase { } func testLiveTransportZeroesFixtureSizeKnobs() throws { - let command = try BenchmarkCommand.parse(["--transport", "live", "--paywalls", "500", "--workflows", "500"]) + let command = try BenchmarkCommand.parse( + ["--transport", "live", "--paywalls", "500", "--workflows", "500"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + ) // Live payloads come from the pinned project, so fixture sizes must not label the row. XCTAssertEqual(command.paywallCount, 0) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index bac12181f7..312a5ffbf7 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -66,9 +66,11 @@ SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" FAILED_ROWS=0 PROJECT_ARGS=() -if [[ "$TRANSPORT" == "live" && -n "${PROJECT_ID:-}" ]]; then +if [[ "$TRANSPORT" == "live" ]]; then + # No keys live in source: resolve the target project's key at run time. + PROJECT_ID="${PROJECT_ID:-5f07e7e3}" if ! command -v mafdet >/dev/null; then - echo "PROJECT_ID requires the mafdet CLI to resolve the project's API key" >&2 + echo "live runs resolve the project API key via the mafdet CLI; install it or run the binary directly with --api-key" >&2 exit 1 fi RESOLVED_KEY="$(mafdet app api-keys --project-id "$PROJECT_ID" 2>/dev/null | python3 -c ' From 49d9ffb88854fe2a5123199e0a0e4291ad69d5f2 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 10:19:04 +0200 Subject: [PATCH 20/34] feat(benchmarks): add app-host launch measurement app A minimal iOS app that links the stock RevenueCat/RevenueCatUI products and measures a real Purchases.configure launch end to end (configure, first customer info, offerings, paywall appeared), exposing the sample as JSON in an accessibility element for an XCUITest runner to collect. API key and app user ID arrive via the launch environment; --wipe-state deletes all SDK disk state for true cold launches. Own Tuist project, local/CI only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmarkApp/Project.swift | 47 ++++ .../App/BenchmarkAppMain.swift | 203 ++++++++++++++++++ .../SDKConfigBenchmarkApp/App/Info.plist | 24 +++ .../App/LaunchMeasurement.swift | 60 ++++++ Workspace.swift | 3 +- 5 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 Projects/SDKConfigBenchmarkApp/Project.swift create mode 100644 Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift create mode 100644 Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist create mode 100644 Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift diff --git a/Projects/SDKConfigBenchmarkApp/Project.swift b/Projects/SDKConfigBenchmarkApp/Project.swift new file mode 100644 index 0000000000..e91304936d --- /dev/null +++ b/Projects/SDKConfigBenchmarkApp/Project.swift @@ -0,0 +1,47 @@ +import ProjectDescription +import ProjectDescriptionHelpers + +// App-host tier of the SDK config benchmark: a minimal iOS app that links the stock +// RevenueCat/RevenueCatUI products (unlike Projects/SDKConfigBenchmark, which compiles the +// SDK sources directly) and measures a real `Purchases.configure` launch end to end. +// Local/CI only; driven by Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh. +// +// The legacy-vs-config variant switch is NOT set here: the SPM-built RevenueCat reads +// `SWIFT_ACTIVE_COMPILATION_CONDITIONS` from Local.xcconfig (see Package.swift), which the +// runner script rewrites per variant. + +let project = Project( + name: "SDKConfigBenchmarkApp", + organizationName: .revenueCatOrgName, + packages: .projectPackages, + settings: .appProject, + targets: [ + .target( + name: "SDKConfigBenchmarkApp", + destinations: [.iPhone], + product: .app, + bundleId: "com.revenuecat.SDKConfigBenchmarkApp", + deploymentTargets: .iOS("16.0"), + infoPlist: "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist", + sources: [ + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/**/*.swift" + ], + dependencies: [ + .revenueCat, + .revenueCatUI + ], + settings: .appTarget(including: ([:] as SettingsDictionary).appendingTuistSwiftConditions()) + ) + ], + schemes: [ + .scheme( + name: "SDKConfigBenchmarkApp", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmarkApp"]), + runAction: .runAction( + configuration: "Debug", + executable: "SDKConfigBenchmarkApp" + ) + ) + ] +) diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift new file mode 100644 index 0000000000..500c213dcb --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift @@ -0,0 +1,203 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// BenchmarkAppMain.swift +// +// Created by Facundo Menzella on 9/7/26. + +import RevenueCat +import RevenueCatUI +import SwiftUI + +/// Measures one real SDK launch end to end: configure, first customer info, offerings, +/// paywall appeared. The XCUITest runner relaunches this app once per iteration (a true +/// process cold start), injecting the API key and user via the launch environment: +/// +/// - `BENCH_API_KEY`: RevenueCat public API key (required; never committed to source) +/// - `BENCH_APP_USER_ID`: app user ID for this launch +/// - `--wipe-state` argument: delete all SDK disk state before configuring (cold launch) +@main +struct SDKConfigBenchmarkApp: App { + + private let clock: LaunchClock + @StateObject private var model: LaunchModel + + init() { + let clock = LaunchClock() + self.clock = clock + + if CommandLine.arguments.contains("--wipe-state") { + Self.wipeSDKState() + } + + let environment = ProcessInfo.processInfo.environment + let model = LaunchModel(clock: clock) + self._model = StateObject(wrappedValue: model) + + guard let apiKey = environment["BENCH_API_KEY"], !apiKey.isEmpty else { + model.fail("BENCH_API_KEY missing from launch environment") + return + } + + Purchases.logLevel = .warn + Purchases.configure( + with: .builder(withAPIKey: apiKey) + .with(appUserID: environment["BENCH_APP_USER_ID"]) + .build() + ) + model.recordConfigured() + model.startObserving() + } + + var body: some Scene { + WindowGroup { + LaunchView(model: self.model) + } + } + + /// Deletes every place the SDK persists state so the next configure is a true cold start. + private static func wipeSDKState() { + let fileManager = FileManager.default + for directory in [FileManager.SearchPathDirectory.cachesDirectory, .applicationSupportDirectory] { + guard let root = fileManager.urls(for: directory, in: .userDomainMask).first, + let contents = try? fileManager.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + else { continue } + for url in contents { + try? fileManager.removeItem(at: url) + } + } + + if let bundleID = Bundle.main.bundleIdentifier { + UserDefaults.standard.removePersistentDomain(forName: bundleID) + } + // The SDK's own suite (UserDefaults.revenueCatSuiteName). + UserDefaults(suiteName: "com.revenuecat.user_defaults")? + .removePersistentDomain(forName: "com.revenuecat.user_defaults") + } + +} + +/// Drives the measured phases after configure and publishes the finished sample. +@MainActor +final class LaunchModel: ObservableObject { + + @Published private(set) var offering: Offering? + @Published private(set) var finishedSampleJSON: String? + + private let clock: LaunchClock + private var sample = LaunchSample() + private var paywallAppeared = false + + init(clock: LaunchClock) { + self.clock = clock + } + + nonisolated func fail(_ message: String) { + Task { @MainActor in + self.sample.error = message + self.finish() + } + } + + nonisolated func recordConfigured() { + let elapsed = self.clock.elapsedMs() + Task { @MainActor in + self.sample.configuredMs = elapsed + } + } + + nonisolated func startObserving() { + Task { @MainActor in + await self.observeLaunch() + } + } + + private func observeLaunch() async { + async let customerInfo: Void = self.awaitFirstCustomerInfo() + + do { + let offerings = try await Purchases.shared.offerings() + self.sample.offeringsMs = self.clock.elapsedMs() + guard let current = offerings.current else { + throw NSError( + domain: "SDKConfigBenchmarkApp", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "project has no current offering"] + ) + } + self.offering = current + } catch { + self.sample.error = String(describing: error) + self.finish() + } + + await customerInfo + self.finishIfComplete() + } + + private func awaitFirstCustomerInfo() async { + for await _ in Purchases.shared.customerInfoStream { + self.sample.customerInfoMs = self.clock.elapsedMs() + break + } + } + + func recordPaywallAppeared() { + guard !self.paywallAppeared else { return } + self.paywallAppeared = true + self.sample.paywallAppearedMs = self.clock.elapsedMs() + self.finishIfComplete() + } + + private func finishIfComplete() { + guard self.finishedSampleJSON == nil else { return } + let done = self.sample.customerInfoMs != nil + && self.sample.offeringsMs != nil + && self.sample.paywallAppearedMs != nil + if done { + self.finish() + } + } + + private func finish() { + guard self.finishedSampleJSON == nil else { return } + let json = self.sample.jsonString() + self.finishedSampleJSON = json + // Also visible in the unified log for smoke testing without the XCUITest runner. + NSLog("BENCH_SAMPLE %@", json) + } + +} + +struct LaunchView: View { + + @ObservedObject var model: LaunchModel + + var body: some View { + ZStack { + if let offering = self.model.offering { + PaywallView(offering: offering) + .onAppear { + self.model.recordPaywallAppeared() + } + } else { + ProgressView() + } + + if let json = self.model.finishedSampleJSON { + // The runner reads the sample from this element's label. + Text(json) + .font(.system(size: 4)) + .opacity(0.02) + .accessibilityIdentifier("benchmark-result") + } + } + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist b/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist new file mode 100644 index 0000000000..9736be2280 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist @@ -0,0 +1,24 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + UILaunchScreen + + + diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift new file mode 100644 index 0000000000..9ceb1fe9b7 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift @@ -0,0 +1,60 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// LaunchMeasurement.swift +// +// Created by Facundo Menzella on 9/7/26. + +import Foundation + +/// One app launch's phase timings, measured from process start (`LaunchClock` creation in +/// the `App` initializer). Encoded as JSON into the `benchmark-result` accessibility element +/// so the XCUITest runner can collect it; also compiled into the benchmark unit-test target. +struct LaunchSample: Codable, Equatable { + + /// Milliseconds until `Purchases.configure` returned. + var configuredMs: Double? + /// Milliseconds until the first `CustomerInfo` was delivered. + var customerInfoMs: Double? + /// Milliseconds until `getOfferings` completed. + var offeringsMs: Double? + /// Milliseconds until the rendered `PaywallView` appeared. + var paywallAppearedMs: Double? + /// Non-nil when the launch could not complete; the runner fails loudly on it. + var error: String? + + func jsonString() -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(self), + let json = String(data: data, encoding: .utf8) else { + return "{\"error\":\"encoding-failed\"}" + } + return json + } + + static func decode(from json: String) -> LaunchSample? { + guard let data = json.data(using: .utf8) else { return nil } + return try? JSONDecoder().decode(LaunchSample.self, from: data) + } + +} + +/// Monotonic clock anchored at creation. Created as the first stored property of the `App` +/// type so elapsed values approximate time-since-process-start for the phases we control. +final class LaunchClock: Sendable { + + private let start = DispatchTime.now() + + func elapsedMs() -> Double { + let nanos = DispatchTime.now().uptimeNanoseconds - self.start.uptimeNanoseconds + return Double(nanos) / 1_000_000 + } + +} diff --git a/Workspace.swift b/Workspace.swift index 6924a7b772..4acda88bdf 100644 --- a/Workspace.swift +++ b/Workspace.swift @@ -13,7 +13,8 @@ var projects: [Path] = [ "./Projects/PaywallValidationTester", "./Projects/BinarySizeTest", "./Projects/RCTTester", - "./Projects/SDKConfigBenchmark" + "./Projects/SDKConfigBenchmark", + "./Projects/SDKConfigBenchmarkApp" ] // These projects depend on external packages (Nimble, SnapshotTesting, OHHTTPStubs, GoogleMobileAds). From 9d22dedb48bf30cfff095c333905d33d9857c8c8 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 10:26:22 +0200 Subject: [PATCH 21/34] feat(benchmarks): add app-launch xcuitest runner Relaunches the benchmark app once per iteration (true process cold starts), collects each launch's LaunchSample, and aggregates one compare.py-compatible JSONL row per scenario (cold and warm), printed as BENCHMARK_ROW and attached to the test result. AppLaunchMetrics is shared into the macOS benchmark unit test target so the aggregation (key fields, nearest-rank percentiles, error accounting) is covered without a simulator. Tests skip without BENCH_API_KEY and BENCH_MODE_LABEL so bare scheme runs never emit mislabeled rows. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmark/Project.swift | 6 +- Projects/SDKConfigBenchmarkApp/Project.swift | 19 +++ .../Tests/AppLaunchMetricsTests.swift | 116 +++++++++++++ .../UITests/AppLaunchBenchmarkUITests.swift | 152 ++++++++++++++++++ .../UITests/AppLaunchMetrics.swift | 126 +++++++++++++++ 5 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift create mode 100644 Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift create mode 100644 Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift diff --git a/Projects/SDKConfigBenchmark/Project.swift b/Projects/SDKConfigBenchmark/Project.swift index c74165b1c2..112b148e48 100644 --- a/Projects/SDKConfigBenchmark/Project.swift +++ b/Projects/SDKConfigBenchmark/Project.swift @@ -73,7 +73,11 @@ let project = Project( deploymentTargets: .macOS("13.0"), infoPlist: .default, sources: [ - "../../Tests/Benchmarks/SDKConfigBenchmark/Tests/**/*.swift" + "../../Tests/Benchmarks/SDKConfigBenchmark/Tests/**/*.swift", + // App-host tier helpers, shared into this target so their aggregation + // logic is unit-testable without booting a simulator. + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift", + "../../Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift" ], dependencies: [ .target(name: "SDKConfigBenchmarkCore") diff --git a/Projects/SDKConfigBenchmarkApp/Project.swift b/Projects/SDKConfigBenchmarkApp/Project.swift index e91304936d..476eb2fad7 100644 --- a/Projects/SDKConfigBenchmarkApp/Project.swift +++ b/Projects/SDKConfigBenchmarkApp/Project.swift @@ -31,6 +31,24 @@ let project = Project( .revenueCatUI ], settings: .appTarget(including: ([:] as SettingsDictionary).appendingTuistSwiftConditions()) + ), + .target( + name: "SDKConfigBenchmarkAppUITests", + destinations: [.iPhone], + product: .uiTests, + bundleId: "com.revenuecat.SDKConfigBenchmarkAppUITests", + deploymentTargets: .iOS("16.0"), + infoPlist: .default, + sources: [ + "../../Tests/TestingApps/SDKConfigBenchmarkApp/UITests/**/*.swift", + // Shared with the app target: the runner decodes the same LaunchSample + // the app encodes. + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkApp") + ], + settings: .appTarget ) ], schemes: [ @@ -38,6 +56,7 @@ let project = Project( name: "SDKConfigBenchmarkApp", shared: true, buildAction: .buildAction(targets: ["SDKConfigBenchmarkApp"]), + testAction: .targets(["SDKConfigBenchmarkAppUITests"]), runAction: .runAction( configuration: "Debug", executable: "SDKConfigBenchmarkApp" diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift new file mode 100644 index 0000000000..45d4c04a60 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -0,0 +1,116 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class AppLaunchMetricsTests: BenchmarkTestCase { + + private static func sample(_ total: Double) -> LaunchSample { + return LaunchSample( + configuredMs: total * 0.1, + customerInfoMs: total * 0.4, + offeringsMs: total * 0.8, + paywallAppearedMs: total, + error: nil + ) + } + + private func decodedRow( + warmupDiscarded: Int = 1, + samples: [LaunchSample?] + ) throws -> [String: Any] { + let row = AppLaunchMetrics.row( + mode: "app-launch-config", + scenario: "cold", + profile: "simulator", + projectID: "5f07e7e3", + warmupDiscarded: warmupDiscarded, + samples: samples + ) + let object = try JSONSerialization.jsonObject(with: Data(row.utf8)) + return try XCTUnwrap(object as? [String: Any]) + } + + func testPercentileMatchesCLIBenchmarkFormula() { + let sorted: [Double] = (1...10).map(Double.init) + for percentile in [50, 90, 95, 99] { + XCTAssertEqual( + AppLaunchMetrics.percentile(percentile, of: sorted), + BenchmarkMetrics.percentile(percentile, of: sorted), + "p\(percentile)" + ) + } + } + + func testRowCarriesEveryComparisonKeyField() throws { + let row = try self.decodedRow(samples: [Self.sample(100), Self.sample(200)]) + + XCTAssertEqual(row["mode"] as? String, "app-launch-config") + XCTAssertEqual(row["transport"] as? String, "live") + XCTAssertEqual(row["scenario"] as? String, "cold") + XCTAssertEqual(row["profile"] as? String, "simulator") + XCTAssertEqual(row["loss_percent"] as? Int, 0) + XCTAssertEqual(row["paywalls"] as? Int, 0) + XCTAssertEqual(row["workflows"] as? Int, 0) + XCTAssertEqual(row["seed"] as? Int, 0) + XCTAssertEqual(row["iterations"] as? Int, 2) + XCTAssertEqual(row["warmup_discarded"] as? Int, 1) + XCTAssertEqual(row["project_id"] as? String, "5f07e7e3") + } + + func testRowStatisticsUsePaywallAppearedOfPostWarmupSamples() throws { + // Warmup discards by index: the 500ms first launch never enters the stats. + let row = try self.decodedRow( + samples: [Self.sample(500), Self.sample(100), Self.sample(300), Self.sample(200)] + ) + + XCTAssertEqual(row["measured_iterations"] as? Int, 3) + XCTAssertEqual(row["mean_ms"] as? Double, 200) + XCTAssertEqual(row["min_ms"] as? Double, 100) + XCTAssertEqual(row["max_ms"] as? Double, 300) + XCTAssertEqual(row["p50_ms"] as? Double, 200) + XCTAssertEqual(row["p95_ms"] as? Double, 300) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 200) + XCTAssertEqual(row["configured_ms_mean"] as? Double, 20) + XCTAssertEqual(row["customer_info_ms_mean"] as? Double, 80) + XCTAssertEqual(row["offerings_ms_mean"] as? Double, 160) + } + + func testIncompleteOrFailedSamplesCountAsErrorsAndNeverEnterStats() throws { + var failed = Self.sample(50) + failed.error = "BENCH_API_KEY missing" + var incomplete = Self.sample(75) + incomplete.paywallAppearedMs = nil + + let row = try self.decodedRow( + warmupDiscarded: 0, + samples: [Self.sample(100), failed, incomplete, nil] + ) + + XCTAssertEqual(row["iterations"] as? Int, 4) + XCTAssertEqual(row["measured_iterations"] as? Int, 1) + XCTAssertEqual(row["error_count"] as? Int, 3) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 3) + XCTAssertEqual(row["mean_ms"] as? Double, 100) + let firstError = try XCTUnwrap(row["first_error"] as? String) + XCTAssertTrue(firstError.contains("BENCH_API_KEY missing")) + } + + func testWarmupWindowErrorsAreCountedButNotPostWarmup() throws { + var failed = Self.sample(50) + failed.error = "boom" + + let row = try self.decodedRow(warmupDiscarded: 1, samples: [failed, Self.sample(100)]) + + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + XCTAssertEqual(row["measured_iterations"] as? Int, 1) + } + + func testLaunchSampleJSONRoundTrip() throws { + let sample = Self.sample(123.456) + let decoded = try XCTUnwrap(LaunchSample.decode(from: sample.jsonString())) + XCTAssertEqual(decoded, sample) + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift new file mode 100644 index 0000000000..1b61413d13 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift @@ -0,0 +1,152 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// AppLaunchBenchmarkUITests.swift +// +// Created by Facundo Menzella on 9/7/26. + +import XCTest + +/// Relaunches the benchmark app once per iteration (each launch is a real process cold +/// start), reads the `LaunchSample` the app exposes, and aggregates one JSONL row per +/// scenario, printed as `BENCHMARK_ROW: {...}` for `run-app-launch.sh` to collect. +/// +/// Configured via the runner's environment (pass with a `TEST_RUNNER_` prefix through +/// xcodebuild): `BENCH_API_KEY` (required), `BENCH_MODE_LABEL` (required; identifies the +/// SDK build variant, e.g. `app-launch-legacy`), `BENCH_ITERATIONS`, `BENCH_WARMUP`, +/// `BENCH_PROJECT_ID`. Both tests skip when the required variables are absent, so running +/// the scheme without the script does not produce mislabeled rows. +final class AppLaunchBenchmarkUITests: XCTestCase { + + private struct Configuration { + let apiKey: String + let modeLabel: String + let iterations: Int + let warmup: Int + let projectID: String + let runNonce: String + } + + override func setUpWithError() throws { + try super.setUpWithError() + self.continueAfterFailure = false + } + + func testColdLaunches() throws { + let configuration = try self.configuration() + + // A fresh user and wiped state every launch: each iteration is a true cold start. + let samples = (0.. Configuration { + let environment = ProcessInfo.processInfo.environment + guard let apiKey = environment["BENCH_API_KEY"], !apiKey.isEmpty else { + throw XCTSkip("BENCH_API_KEY not set; run via run-app-launch.sh") + } + guard let modeLabel = environment["BENCH_MODE_LABEL"], !modeLabel.isEmpty else { + throw XCTSkip("BENCH_MODE_LABEL not set; the row cannot name its SDK build variant") + } + + let iterations = environment["BENCH_ITERATIONS"].flatMap(Int.init) ?? 10 + let warmup = environment["BENCH_WARMUP"].flatMap(Int.init) ?? 2 + guard iterations > 0, warmup >= 0, warmup < iterations else { + throw XCTSkip("BENCH_WARMUP (\(warmup)) must be below BENCH_ITERATIONS (\(iterations))") + } + + return Configuration( + apiKey: apiKey, + modeLabel: modeLabel, + iterations: iterations, + warmup: warmup, + projectID: environment["BENCH_PROJECT_ID"] ?? "5f07e7e3", + runNonce: UUID().uuidString.lowercased().prefix(8).description + ) + } + + private func launchAndCollect( + configuration: Configuration, + appUserID: String, + wipeState: Bool + ) -> LaunchSample? { + let app = XCUIApplication() + app.launchEnvironment = [ + "BENCH_API_KEY": configuration.apiKey, + "BENCH_APP_USER_ID": appUserID + ] + app.launchArguments = wipeState ? ["--wipe-state"] : [] + app.launch() + defer { app.terminate() } + + let result = app.staticTexts["benchmark-result"] + guard result.waitForExistence(timeout: 90) else { + return nil + } + return LaunchSample.decode(from: result.label) + } + + private func report(samples: [LaunchSample?], scenario: String, configuration: Configuration) { + let row = AppLaunchMetrics.row( + mode: configuration.modeLabel, + scenario: scenario, + profile: Self.profile, + projectID: configuration.projectID, + warmupDiscarded: configuration.warmup, + samples: samples + ) + + // The script greps this prefix out of the xcodebuild log to build the JSONL file. + print("BENCHMARK_ROW: \(row)") + let attachment = XCTAttachment(string: row) + attachment.name = "benchmark-row-\(scenario)" + attachment.lifetime = .keepAlways + self.add(attachment) + + let postWarmupErrors = samples.enumerated() + .filter { $0.offset >= configuration.warmup } + .compactMap { AppLaunchMetrics.errorMessage(for: $0.element) } + XCTAssertTrue( + postWarmupErrors.isEmpty, + "\(postWarmupErrors.count) measured launch(es) failed; first: \(postWarmupErrors.first ?? "")" + ) + } + + private static var profile: String { + #if targetEnvironment(simulator) + return "simulator" + #else + return "device" + #endif + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift new file mode 100644 index 0000000000..adaafea18f --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift @@ -0,0 +1,126 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// AppLaunchMetrics.swift +// +// Created by Facundo Menzella on 9/7/26. + +import Foundation + +/// Aggregates per-launch `LaunchSample`s into one JSONL row compatible with the CLI +/// benchmark's `compare.py` (same key fields, same nearest-rank percentiles). Compiled into +/// both the XCUITest runner and the benchmark unit-test target. +enum AppLaunchMetrics { + + /// One row per (mode, scenario) configuration. `samples` has one entry per launch in + /// order; `nil` means the launch produced no readable result. Launches with an `error` + /// or a missing phase count as errors and never enter the statistics. The first + /// `warmupDiscarded` entries are excluded from statistics by index. + // swiftlint:disable:next function_body_length + static func row( + mode: String, + scenario: String, + profile: String, + projectID: String, + warmupDiscarded: Int, + samples: [LaunchSample?] + ) -> String { + let indexed = samples.enumerated() + let errors: [(index: Int, message: String)] = indexed.compactMap { index, sample in + if let message = Self.errorMessage(for: sample) { + return (index, message) + } + return nil + } + let measured: [LaunchSample] = indexed.compactMap { index, sample in + guard index >= warmupDiscarded, + let sample, + Self.errorMessage(for: sample) == nil else { + return nil + } + return sample + } + + var row: [String: Any] = [ + "mode": mode, + "transport": "live", + "scenario": scenario, + "profile": profile, + "loss_percent": 0, + "paywalls": 0, + "workflows": 0, + "seed": 0, + "iterations": samples.count, + "warmup_discarded": warmupDiscarded, + "project_id": projectID, + "measured_iterations": measured.count, + "error_count": errors.count, + "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count + ] + + let totals = measured.compactMap(\.paywallAppearedMs).sorted() + if !totals.isEmpty { + row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) + row["min_ms"] = Self.rounded(totals[0]) + row["max_ms"] = Self.rounded(totals[totals.count - 1]) + for percentile in [50, 90, 95, 99] { + row["p\(percentile)_ms"] = Self.rounded(Self.percentile(percentile, of: totals)) + } + } + + for (key, values) in [ + ("configured_ms_mean", measured.compactMap(\.configuredMs)), + ("customer_info_ms_mean", measured.compactMap(\.customerInfoMs)), + ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), + ("paywall_appeared_ms_mean", measured.compactMap(\.paywallAppearedMs)) + ] where !values.isEmpty { + row[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) + } + + if let firstError = errors.first { + row["first_error"] = "iteration \(firstError.index): \(firstError.message)" + } + + guard let data = try? JSONSerialization.data(withJSONObject: row, options: [.sortedKeys]), + let json = String(data: data, encoding: .utf8) else { + return "{\"error\":\"row-encoding-failed\"}" + } + return json + } + + /// Non-nil when a launch cannot contribute to the statistics: it failed outright, + /// produced nothing, or is missing a phase (a partially measured launch could otherwise + /// look faster than a complete one). + static func errorMessage(for sample: LaunchSample?) -> String? { + guard let sample else { return "no benchmark-result element" } + if let error = sample.error { return error } + for (phase, value) in [ + ("configuredMs", sample.configuredMs), + ("customerInfoMs", sample.customerInfoMs), + ("offeringsMs", sample.offeringsMs), + ("paywallAppearedMs", sample.paywallAppearedMs) + ] where value == nil { + return "sample missing \(phase)" + } + return nil + } + + /// Nearest-rank percentile over an already-sorted array; same formula as the CLI + /// benchmark's `BenchmarkMetrics.percentile` so rows are comparable. + static func percentile(_ percentile: Int, of sorted: [Double]) -> Double { + precondition(!sorted.isEmpty) + let rank = Int((Double(percentile) / 100 * Double(sorted.count)).rounded(.up)) + return sorted[max(0, min(sorted.count - 1, rank - 1))] + } + + private static func rounded(_ value: Double) -> Double { + return (value * 1_000).rounded() / 1_000 + } + +} From 3bd6c215d8d5ae258c2398c0beb94d6fe48df52e Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 10:32:31 +0200 Subject: [PATCH 22/34] feat(benchmarks): add app-launch variant runner and docs run-app-launch.sh builds SDKConfigBenchmarkApp once per SDK variant by rewriting SWIFT_ACTIVE_COMPILATION_CONDITIONS in Local.xcconfig (restored on exit), forces an SPM manifest re-evaluation so a cached manifest can never build the wrong variant, verifies from the build log that the flag reached the SDK compile, runs the XCUITest loop against the live project, and greps the BENCHMARK_ROW lines into JSONL. Nonzero exit when any variant fails, mislabels, or produces fewer rows than expected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 51 ++++- Tests/Benchmarks/SDKConfigBenchmark/README.md | 21 +++ .../SDKConfigBenchmarkApp/run-app-launch.sh | 178 ++++++++++++++++++ 3 files changed, 247 insertions(+), 3 deletions(-) create mode 100755 Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index d91e6bfea1..134f3444da 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -214,11 +214,56 @@ Live observations (as of this run): "stripped offerings" lever below is the measurement that matters for deciding whether to remove paywall data from `/offerings`. +## App-launch tier + +The CLI measures the manager-level fetch flows in isolation on macOS. The app-launch tier +(`Tests/TestingApps/SDKConfigBenchmarkApp/`) measures what the CLI cannot: a real iOS app +that calls `Purchases.configure` like a customer app does, on iOS hardware or simulator, +including process cold start, customer-info and offerings contention on the launch path, and +the time until a `PaywallView` actually appears on screen. + +- The app links the **stock** `RevenueCat`/`RevenueCatUI` products (no benchmark hooks, no + simulated transport), so it exercises exactly the binary that ships. It is live-only, + against the same pinned stress-test project. +- An XCUITest terminates and relaunches the app once per iteration, so every sample is a + true process cold start. `cold` wipes all SDK disk state and uses a fresh app user per + launch; `warm` primes once and relaunches with retained disk state and the same user. +- The legacy/config variant switch is the `ENABLE_REMOTE_CONFIG` compile condition, which + the SPM-built SDK reads from `Local.xcconfig` (`Package.swift` parses it). + `run-app-launch.sh` rewrites that file per variant, forces a manifest re-evaluation, and + verifies from the build log that the flag actually reached the SDK compile. +- Phases per launch: `configured` (configure returned), `customer_info` (first CustomerInfo + delivered), `offerings` (getOfferings completed), `paywall_appeared` (PaywallView onAppear). + The row's percentile statistics are over `paywall_appeared`; a launch missing any phase + counts as an error and fails the run. +- Rows carry `mode` = `app-launch-legacy` / `app-launch-config` and `profile` = `simulator` / + `device`, so `compare.py` never mixes app-tier rows with CLI rows or simulator with device. + +Run it with `bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl`. +Because there is no network control, treat app-tier numbers as end-to-end monitoring on a +real network, not as a controlled A/B; the CLI's simulated transport remains the tool for +isolating variables. + +First sample (simulator, live network against the stress project with 2 inline workflow +blobs published, 4 iterations, 1 warmup discarded; ms to paywall appeared): + +| mode | scenario | p50 | offerings mean | customer info mean | configure mean | +|---|---|---|---|---|---| +| app-launch-legacy | cold | 1209 | 1160 | 694 | 18 | +| app-launch-config | cold | 1365 | 1342 | 835 | 22 | +| app-launch-legacy | warm | 915 | 829 | 635 | 18 | +| app-launch-config | warm | 1050 | 1068 | 792 | 19 | + +The config variant's cold overhead end to end (~13% here) is far below the CLI fixture's +worst case, because this project's published blobs arrive inline in the container instead +of as 100 prefetch-gated CDN downloads. Iteration counts this small are smoke-level; use +`ITERATIONS=25` or more for numbers worth acting on. + ## Known limitations -- **macOS CPU, not device CPU.** Decode and CPU costs are measured on desktop hardware. The - comparison between modes is valid; absolute numbers on device will differ. If absolute - launch-time claims are needed, wrap the same runner in an iOS app target. +- **macOS CPU, not device CPU** for the CLI tier. Decode and CPU costs are measured on + desktop hardware. The comparison between modes is valid; absolute numbers on device will + differ. The app-launch tier above covers absolute, on-device launch timings. - **Loss is approximated**, see the network model section above. - **Fixture payloads are simpler than production.** Paywall component trees are minimal (one stack + one text component per paywall). Byte counts scale with `--paywalls`/`--workflows`, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index fde1a0638e..c0e1e4fa27 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -86,6 +86,27 @@ Every run isolates the SDK's disk caches (ETags, offerings, remote config, blobs fresh temporary directory that is removed on exit, so runs never touch the real user Library and concurrent runs cannot corrupt each other. +## App-launch tier (simulator or device) + +The CLI above measures the manager-level flows in isolation. The app-host tier measures what +a customer actually experiences: a real iOS app (`SDKConfigBenchmarkApp`, linking the stock +`RevenueCat`/`RevenueCatUI` products) that runs `Purchases.configure` and reports the time to +configure, first customer info, offerings, and paywall appeared. An XCUITest relaunches the +app once per iteration, so every sample is a true process cold start; the runner script +builds the app twice, once per SDK variant, by rewriting `SWIFT_ACTIVE_COMPILATION_CONDITIONS` +in `Local.xcconfig` (restored on exit): + +```sh +bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py app-launch.jsonl +``` + +Rows are `compare.py`-compatible, with `mode` = `app-launch-legacy` / `app-launch-config` and +`profile` = `simulator` / `device`. Knobs: `ITERATIONS`, `WARMUP`, `PROJECT_ID`, +`SDK_CONFIG_BENCHMARK_API_KEY` (skips mafdet), and `DESTINATION` (pass +`DESTINATION="platform=iOS,id="` to measure a physical device's real radio). Live only: +there is no simulated transport, no kill-switch mode, and no loss model in this tier. + ## Unit tests ```sh diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh new file mode 100755 index 0000000000..53aab6120c --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# App-host tier of the SDK config benchmark: builds SDKConfigBenchmarkApp twice (legacy SDK +# vs config-endpoint SDK) and drives the XCUITest relaunch loop against the live stress-test +# project, emitting one JSONL row per (variant, scenario) to stdout: +# +# bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl +# +# The legacy/config switch is the ENABLE_REMOTE_CONFIG compile condition, which the SPM-built +# RevenueCat reads from the SWIFT_ACTIVE_COMPILATION_CONDITIONS line of CI.xcconfig (if +# present) or Local.xcconfig (see Package.swift). This script rewrites that file per variant, +# restores it on exit, and builds each variant into its own derived data path. +# +# Environment overrides: +# VARIANTS="legacy config" # SDK build variants to measure +# ITERATIONS=10 WARMUP=2 # app relaunches per scenario / discarded from statistics +# PROJECT_ID= # target RevenueCat project; key resolved via mafdet +# # (test-store app preferred), rows labeled with the id +# SDK_CONFIG_BENCHMARK_API_KEY= # skip mafdet and use this key directly +# DESTINATION="platform=iOS Simulator,id=" # default: first available iPhone +# # simulator; pass a device destination to measure real radio + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DERIVED_DATA_ROOT="${DERIVED_DATA_ROOT:-$REPO_ROOT/.build/sdk-config-benchmark-app}" +VARIANTS="${VARIANTS:-legacy config}" +ITERATIONS="${ITERATIONS:-10}" +WARMUP="${WARMUP:-2}" +PROJECT_ID="${PROJECT_ID:-5f07e7e3}" + +# Package.swift prefers CI.xcconfig over Local.xcconfig, so the variant switch must edit +# whichever file actually wins. +if [[ -f "$REPO_ROOT/CI.xcconfig" ]]; then + XCCONFIG="$REPO_ROOT/CI.xcconfig" +else + XCCONFIG="$REPO_ROOT/Local.xcconfig" +fi + +XCCONFIG_BACKUP="" +XCCONFIG_EXISTED=0 +if [[ -f "$XCCONFIG" ]]; then + XCCONFIG_EXISTED=1 + XCCONFIG_BACKUP="$(mktemp)" + cp "$XCCONFIG" "$XCCONFIG_BACKUP" +fi + +restore_xcconfig() { + if [[ "$XCCONFIG_EXISTED" == "1" ]]; then + cp "$XCCONFIG_BACKUP" "$XCCONFIG" + rm -f "$XCCONFIG_BACKUP" + else + rm -f "$XCCONFIG" + fi + touch "$REPO_ROOT/Package.swift" +} +trap restore_xcconfig EXIT + +# Resolve the API key: env override, else mafdet (no keys live in source). +if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then + RESOLVED_KEY="$SDK_CONFIG_BENCHMARK_API_KEY" +else + if ! command -v mafdet >/dev/null; then + echo "set SDK_CONFIG_BENCHMARK_API_KEY or install the mafdet CLI to resolve the project key" >&2 + exit 1 + fi + RESOLVED_KEY="$(mafdet app api-keys --project-id "$PROJECT_ID" 2>/dev/null | python3 -c ' +import json, sys +keys = json.load(sys.stdin) +keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") +print(keys[0]["key"] if keys else "") +')" +fi +if [[ -z "$RESOLVED_KEY" ]]; then + echo "Could not resolve an API key for project $PROJECT_ID" >&2 + exit 1 +fi +echo "Live target: project $PROJECT_ID" >&2 + +if [[ -z "${DESTINATION:-}" ]]; then + UDID="$(xcrun simctl list devices available -j | python3 -c ' +import json, sys +devices = json.load(sys.stdin)["devices"] +iphones = [d for ds in devices.values() for d in ds if d["name"].startswith("iPhone")] +print(iphones[0]["udid"] if iphones else "") +')" + if [[ -z "$UDID" ]]; then + echo "No available iPhone simulator; pass DESTINATION explicitly" >&2 + exit 1 + fi + DESTINATION="platform=iOS Simulator,id=$UDID" +fi +echo "Destination: $DESTINATION" >&2 + +echo "Generating workspace..." >&2 +(cd "$REPO_ROOT" && tuist generate --no-open SDKConfigBenchmarkApp SDKConfigBenchmarkAppUITests >&2) + +set_swift_conditions() { + local conditions="$1" + if [[ -f "$XCCONFIG" ]] && grep -q '^SWIFT_ACTIVE_COMPILATION_CONDITIONS' "$XCCONFIG"; then + sed -i '' "s/^SWIFT_ACTIVE_COMPILATION_CONDITIONS.*$/SWIFT_ACTIVE_COMPILATION_CONDITIONS = \$(inherited) $conditions/" "$XCCONFIG" + else + # shellcheck disable=SC2016 # $(inherited) is an xcconfig token, not shell + printf '\nSWIFT_ACTIVE_COMPILATION_CONDITIONS = $(inherited) %s\n' "$conditions" >> "$XCCONFIG" + fi + # SwiftPM caches manifest evaluations; the xcconfig is not part of the cache key, so + # force a re-evaluation or a stale flag set could silently build the wrong variant. + touch "$REPO_ROOT/Package.swift" +} + +FAILED_VARIANTS=0 +TOTAL_ROWS=0 + +for variant in $VARIANTS; do + case "$variant" in + legacy) CONDITIONS="" ;; + config) CONDITIONS="ENABLE_REMOTE_CONFIG" ;; + *) echo "Unknown variant $variant (expected legacy or config)" >&2; exit 1 ;; + esac + + echo "=== Variant $variant (conditions: '${CONDITIONS:-none}') ===" >&2 + set_swift_conditions "$CONDITIONS" + + LOG="$(mktemp)" + DERIVED_DATA="$DERIVED_DATA_ROOT/$variant" + if ! env TEST_RUNNER_BENCH_API_KEY="$RESOLVED_KEY" \ + TEST_RUNNER_BENCH_MODE_LABEL="app-launch-$variant" \ + TEST_RUNNER_BENCH_ITERATIONS="$ITERATIONS" \ + TEST_RUNNER_BENCH_WARMUP="$WARMUP" \ + TEST_RUNNER_BENCH_PROJECT_ID="$PROJECT_ID" \ + xcodebuild -workspace "$REPO_ROOT/RevenueCat-Tuist.xcworkspace" \ + -scheme SDKConfigBenchmarkApp \ + -destination "$DESTINATION" \ + -derivedDataPath "$DERIVED_DATA" \ + test > "$LOG" 2>&1; then + echo "Variant $variant FAILED; last log lines:" >&2 + tail -25 "$LOG" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + fi + + # Prove the variant flag reached the SDK compile (guards the manifest-cache hazard). + # Only checkable when this run actually compiled the SDK; cached builds get a warning. + if grep -q "SwiftCompile.*RevenueCat" "$LOG"; then + if [[ -n "$CONDITIONS" ]] && ! grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then + echo "Variant $variant compiled WITHOUT ENABLE_REMOTE_CONFIG; refusing its rows" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + rm -f "$LOG" + continue + fi + if [[ -z "$CONDITIONS" ]] && grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then + echo "Variant $variant compiled WITH ENABLE_REMOTE_CONFIG; refusing its rows" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + rm -f "$LOG" + continue + fi + else + echo "Warning: SDK not recompiled for variant $variant (cached build); flag not re-verified" >&2 + fi + + ROWS="$(grep -o 'BENCHMARK_ROW: {.*}' "$LOG" | sed 's/^BENCHMARK_ROW: //' | sort -u || true)" + ROW_COUNT=0 + if [[ -n "$ROWS" ]]; then + ROW_COUNT="$(printf '%s\n' "$ROWS" | wc -l | tr -d ' ')" + printf '%s\n' "$ROWS" + fi + TOTAL_ROWS=$((TOTAL_ROWS + ROW_COUNT)) + if [[ "$ROW_COUNT" -lt 2 ]]; then + echo "Variant $variant produced $ROW_COUNT row(s), expected 2 (cold + warm)" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + fi + rm -f "$LOG" +done + +if (( FAILED_VARIANTS > 0 )); then + echo "$FAILED_VARIANTS variant check(s) failed" >&2 + exit 1 +fi +echo "Done: $TOTAL_ROWS row(s)" >&2 From ee8c748a15b60c4a6825976b52e80e6cd1378bf5 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 10:56:19 +0200 Subject: [PATCH 23/34] feat(benchmarks): inject TUIST_BENCH_API_KEY into the app scheme run environment Lets the benchmark app be launched straight from Xcode: TUIST_BENCH_API_KEY= tuist generate SDKConfigBenchmarkApp. The key lands only in the generated (gitignored) scheme, never in source. Without it, a direct run still reports the missing key by design. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmarkApp/Project.swift | 16 +++++++++++++++- .../ProjectDescriptionHelpers/Environment.swift | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Projects/SDKConfigBenchmarkApp/Project.swift b/Projects/SDKConfigBenchmarkApp/Project.swift index 476eb2fad7..28fff5e453 100644 --- a/Projects/SDKConfigBenchmarkApp/Project.swift +++ b/Projects/SDKConfigBenchmarkApp/Project.swift @@ -10,6 +10,19 @@ import ProjectDescriptionHelpers // `SWIFT_ACTIVE_COMPILATION_CONDITIONS` from Local.xcconfig (see Package.swift), which the // runner script rewrites per variant. +// Injected into the scheme's run environment so the app can be launched straight from Xcode: +// TUIST_BENCH_API_KEY= tuist generate SDKConfigBenchmarkApp +// Without it, a direct run reports "BENCH_API_KEY missing" by design (keys never live in +// source or in committed scheme files). +let runEnvironment: [String: EnvironmentVariable] = { + var environment: [String: EnvironmentVariable] = [:] + if let apiKey = Environment.benchApiKey { + environment["BENCH_API_KEY"] = .environmentVariable(value: apiKey, isEnabled: true) + environment["BENCH_APP_USER_ID"] = .environmentVariable(value: "bench-xcode-run", isEnabled: true) + } + return environment +}() + let project = Project( name: "SDKConfigBenchmarkApp", organizationName: .revenueCatOrgName, @@ -59,7 +72,8 @@ let project = Project( testAction: .targets(["SDKConfigBenchmarkAppUITests"]), runAction: .runAction( configuration: "Debug", - executable: "SDKConfigBenchmarkApp" + executable: "SDKConfigBenchmarkApp", + arguments: .arguments(environmentVariables: runEnvironment) ) ) ] diff --git a/Tuist/ProjectDescriptionHelpers/Environment.swift b/Tuist/ProjectDescriptionHelpers/Environment.swift index 1dceb01bd1..2043f7b2e6 100644 --- a/Tuist/ProjectDescriptionHelpers/Environment.swift +++ b/Tuist/ProjectDescriptionHelpers/Environment.swift @@ -107,6 +107,19 @@ extension Environment { return value.isEmpty ? nil : value } + /// Returns the API key to inject into the SDKConfigBenchmarkApp scheme's run environment, + /// so the app can be launched directly from Xcode without the XCUITest runner. + /// + /// Example usage: + /// ```bash + /// TUIST_BENCH_API_KEY=$(mafdet app api-keys --project-id 5f07e7e3 | jq -r '.[] | select(.app_store_type == "test_store") | .key') \ + /// tuist generate SDKConfigBenchmarkApp + /// ``` + public static var benchApiKey: String? { + let value = ProcessInfo.processInfo.environment["TUIST_BENCH_API_KEY"] ?? "" + return value.isEmpty ? nil : value + } + /// Returns extra launch arguments to inject into scheme run actions, enabled by default. /// /// Example usage: From 42762f40ef8981a4d1bb6d0800316960465faead Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 11:40:29 +0200 Subject: [PATCH 24/34] feat(benchmarks): register blobs and config-path phases in both tiers Both tiers now record how many blobs each launch stored, split inline vs CDN-downloaded, with byte totals and size extremes (largest inline, smallest downloaded) that empirically bracket the backend's inline-size budget (currently ~3MB). The CLI attributes via its blob store plus transport events; the app observes the stock SDK's log stream with a parser pinned to RemoteConfigStrings by unit tests. The app additionally stamps config_persisted and last_blob_stored phases. Also applies three review findings: UITests build and run under Release (the rows claim shipping-SDK numbers), the headline percentile moves from the PaywallView wrapper's onAppear to the SDK's content-appeared event (paywall_impression, or workflows_step_started for workflow paywalls, via the internal events-listener SPI), and every measured launch now proves at runtime which SDK variant is in the binary (the config refresh log fires on every config-variant launch; rows whose configPathActive contradicts their label fail the run, cached builds included). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Projects/SDKConfigBenchmarkApp/Project.swift | 7 +- .../CONFIG_ENDPOINT_BENCHMARKS.md | 25 ++- Tests/Benchmarks/SDKConfigBenchmark/README.md | 8 + .../Sources/BenchmarkCommand.swift | 2 + .../Sources/BenchmarkMetrics.swift | 81 +++++++++- .../Sources/BenchmarkRunner.swift | 35 ++++- .../Sources/BenchmarkSDKStack.swift | 12 +- .../Tests/AppLaunchMetricsTests.swift | 123 +++++++++++++-- .../Tests/BenchmarkMetricsTests.swift | 61 +++++++- .../BenchmarkRunnerValidationTests.swift | 56 +++++++ .../App/BenchmarkAppMain.swift | 60 ++++++- .../App/LaunchMeasurement.swift | 147 +++++++++++++++++- .../UITests/AppLaunchBenchmarkUITests.swift | 20 ++- .../UITests/AppLaunchMetrics.swift | 24 ++- .../SDKConfigBenchmarkApp/run-app-launch.sh | 14 +- 15 files changed, 644 insertions(+), 31 deletions(-) diff --git a/Projects/SDKConfigBenchmarkApp/Project.swift b/Projects/SDKConfigBenchmarkApp/Project.swift index 28fff5e453..7b02bf043f 100644 --- a/Projects/SDKConfigBenchmarkApp/Project.swift +++ b/Projects/SDKConfigBenchmarkApp/Project.swift @@ -69,7 +69,12 @@ let project = Project( name: "SDKConfigBenchmarkApp", shared: true, buildAction: .buildAction(targets: ["SDKConfigBenchmarkApp"]), - testAction: .targets(["SDKConfigBenchmarkAppUITests"]), + // Release: the rows claim to measure the shipping SDK, so Debug-built + // (unoptimized, assertion-enabled) frameworks would misrepresent it. + testAction: .targets( + ["SDKConfigBenchmarkAppUITests"], + configuration: "Release" + ), runAction: .runAction( configuration: "Debug", executable: "SDKConfigBenchmarkApp", diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index 134f3444da..479eed78bf 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -233,12 +233,31 @@ the time until a `PaywallView` actually appears on screen. `run-app-launch.sh` rewrites that file per variant, forces a manifest re-evaluation, and verifies from the build log that the flag actually reached the SDK compile. - Phases per launch: `configured` (configure returned), `customer_info` (first CustomerInfo - delivered), `offerings` (getOfferings completed), `paywall_appeared` (PaywallView onAppear). - The row's percentile statistics are over `paywall_appeared`; a launch missing any phase - counts as an error and fails the run. + delivered), `offerings` (getOfferings completed), `paywall_appeared` (the PaywallView + wrapper mounted), and `paywall_impression` (the SDK tracked that paywall CONTENT actually + appeared: `paywall_impression` for classic paywalls, `workflows_step_started` for workflow + paywalls, observed via the SDK's internal events-listener SPI). The row's percentile + statistics are over `paywall_impression`; a launch missing any phase counts as an error and + fails the run. +- Config-path phases and blob registration, observed from the stock SDK's log stream (the + parser is pinned to `RemoteConfigStrings` by unit tests): `config_persisted_ms_mean`, + `last_blob_stored_ms_mean`, `blobs_inline_mean` vs `blobs_downloaded_mean`, `blob_bytes_mean`, + and the size extremes `max_inline_blob_bytes` / `min_downloaded_blob_bytes`. The extremes + bracket the backend's inline-size budget empirically (blobs under the budget, currently + ~3MB, should arrive inline; a small blob arriving via CDN is a regression signal). The CLI + tier registers the same fields via its blob store and transport events. +- Tests build and run under **Release** (the rows claim shipping-SDK numbers), and every + measured launch must prove at runtime which SDK variant is inside the binary: the config + path's refresh log fires on every config-variant launch, and the runner fails on any launch + whose `configPathActive` contradicts the row's label, cached builds included. - Rows carry `mode` = `app-launch-legacy` / `app-launch-config` and `profile` = `simulator` / `device`, so `compare.py` never mixes app-tier rows with CLI rows or simulator with device. +The goal of this tier is to run the real app N times, via any automation that can drive +`xcodebuild test` (locally, CI, or tooling like the baguette CLI), and get the same kind of +comparable, gateable JSONL the CLI benchmark produces, measuring both systems end to end +exactly as a customer app experiences them. + Run it with `bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl`. Because there is no network control, treat app-tier numbers as end-to-end monitoring on a real network, not as a controlled A/B; the CLI's simulated transport remains the tool for diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index c0e1e4fa27..58273ff0ce 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -107,6 +107,14 @@ Rows are `compare.py`-compatible, with `mode` = `app-launch-legacy` / `app-launc `DESTINATION="platform=iOS,id="` to measure a physical device's real radio). Live only: there is no simulated transport, no kill-switch mode, and no loss model in this tier. +Each launch also registers the config path when it runs: config persisted time, blob counts +split inline vs CDN-downloaded with byte totals, and size extremes that bracket the backend's +inline-size budget. The headline percentile is time to `paywall_impression` (content actually +on screen); tests run under Release and fail if a launch's observed SDK variant contradicts +the row's label. The intent is that any automation able to drive `xcodebuild test` (CI, the +baguette CLI, a cron box) can run the app N times and collect the same gateable JSONL as the +CLI matrix. + ## Unit tests ```sh diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 36ff1f4e4b..0a5eb8ccb9 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -78,6 +78,8 @@ struct BenchmarkCommand { "post_warmup_error_count", "mean_ms", "min_ms", "max_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "request_count_mean", "bytes_received_mean", "failed_requests_total", "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", + "blobs_inline_mean", "blobs_downloaded_mean", "blob_bytes_mean", "max_inline_blob_bytes", + "min_downloaded_blob_bytes", "config_persisted_ms_mean", "last_blob_stored_ms_mean", "first_error", "project_id" ] diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index f1bdae8992..80e0873e79 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -1,5 +1,68 @@ import Foundation +/// Blobs stored during one iteration, split by how they arrived: written from the config +/// container (inline) vs fetched from a blob source (downloaded). The size extremes bracket +/// the backend's inline-size budget empirically: the largest blob seen inline and the +/// smallest blob that needed a download. +struct BlobAccounting { + + let inlineCount: Int + let downloadedCount: Int + let totalBytes: Int + let maxInlineBytes: Int? + let minDownloadedBytes: Int? + + static let empty = BlobAccounting( + inlineCount: 0, + downloadedCount: 0, + totalBytes: 0, + maxInlineBytes: nil, + minDownloadedBytes: nil + ) + + /// Attributes newly stored refs: a new ref that has a successful blob request this + /// iteration was downloaded; any other new ref can only have come from the container. + init(newRefSizes: [String: Int], downloadedRefs: Set) { + var inlineCount = 0 + var downloadedCount = 0 + var totalBytes = 0 + var maxInlineBytes: Int? + var minDownloadedBytes: Int? + + for (ref, byteCount) in newRefSizes { + totalBytes += byteCount + if downloadedRefs.contains(ref) { + downloadedCount += 1 + minDownloadedBytes = min(minDownloadedBytes ?? .max, byteCount) + } else { + inlineCount += 1 + maxInlineBytes = max(maxInlineBytes ?? .min, byteCount) + } + } + + self.inlineCount = inlineCount + self.downloadedCount = downloadedCount + self.totalBytes = totalBytes + self.maxInlineBytes = maxInlineBytes + self.minDownloadedBytes = minDownloadedBytes + } + + private init( + inlineCount: Int, + downloadedCount: Int, + totalBytes: Int, + maxInlineBytes: Int?, + minDownloadedBytes: Int? + ) { + self.inlineCount = inlineCount + self.downloadedCount = downloadedCount + self.totalBytes = totalBytes + self.maxInlineBytes = maxInlineBytes + self.minDownloadedBytes = minDownloadedBytes + } + +} + /// One measured simulated launch, with phases attributed from the transport event log. struct IterationMeasurement { @@ -19,8 +82,10 @@ struct IterationMeasurement { let offeringsStatusCodes: [Int] let configStatusCodes: [Int] let blobRequestCount: Int + /// Blobs stored this iteration, attributed inline vs downloaded via the blob store. + let blobs: BlobAccounting - init(totalMs: Double, events: [TransportEvent]) { + init(totalMs: Double, events: [TransportEvent], blobAccounting: BlobAccounting = .empty) { self.totalMs = totalMs let byKind = Dictionary(grouping: events, by: \.kind) let offerings = byKind[.offerings] ?? [] @@ -36,6 +101,7 @@ struct IterationMeasurement { self.offeringsStatusCodes = offerings.map(\.statusCode) self.configStatusCodes = config.map(\.statusCode) self.blobRequestCount = blobs.count + self.blobs = blobAccounting } private static func span(of events: [TransportEvent]) -> Double? { @@ -140,11 +206,22 @@ struct BenchmarkMetrics { for (key, values) in [ ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), ("config_ms_mean", measured.compactMap(\.configMs)), - ("blob_ms_mean", measured.compactMap(\.blobMs)) + ("blob_ms_mean", measured.compactMap(\.blobMs)), + ("blobs_inline_mean", measured.map { Double($0.blobs.inlineCount) }), + ("blobs_downloaded_mean", measured.map { Double($0.blobs.downloadedCount) }), + ("blob_bytes_mean", measured.map { Double($0.blobs.totalBytes) }) ] where !values.isEmpty { aggregates[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) } + // Size extremes across the run bracket the backend's inline-size budget. + if let maxInline = measured.compactMap(\.blobs.maxInlineBytes).max() { + aggregates["max_inline_blob_bytes"] = maxInline + } + if let minDownloaded = measured.compactMap(\.blobs.minDownloadedBytes).min() { + aggregates["min_downloaded_blob_bytes"] = minDownloaded + } + return aggregates } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index d59f576dcd..b4be9d0a5c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -129,6 +129,7 @@ private extension BenchmarkRunner { stack.clearAllDiskState() } _ = SimulatedTransportURLProtocol.drainEvents() + let blobRefsBeforeLaunch = stack.blobStore?.cachedRefs() ?? [] let totalMs: Double do { @@ -147,7 +148,15 @@ private extension BenchmarkRunner { // AND the config request (when wired) has landed, then keep only this iteration's // events so stragglers from earlier launches can't inflate the row. let events = self.collectEvents(forIteration: iteration) - return IterationMeasurement(totalMs: totalMs, events: events) + return IterationMeasurement( + totalMs: totalMs, + events: events, + blobAccounting: Self.blobAccounting( + blobStore: stack.blobStore, + refsBeforeLaunch: blobRefsBeforeLaunch, + events: events + ) + ) } /// Drains transport events until quiescence, additionally waiting (bounded) for this @@ -217,6 +226,30 @@ private extension BenchmarkRunner { extension BenchmarkRunner { + /// Attributes this iteration's newly stored blobs (all disk reads happen after the timed + /// window): a new ref with a successful blob request was downloaded; any other new ref can + /// only have arrived inline in the config container. + static func blobAccounting( + blobStore: RemoteConfigBlobStoreType?, + refsBeforeLaunch: Set, + events: [TransportEvent] + ) -> BlobAccounting { + guard let blobStore else { return .empty } + + let newRefs = blobStore.cachedRefs().subtracting(refsBeforeLaunch) + guard !newRefs.isEmpty else { return .empty } + + let downloadedRefs = Set( + events + .filter { $0.kind == .blob && !$0.failed } + .map { ($0.path as NSString).lastPathComponent } + ) + let newRefSizes = newRefs.reduce(into: [String: Int]()) { sizes, ref in + sizes[ref] = blobStore.read(ref: ref)?.count ?? 0 + } + return BlobAccounting(newRefSizes: newRefSizes, downloadedRefs: downloadedRefs) + } + static func validateWarmMeasurement( _ measurement: IterationMeasurement, mode: BenchmarkMode, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift index 4dfb8d2052..74c3a114d2 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift @@ -12,6 +12,9 @@ final class BenchmarkSDKStack { let offeringsManager: OfferingsManager let remoteConfigManager: RemoteConfigManagerType? + /// The manager's content-addressed blob store, exposed so the runner can attribute + /// per-iteration blob storage (inline vs downloaded) and sizes. Nil in legacy mode. + let blobStore: RemoteConfigBlobStoreType? let httpClient: HTTPClient let deviceCache: DeviceCache @@ -55,10 +58,12 @@ final class BenchmarkSDKStack { let deviceCache = DeviceCache(systemInfo: systemInfo, userDefaults: userDefaults) self.deviceCache = deviceCache - let remoteConfigManager = mode.usesRemoteConfig + let remoteConfigStack = mode.usesRemoteConfig ? Self.makeRemoteConfigManager(backendConfiguration: backendConfiguration, appUserID: appUserID) : nil + let remoteConfigManager = remoteConfigStack?.manager self.remoteConfigManager = remoteConfigManager + self.blobStore = remoteConfigStack?.blobStore self.offeringsManager = OfferingsManager( deviceCache: deviceCache, @@ -109,7 +114,7 @@ private extension BenchmarkSDKStack { static func makeRemoteConfigManager( backendConfiguration: BackendConfiguration, appUserID: String - ) -> RemoteConfigManagerType { + ) -> (manager: RemoteConfigManagerType, blobStore: RemoteConfigBlobStoreType) { let diskCache = RemoteConfigDiskCache() let blobStore = RemoteConfigBlobStore() let sourceProvider = RemoteConfigSourceProvider(topicStore: diskCache) @@ -120,13 +125,14 @@ private extension BenchmarkSDKStack { session: SimulatedTransportURLProtocol.makeSession() ) ) - return RemoteConfigManager( + let manager = RemoteConfigManager( remoteConfigAPI: RemoteConfigAPI(backendConfig: backendConfiguration), diskCache: diskCache, blobStore: blobStore, blobFetcher: blobFetcher, currentUserProvider: BenchmarkCurrentUserProvider(appUserID: appUserID) ) + return (manager, blobStore) } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift index 45d4c04a60..8cc8d2ddac 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -5,13 +5,13 @@ import XCTest final class AppLaunchMetricsTests: BenchmarkTestCase { private static func sample(_ total: Double) -> LaunchSample { - return LaunchSample( - configuredMs: total * 0.1, - customerInfoMs: total * 0.4, - offeringsMs: total * 0.8, - paywallAppearedMs: total, - error: nil - ) + var sample = LaunchSample() + sample.configuredMs = total * 0.1 + sample.customerInfoMs = total * 0.4 + sample.offeringsMs = total * 0.8 + sample.paywallAppearedMs = total * 0.9 + sample.paywallImpressionMs = total + return sample } private func decodedRow( @@ -57,8 +57,9 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertEqual(row["project_id"] as? String, "5f07e7e3") } - func testRowStatisticsUsePaywallAppearedOfPostWarmupSamples() throws { - // Warmup discards by index: the 500ms first launch never enters the stats. + func testRowStatisticsUsePaywallImpressionOfPostWarmupSamples() throws { + // Warmup discards by index: the 500ms first launch never enters the stats. The + // headline percentiles use the impression mark (content appeared), not the wrapper. let row = try self.decodedRow( samples: [Self.sample(500), Self.sample(100), Self.sample(300), Self.sample(200)] ) @@ -70,12 +71,20 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertEqual(row["p50_ms"] as? Double, 200) XCTAssertEqual(row["p95_ms"] as? Double, 300) XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) - XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 200) + XCTAssertEqual(row["paywall_impression_ms_mean"] as? Double, 200) + XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 180) XCTAssertEqual(row["configured_ms_mean"] as? Double, 20) XCTAssertEqual(row["customer_info_ms_mean"] as? Double, 80) XCTAssertEqual(row["offerings_ms_mean"] as? Double, 160) } + func testSampleMissingImpressionCountsAsError() { + var missingImpression = Self.sample(100) + missingImpression.paywallImpressionMs = nil + + XCTAssertNotNil(AppLaunchMetrics.errorMessage(for: missingImpression)) + } + func testIncompleteOrFailedSamplesCountAsErrorsAndNeverEnterStats() throws { var failed = Self.sample(50) failed.error = "BENCH_API_KEY missing" @@ -108,9 +117,101 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { } func testLaunchSampleJSONRoundTrip() throws { - let sample = Self.sample(123.456) + var sample = Self.sample(123.456) + sample.configPersistedMs = 88.5 + sample.blobsInline = 2 + sample.blobsDownloaded = 1 + sample.blobBytes = 40_000 + sample.maxInlineBlobBytes = 30_000 + sample.minDownloadedBlobBytes = 10_000 let decoded = try XCTUnwrap(LaunchSample.decode(from: sample.jsonString())) XCTAssertEqual(decoded, sample) } + func testRowAggregatesBlobAndConfigPathFields() throws { + var first = Self.sample(100) + first.configPersistedMs = 80 + first.lastBlobStoredMs = 90 + first.blobsInline = 2 + first.blobsDownloaded = 1 + first.blobBytes = 30_000 + first.maxInlineBlobBytes = 20_000 + first.minDownloadedBlobBytes = 9_000 + + var second = Self.sample(200) + second.configPersistedMs = 120 + second.lastBlobStoredMs = 150 + second.blobsInline = 2 + second.blobsDownloaded = 3 + second.blobBytes = 50_000 + second.maxInlineBlobBytes = 26_000 + second.minDownloadedBlobBytes = 7_000 + + let row = try self.decodedRow(warmupDiscarded: 0, samples: [first, second]) + + XCTAssertEqual(row["config_persisted_ms_mean"] as? Double, 100) + XCTAssertEqual(row["last_blob_stored_ms_mean"] as? Double, 120) + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 2) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 2) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 40_000) + // Extremes across the run expose the backend's inline-size budget empirically. + XCTAssertEqual(row["max_inline_blob_bytes"] as? Int, 26_000) + XCTAssertEqual(row["min_downloaded_blob_bytes"] as? Int, 7_000) + } + + func testRowOmitsConfigPathFieldsWhenNeverObserved() throws { + // A legacy-variant run: no config persist, no blobs. Counts stay (provably zero); + // timing means and size extremes are omitted rather than invented. + let row = try self.decodedRow(warmupDiscarded: 0, samples: [Self.sample(100)]) + + XCTAssertNil(row["config_persisted_ms_mean"]) + XCTAssertNil(row["last_blob_stored_ms_mean"]) + XCTAssertNil(row["max_inline_blob_bytes"]) + XCTAssertNil(row["min_downloaded_blob_bytes"]) + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 0) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 0) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 0) + } + + // MARK: - Blob log parsing + + /// The app tier observes the config path through the stock SDK's log stream, so the + /// parser's phrases must track `RemoteConfigStrings` exactly. These build the real SDK + /// messages; if the log copy changes, this fails instead of the app silently reporting 0. + func testBlobLogParserMatchesRealSDKLogStrings() throws { + let url = try XCTUnwrap(URL(string: "https://config.revenuecat-static.com/abc")) + let downloaded = RemoteConfigStrings.storedBlob("ref-a", byteCount: 41_213, url).description + XCTAssertEqual( + BlobLogParser.blobEvent(from: downloaded), + BlobStorageEvent(source: .downloaded, byteCount: 41_213) + ) + + let inline = RemoteConfigStrings.storedInlineBlob("ref-b", byteCount: 512).description + XCTAssertEqual( + BlobLogParser.blobEvent(from: inline), + BlobStorageEvent(source: .inline, byteCount: 512) + ) + + let persisted = RemoteConfigStrings.persistedConfiguration( + domain: "app", activeTopicCount: 1, referencedBlobCount: 2 + ).description + XCTAssertTrue(BlobLogParser.isConfigPersisted(persisted)) + XCTAssertNil(BlobLogParser.blobEvent(from: persisted)) + + // Fires on every config-variant launch (cold and warm): the runtime variant proof. + let refreshing = RemoteConfigStrings.refreshing( + domain: "app", manifestPresent: false, isAppBackgrounded: false + ).description + XCTAssertTrue(BlobLogParser.isConfigRefreshStarted(refreshing)) + XCTAssertFalse(BlobLogParser.isConfigRefreshStarted(persisted)) + } + + func testBlobLogParserToleratesLoggerPrefixesAndIgnoresOtherMessages() { + let prefixed = "[Purchases] - VERBOSE: 😻 Stored inline remote config blob 'r' with 77 bytes." + XCTAssertEqual(BlobLogParser.blobEvent(from: prefixed)?.byteCount, 77) + + XCTAssertNil(BlobLogParser.blobEvent(from: "Prefetching 3 remote config blobs requested.")) + XCTAssertFalse(BlobLogParser.isConfigPersisted("Received remote config with 1 active topics.")) + } + } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift index d3629b423d..1a6ed6a889 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -4,8 +4,11 @@ import XCTest final class BenchmarkMetricsTests: BenchmarkTestCase { - private func measurement(totalMs: Double) -> IterationMeasurement { - return IterationMeasurement(totalMs: totalMs, events: []) + private func measurement( + totalMs: Double, + blobAccounting: BlobAccounting = .empty + ) -> IterationMeasurement { + return IterationMeasurement(totalMs: totalMs, events: [], blobAccounting: blobAccounting) } func testNearestRankPercentiles() { @@ -144,4 +147,58 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { XCTAssertEqual(measurement.offeringsMs ?? 0, 45, accuracy: 0.001) } + // MARK: - Blob accounting + + func testBlobAccountingAttributesInlineVersusDownloadedAndTracksExtremes() { + let accounting = BlobAccounting( + newRefSizes: ["inline-big": 30_000, "inline-small": 500, "cdn-a": 9_000, "cdn-b": 12_000], + downloadedRefs: ["cdn-a", "cdn-b", "cdn-never-stored"] + ) + + XCTAssertEqual(accounting.inlineCount, 2) + XCTAssertEqual(accounting.downloadedCount, 2) + XCTAssertEqual(accounting.totalBytes, 51_500) + // The extremes bracket the backend's inline-size budget. + XCTAssertEqual(accounting.maxInlineBytes, 30_000) + XCTAssertEqual(accounting.minDownloadedBytes, 9_000) + } + + func testJSONLRowCarriesBlobAccountingAggregates() throws { + var command = BenchmarkCommand() + command.iterations = 2 + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10, blobAccounting: BlobAccounting( + newRefSizes: ["i1": 1_000, "d1": 4_000], downloadedRefs: ["d1"] + )), iteration: 0) + metrics.record(self.measurement(totalMs: 20, blobAccounting: BlobAccounting( + newRefSizes: ["i2": 3_000, "d2": 2_000], downloadedRefs: ["d2"] + )), iteration: 1) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 1) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 1) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 5_000) + XCTAssertEqual(row["max_inline_blob_bytes"] as? Int, 3_000) + XCTAssertEqual(row["min_downloaded_blob_bytes"] as? Int, 2_000) + } + + func testJSONLRowOmitsBlobExtremesWhenNoBlobsWereStored() throws { + var command = BenchmarkCommand() + command.iterations = 1 + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10), iteration: 0) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 0) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 0) + XCTAssertNil(row["max_inline_blob_bytes"]) + XCTAssertNil(row["min_downloaded_blob_bytes"]) + } + } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift index d56fbe6eca..8f93281d19 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -96,4 +96,60 @@ final class BenchmarkRunnerValidationTests: BenchmarkTestCase { )) } + // MARK: - Blob accounting + + func testBlobAccountingCountsOnlyRefsStoredThisIterationAndAttributesByRequest() throws { + let store = InMemoryBlobStore(contents: [ + "pre-existing": Data(count: 100), + "inline-new": Data(count: 2_000), + "cdn-new": Data(count: 8_000) + ]) + let url = try XCTUnwrap(URL(string: "https://cdn.revenuecat.local/blobs/cdn-new")) + let events = [ + TransportEvent.success(url: url, iteration: 0, statusCode: 200, + bytesReceived: 8_000, startedAt: DispatchTime.now()) + ] + + let accounting = BenchmarkRunner.blobAccounting( + blobStore: store, + refsBeforeLaunch: ["pre-existing"], + events: events + ) + + XCTAssertEqual(accounting.inlineCount, 1) + XCTAssertEqual(accounting.downloadedCount, 1) + XCTAssertEqual(accounting.totalBytes, 10_000) + XCTAssertEqual(accounting.maxInlineBytes, 2_000) + XCTAssertEqual(accounting.minDownloadedBytes, 8_000) + } + + func testBlobAccountingIsEmptyForLegacyModeWithoutABlobStore() { + let accounting = BenchmarkRunner.blobAccounting(blobStore: nil, refsBeforeLaunch: [], events: []) + + XCTAssertEqual(accounting.inlineCount, 0) + XCTAssertEqual(accounting.downloadedCount, 0) + XCTAssertNil(accounting.maxInlineBytes) + XCTAssertNil(accounting.minDownloadedBytes) + } + +} + +private final class InMemoryBlobStore: RemoteConfigBlobStoreType { + + private var contents: [String: Data] + + init(contents: [String: Data]) { + self.contents = contents + } + + func contains(ref: String) -> Bool { return self.contents[ref] != nil } + func read(ref: String) -> Data? { return self.contents[ref] } + func write(ref: String, bytes: UnsafeRawBufferPointer) -> Bool { + self.contents[ref] = Data(bytes.bindMemory(to: UInt8.self)) + return true + } + func cachedRefs() -> Set { return Set(self.contents.keys) } + func retainOnly(_ refs: Set) { self.contents = self.contents.filter { refs.contains($0.key) } } + func clear() { self.contents = [:] } + } diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift index 500c213dcb..be7dc1505e 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift @@ -11,7 +11,7 @@ // // Created by Facundo Menzella on 9/7/26. -import RevenueCat +@_spi(Internal) import RevenueCat import RevenueCatUI import SwiftUI @@ -45,13 +45,24 @@ struct SDKConfigBenchmarkApp: App { return } - Purchases.logLevel = .warn + // The config path (config persisted, each blob stored inline vs downloaded) is only + // observable through the SDK's log stream, so run verbose and parse. The logging + // overhead is inside the measured window but identical across variants. + let observer = model.blobObserver + Purchases.logLevel = .verbose + Purchases.verboseLogHandler = { _, message, _, _, _ in + observer.ingest(message) + } Purchases.configure( with: .builder(withAPIKey: apiKey) .with(appUserID: environment["BENCH_APP_USER_ID"]) .build() ) model.recordConfigured() + // The impression event fires when the paywall CONTENT view appears (resolved paywall + // or explicit fallback), unlike the wrapper's onAppear, which can precede it while a + // loading state shows. It is the headline "user sees the paywall" mark. + Purchases.shared.eventsListener = model.impressionListener model.startObserving() } @@ -90,12 +101,22 @@ final class LaunchModel: ObservableObject { @Published private(set) var offering: Offering? @Published private(set) var finishedSampleJSON: String? + let blobObserver: BlobObserver + /// Retained here: the SDK does not keep the events listener alive. + private(set) var impressionListener: PaywallImpressionListener! + private let clock: LaunchClock private var sample = LaunchSample() private var paywallAppeared = false init(clock: LaunchClock) { self.clock = clock + self.blobObserver = BlobObserver(clock: clock) + self.impressionListener = PaywallImpressionListener(clock: clock) { [weak self] elapsedMs in + Task { @MainActor in + self?.recordPaywallImpression(elapsedMs: elapsedMs) + } + } } nonisolated func fail(_ message: String) { @@ -155,11 +176,18 @@ final class LaunchModel: ObservableObject { self.finishIfComplete() } + private func recordPaywallImpression(elapsedMs: Double) { + guard self.sample.paywallImpressionMs == nil else { return } + self.sample.paywallImpressionMs = elapsedMs + self.finishIfComplete() + } + private func finishIfComplete() { guard self.finishedSampleJSON == nil else { return } let done = self.sample.customerInfoMs != nil && self.sample.offeringsMs != nil && self.sample.paywallAppearedMs != nil + && self.sample.paywallImpressionMs != nil if done { self.finish() } @@ -167,6 +195,7 @@ final class LaunchModel: ObservableObject { private func finish() { guard self.finishedSampleJSON == nil else { return } + self.blobObserver.apply(to: &self.sample) let json = self.sample.jsonString() self.finishedSampleJSON = json // Also visible in the unified log for smoke testing without the XCUITest runner. @@ -175,6 +204,33 @@ final class LaunchModel: ObservableObject { } +/// Stamps the moment the SDK tracks that paywall content actually appeared (via the SDK's +/// internal events listener SPI), unlike the wrapper's `onAppear`, which can fire while a +/// loading state still shows. A classic paywall tracks `paywall_impression`; a workflow +/// paywall tracks `workflows_step_started` when its first step renders. +final class PaywallImpressionListener: EventsListener { + + private static let contentAppearedEventTypes: Set = [ + "paywall_impression", + "workflows_step_started" + ] + + private let clock: LaunchClock + private let onImpression: (Double) -> Void + + init(clock: LaunchClock, onImpression: @escaping (Double) -> Void) { + self.clock = clock + self.onImpression = onImpression + } + + func onEventTracked(_ event: [String: Any]) { + guard let type = event["type"] as? String, + Self.contentAppearedEventTypes.contains(type) else { return } + self.onImpression(self.clock.elapsedMs()) + } + +} + struct LaunchView: View { @ObservedObject var model: LaunchModel diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift index 9ceb1fe9b7..53124d546c 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift @@ -24,8 +24,28 @@ struct LaunchSample: Codable, Equatable { var customerInfoMs: Double? /// Milliseconds until `getOfferings` completed. var offeringsMs: Double? - /// Milliseconds until the rendered `PaywallView` appeared. + /// Milliseconds until the `PaywallView` wrapper mounted (may still be a loading state). var paywallAppearedMs: Double? + /// Milliseconds until the SDK tracked the paywall impression: the content view (resolved + /// paywall or explicit fallback) actually appeared. This is the headline number. + var paywallImpressionMs: Double? + /// Whether the config endpoint path ran this launch (its refresh was observed in the log). + /// Runtime proof of which SDK variant is actually inside the binary. + var configPathActive: Bool = false + /// Milliseconds until the config endpoint response was persisted (config variant only; + /// observed via the SDK's log stream, nil when the config path never ran). + var configPersistedMs: Double? + /// Milliseconds until the last blob landed in the store (nil when no blobs were stored). + var lastBlobStoredMs: Double? + /// Blobs written to the store this launch, split by how they arrived. + var blobsInline: Int = 0 + var blobsDownloaded: Int = 0 + /// Total bytes of blobs stored this launch. + var blobBytes: Int = 0 + /// Size extremes, for empirically checking the backend's inline-size budget (blobs under + /// the budget should arrive inline; a small blob arriving via CDN is a regression signal). + var maxInlineBlobBytes: Int? + var minDownloadedBlobBytes: Int? /// Non-nil when the launch could not complete; the runner fails loudly on it. var error: String? @@ -46,6 +66,131 @@ struct LaunchSample: Codable, Equatable { } +/// One blob landing in the SDK's content-addressed store, observed from the SDK's log stream. +struct BlobStorageEvent: Equatable { + + enum Source: String { + /// Delivered inside the config container response. + case inline + /// Fetched separately from a blob source (CDN). + case downloaded + } + + let source: Source + let byteCount: Int + +} + +/// Extracts config-path progress from the SDK's log messages (the only observable the stock +/// SDK offers an app). Substring-based so logger prefixes (level, emoji) don't matter; the +/// exact phrases are pinned to `RemoteConfigStrings` by unit tests in the benchmark suite. +enum BlobLogParser { + + private static let inlineMarker = "Stored inline remote config blob '" + private static let downloadedMarker = "Stored remote config blob '" + private static let configPersistedMarker = "Persisted remote config for domain '" + private static let configRefreshingMarker = "Refreshing remote config for domain '" + + static func blobEvent(from message: String) -> BlobStorageEvent? { + let source: BlobStorageEvent.Source + if message.contains(Self.inlineMarker) { + source = .inline + } else if message.contains(Self.downloadedMarker) { + source = .downloaded + } else { + return nil + } + + guard let byteCount = Self.integer(before: " bytes", in: message) else { return nil } + return BlobStorageEvent(source: source, byteCount: byteCount) + } + + static func isConfigPersisted(_ message: String) -> Bool { + return message.contains(Self.configPersistedMarker) + } + + /// Fires on every config-variant launch (cold 200 and warm 204 alike), so it doubles as + /// runtime proof that `ENABLE_REMOTE_CONFIG` is compiled into the SDK being measured. + static func isConfigRefreshStarted(_ message: String) -> Bool { + return message.contains(Self.configRefreshingMarker) + } + + private static func integer(before suffix: String, in message: String) -> Int? { + guard let suffixRange = message.range(of: suffix) else { return nil } + let digits = message[..= configuration.warmup } + .filter { ($0.element?.configPathActive ?? false) != expectsConfigPath } + XCTAssertTrue( + mismatches.isEmpty, + "\(mismatches.count) launch(es) contradict the \(configuration.modeLabel) label " + + "(expected configPathActive == \(expectsConfigPath)); wrong SDK variant in the binary?" + ) + } } private static var profile: String { diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift index adaafea18f..6fae4a1088 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift @@ -64,7 +64,9 @@ enum AppLaunchMetrics { "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count ] - let totals = measured.compactMap(\.paywallAppearedMs).sorted() + // Headline statistic: time until paywall CONTENT appeared (the SDK's impression + // mark), not the wrapper mount, which can precede content by a loading state. + let totals = measured.compactMap(\.paywallImpressionMs).sorted() if !totals.isEmpty { row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) row["min_ms"] = Self.rounded(totals[0]) @@ -78,11 +80,26 @@ enum AppLaunchMetrics { ("configured_ms_mean", measured.compactMap(\.configuredMs)), ("customer_info_ms_mean", measured.compactMap(\.customerInfoMs)), ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), - ("paywall_appeared_ms_mean", measured.compactMap(\.paywallAppearedMs)) + ("paywall_appeared_ms_mean", measured.compactMap(\.paywallAppearedMs)), + ("paywall_impression_ms_mean", measured.compactMap(\.paywallImpressionMs)), + ("config_persisted_ms_mean", measured.compactMap(\.configPersistedMs)), + ("last_blob_stored_ms_mean", measured.compactMap(\.lastBlobStoredMs)), + ("blobs_inline_mean", measured.map { Double($0.blobsInline) }), + ("blobs_downloaded_mean", measured.map { Double($0.blobsDownloaded) }), + ("blob_bytes_mean", measured.map { Double($0.blobBytes) }) ] where !values.isEmpty { row[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) } + // Size extremes across the run: together they bracket the backend's inline-size + // budget (largest blob seen inline vs smallest blob that needed a CDN download). + if let maxInline = measured.compactMap(\.maxInlineBlobBytes).max() { + row["max_inline_blob_bytes"] = maxInline + } + if let minDownloaded = measured.compactMap(\.minDownloadedBlobBytes).min() { + row["min_downloaded_blob_bytes"] = minDownloaded + } + if let firstError = errors.first { row["first_error"] = "iteration \(firstError.index): \(firstError.message)" } @@ -104,7 +121,8 @@ enum AppLaunchMetrics { ("configuredMs", sample.configuredMs), ("customerInfoMs", sample.customerInfoMs), ("offeringsMs", sample.offeringsMs), - ("paywallAppearedMs", sample.paywallAppearedMs) + ("paywallAppearedMs", sample.paywallAppearedMs), + ("paywallImpressionMs", sample.paywallImpressionMs) ] where value == nil { return "sample missing \(phase)" } diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh index 53aab6120c..21b85aeed2 100755 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -123,11 +123,14 @@ for variant in $VARIANTS; do LOG="$(mktemp)" DERIVED_DATA="$DERIVED_DATA_ROOT/$variant" + EXPECT_CONFIG_PATH=0 + [[ -n "$CONDITIONS" ]] && EXPECT_CONFIG_PATH=1 if ! env TEST_RUNNER_BENCH_API_KEY="$RESOLVED_KEY" \ TEST_RUNNER_BENCH_MODE_LABEL="app-launch-$variant" \ TEST_RUNNER_BENCH_ITERATIONS="$ITERATIONS" \ TEST_RUNNER_BENCH_WARMUP="$WARMUP" \ TEST_RUNNER_BENCH_PROJECT_ID="$PROJECT_ID" \ + TEST_RUNNER_BENCH_EXPECT_CONFIG_PATH="$EXPECT_CONFIG_PATH" \ xcodebuild -workspace "$REPO_ROOT/RevenueCat-Tuist.xcworkspace" \ -scheme SDKConfigBenchmarkApp \ -destination "$DESTINATION" \ @@ -139,8 +142,17 @@ for variant in $VARIANTS; do fi # Prove the variant flag reached the SDK compile (guards the manifest-cache hazard). - # Only checkable when this run actually compiled the SDK; cached builds get a warning. + # Only checkable when this run actually compiled the SDK; cached builds still get + # runtime verification via BENCH_EXPECT_CONFIG_PATH (each launched binary reports + # whether the config path ran, and the tests fail on a mismatch). if grep -q "SwiftCompile.*RevenueCat" "$LOG"; then + # The rows claim shipping-SDK numbers: refuse Debug-built frameworks. + if ! grep -q "Release-iphone" "$LOG"; then + echo "Variant $variant compiled without a Release configuration; refusing its rows" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + rm -f "$LOG" + continue + fi if [[ -n "$CONDITIONS" ]] && ! grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then echo "Variant $variant compiled WITHOUT ENABLE_REMOTE_CONFIG; refusing its rows" >&2 FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) From fefc034fa9f1396772b617bb4f61c2858ccd4f55 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 11:44:56 +0200 Subject: [PATCH 25/34] other(benchmarks): exclude Tuist-only benchmark targets from the xcodeproj sync check The SDKConfigBenchmark CLI and SDKConfigBenchmarkApp files live only in their Tuist projects and are intentionally absent from RevenueCat.xcodeproj, same as Tests/APITesters. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Dangerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dangerfile b/Dangerfile index 01ec6063b8..19e7e1928e 100644 --- a/Dangerfile +++ b/Dangerfile @@ -4,7 +4,11 @@ require 'pathname' require 'set' EXCLUDED_SWIFT_PATH_PREFIXES = [ - 'Tests/APITesters/' + 'Tests/APITesters/', + # Tuist-only benchmark targets (Projects/SDKConfigBenchmark and + # Projects/SDKConfigBenchmarkApp); intentionally absent from RevenueCat.xcodeproj. + 'Tests/Benchmarks/', + 'Tests/TestingApps/SDKConfigBenchmarkApp/' ].freeze COMMENT_MARKER = "" From a2edcb052ef03c95a756de2b63242376de5468f0 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 12:09:21 +0200 Subject: [PATCH 26/34] fix(benchmarks): make the app-launch variant switch survive cached derived data Two fixes found by running the Release configuration end to end. SwiftPM caches Package.swift evaluations by content hash (mtime is ignored), so the previous touch-based invalidation silently reused the prior variant's flags whenever a derived data path was reused; the script now maintains a variant marker comment in Package.swift (removed on exit) so the manifest re-evaluates exactly when the conditions change. And both variants now compile BYPASS_SIMULATED_STORE_RELEASE_CHECK, the SDK's designed opt-out for the guard that crashes Release builds configured with a Test Store key, which live runs use. First Release numbers recorded in the doc, including a real finding from the new blob registration: a 65-byte blob arrived via CDN instead of inline. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 37 +++++++++++-------- .../SDKConfigBenchmarkApp/run-app-launch.sh | 32 +++++++++++----- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index 479eed78bf..43581bfd8f 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -249,7 +249,10 @@ the time until a `PaywallView` actually appears on screen. - Tests build and run under **Release** (the rows claim shipping-SDK numbers), and every measured launch must prove at runtime which SDK variant is inside the binary: the config path's refresh log fires on every config-variant launch, and the runner fails on any launch - whose `configPathActive` contradicts the row's label, cached builds included. + whose `configPathActive` contradicts the row's label, cached builds included. Because live + runs use the project's Test Store key, which the SDK refuses (by crashing) in Release + builds, both variants also compile the SDK's designed opt-out for that guard, + `BYPASS_SIMULATED_STORE_RELEASE_CHECK`. - Rows carry `mode` = `app-launch-legacy` / `app-launch-config` and `profile` = `simulator` / `device`, so `compare.py` never mixes app-tier rows with CLI rows or simulator with device. @@ -263,20 +266,24 @@ Because there is no network control, treat app-tier numbers as end-to-end monito real network, not as a controlled A/B; the CLI's simulated transport remains the tool for isolating variables. -First sample (simulator, live network against the stress project with 2 inline workflow -blobs published, 4 iterations, 1 warmup discarded; ms to paywall appeared): - -| mode | scenario | p50 | offerings mean | customer info mean | configure mean | -|---|---|---|---|---|---| -| app-launch-legacy | cold | 1209 | 1160 | 694 | 18 | -| app-launch-config | cold | 1365 | 1342 | 835 | 22 | -| app-launch-legacy | warm | 915 | 829 | 635 | 18 | -| app-launch-config | warm | 1050 | 1068 | 792 | 19 | - -The config variant's cold overhead end to end (~13% here) is far below the CLI fixture's -worst case, because this project's published blobs arrive inline in the container instead -of as 100 prefetch-gated CDN downloads. Iteration counts this small are smoke-level; use -`ITERATIONS=25` or more for numbers worth acting on. +First Release sample (simulator, live network against the stress project, 3 iterations, +1 warmup discarded; p50 over time-to-paywall-impression, phase columns are means in ms): + +| mode | scenario | p50 | impression | wrapper | offerings | customer info | config persisted | blobs | +|---|---|---|---|---|---|---|---|---| +| app-launch-legacy | cold | 1174 | 1244 | 1095 | 1030 | 706 | - | - | +| app-launch-config | cold | 1607 | 1674 | 1347 | 1327 | 922 | 716 | 2 inline + 4 CDN, 161KB | +| app-launch-legacy | warm | 753 | 783 | 762 | 669 | 480 | - | - | +| app-launch-config | warm | 934 | 973 | 840 | 821 | 577 | - (204) | 0 (already on disk) | + +Notable in this sample: config cold runs ~35% over legacy end to end; the impression mark +trails the wrapper mount by ~330ms on config cold (vs ~90ms on legacy warm), which is the +loading-state gap the wrapper-based timing used to hide; and the blob registration caught a +real signal on its first Release run: `min_downloaded_blob_bytes` = **65** — a 65-byte blob +was CDN-fetched despite being far under the inline budget (worth raising with the config +team: either those blobs should be inlined, or the inline policy is scoped more narrowly +than "size under budget"). Iteration counts this small are smoke-level; use `ITERATIONS=25` +or more for numbers worth acting on. ## Known limitations diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh index 21b85aeed2..1dc673f457 100755 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -45,6 +45,8 @@ if [[ -f "$XCCONFIG" ]]; then cp "$XCCONFIG" "$XCCONFIG_BACKUP" fi +VARIANT_MARKER="// sdk-config-benchmark-variant:" + restore_xcconfig() { if [[ "$XCCONFIG_EXISTED" == "1" ]]; then cp "$XCCONFIG_BACKUP" "$XCCONFIG" @@ -52,7 +54,7 @@ restore_xcconfig() { else rm -f "$XCCONFIG" fi - touch "$REPO_ROOT/Package.swift" + sed -i '' "/^${VARIANT_MARKER//\//\\/}/d" "$REPO_ROOT/Package.swift" } trap restore_xcconfig EXIT @@ -103,18 +105,30 @@ set_swift_conditions() { # shellcheck disable=SC2016 # $(inherited) is an xcconfig token, not shell printf '\nSWIFT_ACTIVE_COMPILATION_CONDITIONS = $(inherited) %s\n' "$conditions" >> "$XCCONFIG" fi - # SwiftPM caches manifest evaluations; the xcconfig is not part of the cache key, so - # force a re-evaluation or a stale flag set could silently build the wrong variant. - touch "$REPO_ROOT/Package.swift" + # SwiftPM caches manifest evaluations by Package.swift CONTENT (mtime is ignored), and + # the xcconfig is not part of the cache key: without a content change, a reused derived + # data path silently builds the previous variant's flags. A marker comment carrying the + # current conditions (removed on exit) forces re-evaluation exactly when they change. + sed -i '' "/^${VARIANT_MARKER//\//\\/}/d" "$REPO_ROOT/Package.swift" + printf '%s %s\n' "$VARIANT_MARKER" "$conditions" >> "$REPO_ROOT/Package.swift" } FAILED_VARIANTS=0 TOTAL_ROWS=0 for variant in $VARIANTS; do + # BYPASS_SIMULATED_STORE_RELEASE_CHECK: tests build Release (shipping-SDK numbers), and + # live runs use the project's Test Store key, which the SDK otherwise refuses (by + # crashing) in Release builds. This is the SDK's designed opt-out for that guard. case "$variant" in - legacy) CONDITIONS="" ;; - config) CONDITIONS="ENABLE_REMOTE_CONFIG" ;; + legacy) + CONDITIONS="BYPASS_SIMULATED_STORE_RELEASE_CHECK" + EXPECT_CONFIG_PATH=0 + ;; + config) + CONDITIONS="ENABLE_REMOTE_CONFIG BYPASS_SIMULATED_STORE_RELEASE_CHECK" + EXPECT_CONFIG_PATH=1 + ;; *) echo "Unknown variant $variant (expected legacy or config)" >&2; exit 1 ;; esac @@ -123,8 +137,6 @@ for variant in $VARIANTS; do LOG="$(mktemp)" DERIVED_DATA="$DERIVED_DATA_ROOT/$variant" - EXPECT_CONFIG_PATH=0 - [[ -n "$CONDITIONS" ]] && EXPECT_CONFIG_PATH=1 if ! env TEST_RUNNER_BENCH_API_KEY="$RESOLVED_KEY" \ TEST_RUNNER_BENCH_MODE_LABEL="app-launch-$variant" \ TEST_RUNNER_BENCH_ITERATIONS="$ITERATIONS" \ @@ -153,13 +165,13 @@ for variant in $VARIANTS; do rm -f "$LOG" continue fi - if [[ -n "$CONDITIONS" ]] && ! grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then + if [[ "$EXPECT_CONFIG_PATH" == "1" ]] && ! grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then echo "Variant $variant compiled WITHOUT ENABLE_REMOTE_CONFIG; refusing its rows" >&2 FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) rm -f "$LOG" continue fi - if [[ -z "$CONDITIONS" ]] && grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then + if [[ "$EXPECT_CONFIG_PATH" == "0" ]] && grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then echo "Variant $variant compiled WITH ENABLE_REMOTE_CONFIG; refusing its rows" >&2 FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) rm -f "$LOG" From b512e7a5347463be2314f3b1e5e4cce0c749ad40 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 12:57:26 +0200 Subject: [PATCH 27/34] fix(benchmarks): require a successful config outcome and quarantine failed-run rows Two adversarial-review findings. configPathActive only proved the config refresh started; a transient refresh failure falls back to legacy delivery and would have passed as a clean config-variant row measuring the wrong system. Launches now record the refresh's terminal outcome (persisted, not_modified, failed) from the log stream, and config-variant runs require persisted on cold and persisted or not_modified on warm. And the runner script no longer extracts BENCHMARK_ROW lines from a failed xcodebuild invocation, since the tests print rows before their validity assertions run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Tests/AppLaunchMetricsTests.swift | 10 +++++ .../App/LaunchMeasurement.swift | 45 ++++++++++++++++--- .../UITests/AppLaunchBenchmarkUITests.swift | 20 ++++++++- .../SDKConfigBenchmarkApp/run-app-launch.sh | 5 +++ 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift index 8cc8d2ddac..e025a6747e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -124,6 +124,8 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { sample.blobBytes = 40_000 sample.maxInlineBlobBytes = 30_000 sample.minDownloadedBlobBytes = 10_000 + sample.configPathActive = true + sample.configOutcome = "persisted" let decoded = try XCTUnwrap(LaunchSample.decode(from: sample.jsonString())) XCTAssertEqual(decoded, sample) } @@ -204,6 +206,14 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { ).description XCTAssertTrue(BlobLogParser.isConfigRefreshStarted(refreshing)) XCTAssertFalse(BlobLogParser.isConfigRefreshStarted(persisted)) + + // Terminal outcomes: a failed refresh must never pass as a config-path measurement. + XCTAssertTrue(BlobLogParser.isConfigNotModified(RemoteConfigStrings.notModified.description)) + XCTAssertTrue(BlobLogParser.isConfigRefreshFailed( + RemoteConfigStrings.refreshFailed(.missingAppUserID()).description + )) + XCTAssertFalse(BlobLogParser.isConfigNotModified(refreshing)) + XCTAssertFalse(BlobLogParser.isConfigRefreshFailed(refreshing)) } func testBlobLogParserToleratesLoggerPrefixesAndIgnoresOtherMessages() { diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift index 53124d546c..d6435b8056 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift @@ -32,6 +32,10 @@ struct LaunchSample: Codable, Equatable { /// Whether the config endpoint path ran this launch (its refresh was observed in the log). /// Runtime proof of which SDK variant is actually inside the binary. var configPathActive: Bool = false + /// Terminal state of the config refresh: `persisted`, `not_modified`, or `failed`. + /// A config-variant launch whose refresh failed silently measures legacy fallback + /// behavior, so the runner requires a success outcome, not just an active path. + var configOutcome: String? /// Milliseconds until the config endpoint response was persisted (config variant only; /// observed via the SDK's log stream, nil when the config path never ran). var configPersistedMs: Double? @@ -90,6 +94,8 @@ enum BlobLogParser { private static let downloadedMarker = "Stored remote config blob '" private static let configPersistedMarker = "Persisted remote config for domain '" private static let configRefreshingMarker = "Refreshing remote config for domain '" + private static let configNotModifiedMarker = "Remote config was not modified" + private static let configRefreshFailedMarker = "Remote config refresh failed" static func blobEvent(from message: String) -> BlobStorageEvent? { let source: BlobStorageEvent.Source @@ -115,6 +121,17 @@ enum BlobLogParser { return message.contains(Self.configRefreshingMarker) } + /// The refresh revalidated via a manifest 204 (warm path's terminal success state). + static func isConfigNotModified(_ message: String) -> Bool { + return message.contains(Self.configNotModifiedMarker) + } + + /// The refresh failed and the SDK fell back to cached/legacy behavior. A launch ending + /// here must not pass as a config-path measurement. + static func isConfigRefreshFailed(_ message: String) -> Bool { + return message.contains(Self.configRefreshFailedMarker) + } + private static func integer(before suffix: String, in message: String) -> Int? { guard let suffixRange = message.range(of: suffix) else { return nil } let digits = message[..= configuration.warmup } + let measured = samples.enumerated().filter { $0.offset >= configuration.warmup } + let mismatches = measured .filter { ($0.element?.configPathActive ?? false) != expectsConfigPath } XCTAssertTrue( mismatches.isEmpty, "\(mismatches.count) launch(es) contradict the \(configuration.modeLabel) label " + "(expected configPathActive == \(expectsConfigPath)); wrong SDK variant in the binary?" ) + + // An active config path is not enough: a failed refresh silently falls back to + // legacy delivery, so the row would measure the wrong system with clean timings. + // Cold launches must persist a fresh config; warm launches may revalidate (204). + if expectsConfigPath { + let acceptedOutcomes = scenario == "cold" ? ["persisted"] : ["persisted", "not_modified"] + let badOutcomes = measured + .map { ($0.offset, $0.element?.configOutcome ?? "none") } + .filter { !acceptedOutcomes.contains($0.1) } + XCTAssertTrue( + badOutcomes.isEmpty, + "\(badOutcomes.count) config-variant launch(es) lack a successful config outcome " + + "(accepted: \(acceptedOutcomes)); first: iteration \(badOutcomes.first?.0 ?? -1) " + + "= \(badOutcomes.first?.1 ?? "")" + ) + } } } diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh index 1dc673f457..5c8ab36ec2 100755 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -151,6 +151,11 @@ for variant in $VARIANTS; do echo "Variant $variant FAILED; last log lines:" >&2 tail -25 "$LOG" >&2 FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + # A failed invocation must not leak rows into the JSONL: the tests print + # BENCHMARK_ROW before their validity assertions, so rows from a failed run may + # look clean while measuring the wrong thing. + rm -f "$LOG" + continue fi # Prove the variant flag reached the SDK compile (guards the manifest-cache hazard). From 0efad911ce41576d981f6056b3253314a00220d2 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 13:12:55 +0200 Subject: [PATCH 28/34] other(benchmarks): apply simplify-pass cleanups to the app-launch tier Move the cold-launch state wipe before the clock anchor (harness cleanup I/O was counted inside every cold phase), centralize the SDK-copied literals (content-appeared event types, UserDefaults suite name) into a shared SDKObservedValues pinned by unit tests against the real SDK events and suite, extract the mafdet key resolution shared by both tiers into resolve-api-key.sh, and fold the runner script's four copy-pasted failure blocks and mirrored flag greps into one helper and one polarity check. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Tests/AppLaunchMetricsTests.swift | 40 +++++++++++- .../SDKConfigBenchmark/resolve-api-key.sh | 40 ++++++++++++ .../SDKConfigBenchmark/run-matrix.sh | 22 ++----- .../App/BenchmarkAppMain.swift | 20 +++--- .../App/LaunchMeasurement.swift | 23 ++++++- .../UITests/AppLaunchBenchmarkUITests.swift | 9 +-- .../SDKConfigBenchmarkApp/run-app-launch.sh | 61 +++++++------------ 7 files changed, 140 insertions(+), 75 deletions(-) create mode 100644 Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift index e025a6747e..70393b0ae2 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -1,6 +1,7 @@ import XCTest -@testable import SDKConfigBenchmarkCore +// swiftlint:disable:next attributes +@_spi(Internal) @testable import SDKConfigBenchmarkCore final class AppLaunchMetricsTests: BenchmarkTestCase { @@ -216,6 +217,43 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertFalse(BlobLogParser.isConfigRefreshFailed(refreshing)) } + /// The app decides "content appeared" from event-type strings it copies out of the SDK; + /// these build the real SDK events and assert the copies still match. + func testContentAppearedEventTypesMatchRealSDKEvents() throws { + let paywallData = PaywallEvent.Data( + paywallIdentifier: nil, + offeringIdentifier: "offering", + paywallRevision: 0, + sessionID: .init(), + displayMode: .fullScreen, + localeIdentifier: "en_US", + darkMode: false, + source: nil, + presentedOfferingContext: .init(offeringIdentifier: "offering") + ) + let impression = PaywallEvent.impression(.init(), paywallData) + let impressionType = try XCTUnwrap(impression.toMap()["type"] as? String) + XCTAssertTrue(SDKObservedValues.contentAppearedEventTypes.contains(impressionType)) + + let stepStarted = WorkflowEvent.stepStarted(.init(), .init(workflowId: "w", stepId: "s")) + let stepType = try XCTUnwrap(stepStarted.toMap()["type"] as? String) + XCTAssertTrue(SDKObservedValues.contentAppearedEventTypes.contains(stepType)) + } + + /// The app wipes the SDK's UserDefaults suite by a copied name (the SDK's constant is + /// private). Behavioral pin: a value written through the SDK's own suite must be visible + /// through a suite created with the copied name. + func testRevenueCatUserDefaultsSuiteNameMatchesRealSDKSuite() throws { + let key = "benchmark-suite-pin-\(UUID().uuidString)" + UserDefaults.revenueCatSuite.set(true, forKey: key) + defer { UserDefaults.revenueCatSuite.removeObject(forKey: key) } + + let mirror = try XCTUnwrap( + UserDefaults(suiteName: SDKObservedValues.revenueCatUserDefaultsSuiteName) + ) + XCTAssertTrue(mirror.bool(forKey: key)) + } + func testBlobLogParserToleratesLoggerPrefixesAndIgnoresOtherMessages() { let prefixed = "[Purchases] - VERBOSE: 😻 Stored inline remote config blob 'r' with 77 bytes." XCTAssertEqual(BlobLogParser.blobEvent(from: prefixed)?.byteCount, 77) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh new file mode 100644 index 0000000000..afd8205d39 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Shared API-key resolution for both benchmark tiers (run-matrix.sh and run-app-launch.sh), +# so live runs of the two tiers always resolve the same key for the same project and stay +# comparable. No keys live in source. +# +# Usage (from a script with `set -euo pipefail`): +# source ".../resolve-api-key.sh" +# KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" +# +# Resolution order: the SDK_CONFIG_BENCHMARK_API_KEY environment variable, else mafdet +# (preferring the project's test-store app, whose products actually exist). Exits nonzero +# with a message on stderr when no key can be resolved. + +resolve_benchmark_api_key() { + local project_id="$1" + local key + + if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then + key="$SDK_CONFIG_BENCHMARK_API_KEY" + else + if ! command -v mafdet >/dev/null; then + echo "set SDK_CONFIG_BENCHMARK_API_KEY or install the mafdet CLI to resolve the project key" >&2 + return 1 + fi + key="$(mafdet app api-keys --project-id "$project_id" 2>/dev/null | python3 -c ' +import json, sys +keys = json.load(sys.stdin) +keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") +print(keys[0]["key"] if keys else "") +')" + fi + + if [[ -z "$key" ]]; then + echo "Could not resolve an API key for project $project_id" >&2 + return 1 + fi + + printf '%s' "$key" +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index 312a5ffbf7..6e58547d91 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -67,23 +67,13 @@ FAILED_ROWS=0 PROJECT_ARGS=() if [[ "$TRANSPORT" == "live" ]]; then - # No keys live in source: resolve the target project's key at run time. + # No keys live in source: resolve the target project's key at run time (shared with + # the app-launch tier so both tiers always measure the same key for the same project). PROJECT_ID="${PROJECT_ID:-5f07e7e3}" - if ! command -v mafdet >/dev/null; then - echo "live runs resolve the project API key via the mafdet CLI; install it or run the binary directly with --api-key" >&2 - exit 1 - fi - RESOLVED_KEY="$(mafdet app api-keys --project-id "$PROJECT_ID" 2>/dev/null | python3 -c ' -import json, sys -keys = json.load(sys.stdin) -keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") -print(keys[0]["key"] if keys else "") -')" - if [[ -z "$RESOLVED_KEY" ]]; then - echo "Could not resolve an API key for project $PROJECT_ID via mafdet" >&2 - exit 1 - fi - echo "Live target: project $PROJECT_ID (key resolved via mafdet)" >&2 + # shellcheck source=resolve-api-key.sh disable=SC1091 + source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" + RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" + echo "Live target: project $PROJECT_ID" >&2 PROJECT_ARGS=(--api-key "$RESOLVED_KEY" --project-id "$PROJECT_ID") fi diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift index be7dc1505e..a82f439c5c 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift @@ -29,13 +29,15 @@ struct SDKConfigBenchmarkApp: App { @StateObject private var model: LaunchModel init() { - let clock = LaunchClock() - self.clock = clock - + // Wipe before anchoring the clock: harness cleanup I/O must never count as SDK + // launch time (it would inflate every cold phase by the previous launch's litter). if CommandLine.arguments.contains("--wipe-state") { Self.wipeSDKState() } + let clock = LaunchClock() + self.clock = clock + let environment = ProcessInfo.processInfo.environment let model = LaunchModel(clock: clock) self._model = StateObject(wrappedValue: model) @@ -87,9 +89,8 @@ struct SDKConfigBenchmarkApp: App { if let bundleID = Bundle.main.bundleIdentifier { UserDefaults.standard.removePersistentDomain(forName: bundleID) } - // The SDK's own suite (UserDefaults.revenueCatSuiteName). - UserDefaults(suiteName: "com.revenuecat.user_defaults")? - .removePersistentDomain(forName: "com.revenuecat.user_defaults") + let sdkSuite = SDKObservedValues.revenueCatUserDefaultsSuiteName + UserDefaults(suiteName: sdkSuite)?.removePersistentDomain(forName: sdkSuite) } } @@ -210,11 +211,6 @@ final class LaunchModel: ObservableObject { /// paywall tracks `workflows_step_started` when its first step renders. final class PaywallImpressionListener: EventsListener { - private static let contentAppearedEventTypes: Set = [ - "paywall_impression", - "workflows_step_started" - ] - private let clock: LaunchClock private let onImpression: (Double) -> Void @@ -225,7 +221,7 @@ final class PaywallImpressionListener: EventsListener { func onEventTracked(_ event: [String: Any]) { guard let type = event["type"] as? String, - Self.contentAppearedEventTypes.contains(type) else { return } + SDKObservedValues.contentAppearedEventTypes.contains(type) else { return } self.onImpression(self.clock.elapsedMs()) } diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift index d6435b8056..acc507036a 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift @@ -64,12 +64,31 @@ struct LaunchSample: Codable, Equatable { } static func decode(from json: String) -> LaunchSample? { - guard let data = json.data(using: .utf8) else { return nil } - return try? JSONDecoder().decode(LaunchSample.self, from: data) + return try? JSONDecoder().decode(LaunchSample.self, from: Data(json.utf8)) } } +/// SDK values the app copies because the SDK does not export them. Each one is pinned to +/// the real SDK by a unit test in the benchmark suite, so drift over there fails a test +/// instead of silently corrupting rows (a stale suite name would stop `--wipe-state` from +/// wiping and make "cold" rows measure warm behavior; stale event types would time out +/// every launch). +enum SDKObservedValues { + + /// Event types meaning "paywall content actually appeared": a classic paywall tracks + /// `paywall_impression`; a workflow paywall tracks `workflows_step_started`. + static let contentAppearedEventTypes: Set = [ + "paywall_impression", + "workflows_step_started" + ] + + /// The UserDefaults suite the SDK persists into (`UserDefaults.revenueCatSuiteName`, + /// which is private). + static let revenueCatUserDefaultsSuiteName = "com.revenuecat.user_defaults" + +} + /// One blob landing in the SDK's content-addressed store, observed from the SDK's log stream. struct BlobStorageEvent: Equatable { diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift index d4d7ce9722..32fe53e1a1 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift @@ -137,9 +137,11 @@ final class AppLaunchBenchmarkUITests: XCTestCase { attachment.lifetime = .keepAlways self.add(attachment) - let postWarmupErrors = samples.enumerated() - .filter { $0.offset >= configuration.warmup } - .compactMap { AppLaunchMetrics.errorMessage(for: $0.element) } + // One definition of "measured sample" (post-warmup, by index) for every assertion, + // matching the window the row's statistics were aggregated from. + let measured = samples.enumerated().filter { $0.offset >= configuration.warmup } + + let postWarmupErrors = measured.compactMap { AppLaunchMetrics.errorMessage(for: $0.element) } XCTAssertTrue( postWarmupErrors.isEmpty, "\(postWarmupErrors.count) measured launch(es) failed; first: \(postWarmupErrors.first ?? "")" @@ -148,7 +150,6 @@ final class AppLaunchBenchmarkUITests: XCTestCase { // Runtime variant proof: the launched binary must have actually run (or not run) // the config path; a mislabeled row would poison every later comparison. if let expectsConfigPath = configuration.expectsConfigPath { - let measured = samples.enumerated().filter { $0.offset >= configuration.warmup } let mismatches = measured .filter { ($0.element?.configPathActive ?? false) != expectsConfigPath } XCTAssertTrue( diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh index 5c8ab36ec2..1c592c597f 100755 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -38,9 +38,7 @@ else fi XCCONFIG_BACKUP="" -XCCONFIG_EXISTED=0 if [[ -f "$XCCONFIG" ]]; then - XCCONFIG_EXISTED=1 XCCONFIG_BACKUP="$(mktemp)" cp "$XCCONFIG" "$XCCONFIG_BACKUP" fi @@ -48,7 +46,7 @@ fi VARIANT_MARKER="// sdk-config-benchmark-variant:" restore_xcconfig() { - if [[ "$XCCONFIG_EXISTED" == "1" ]]; then + if [[ -n "$XCCONFIG_BACKUP" ]]; then cp "$XCCONFIG_BACKUP" "$XCCONFIG" rm -f "$XCCONFIG_BACKUP" else @@ -58,25 +56,11 @@ restore_xcconfig() { } trap restore_xcconfig EXIT -# Resolve the API key: env override, else mafdet (no keys live in source). -if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then - RESOLVED_KEY="$SDK_CONFIG_BENCHMARK_API_KEY" -else - if ! command -v mafdet >/dev/null; then - echo "set SDK_CONFIG_BENCHMARK_API_KEY or install the mafdet CLI to resolve the project key" >&2 - exit 1 - fi - RESOLVED_KEY="$(mafdet app api-keys --project-id "$PROJECT_ID" 2>/dev/null | python3 -c ' -import json, sys -keys = json.load(sys.stdin) -keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") -print(keys[0]["key"] if keys else "") -')" -fi -if [[ -z "$RESOLVED_KEY" ]]; then - echo "Could not resolve an API key for project $PROJECT_ID" >&2 - exit 1 -fi +# Resolve the API key (shared with the CLI tier's run-matrix.sh: env override, else mafdet; +# no keys live in source). +# shellcheck source=../../Benchmarks/SDKConfigBenchmark/resolve-api-key.sh disable=SC1091 +source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" +RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" echo "Live target: project $PROJECT_ID" >&2 if [[ -z "${DESTINATION:-}" ]]; then @@ -116,6 +100,15 @@ set_swift_conditions() { FAILED_VARIANTS=0 TOTAL_ROWS=0 +# Records a variant failure and discards its log. Rows from a failed or unverified variant +# must never reach stdout: the tests print BENCHMARK_ROW before their validity assertions, +# so a leaked row can look clean while measuring the wrong thing. Follow with `continue`. +fail_variant() { + echo "$1" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + rm -f "$LOG" +} + for variant in $VARIANTS; do # BYPASS_SIMULATED_STORE_RELEASE_CHECK: tests build Release (shipping-SDK numbers), and # live runs use the project's Test Store key, which the SDK otherwise refuses (by @@ -150,11 +143,7 @@ for variant in $VARIANTS; do test > "$LOG" 2>&1; then echo "Variant $variant FAILED; last log lines:" >&2 tail -25 "$LOG" >&2 - FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) - # A failed invocation must not leak rows into the JSONL: the tests print - # BENCHMARK_ROW before their validity assertions, so rows from a failed run may - # look clean while measuring the wrong thing. - rm -f "$LOG" + fail_variant "Variant $variant: xcodebuild test failed" continue fi @@ -165,21 +154,13 @@ for variant in $VARIANTS; do if grep -q "SwiftCompile.*RevenueCat" "$LOG"; then # The rows claim shipping-SDK numbers: refuse Debug-built frameworks. if ! grep -q "Release-iphone" "$LOG"; then - echo "Variant $variant compiled without a Release configuration; refusing its rows" >&2 - FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) - rm -f "$LOG" - continue - fi - if [[ "$EXPECT_CONFIG_PATH" == "1" ]] && ! grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then - echo "Variant $variant compiled WITHOUT ENABLE_REMOTE_CONFIG; refusing its rows" >&2 - FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) - rm -f "$LOG" + fail_variant "Variant $variant compiled without a Release configuration; refusing its rows" continue fi - if [[ "$EXPECT_CONFIG_PATH" == "0" ]] && grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG"; then - echo "Variant $variant compiled WITH ENABLE_REMOTE_CONFIG; refusing its rows" >&2 - FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) - rm -f "$LOG" + HAS_CONFIG_FLAG=0 + grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG" && HAS_CONFIG_FLAG=1 + if [[ "$HAS_CONFIG_FLAG" != "$EXPECT_CONFIG_PATH" ]]; then + fail_variant "Variant $variant compiled with ENABLE_REMOTE_CONFIG=$HAS_CONFIG_FLAG, expected $EXPECT_CONFIG_PATH; refusing its rows" continue fi else From 2ed2191aa5e1bb9fbef9b2a42dfc43358b315829 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 13:48:56 +0200 Subject: [PATCH 29/34] fix(benchmarks): validate cold config exchanges and warm priming Convergence-pass review findings, all measurement-validity gates. The CLI now validates cold iterations too: config mode requires a successful config request (a failed refresh delivers offerings via legacy fallback and would pass as a config row), kill-switch mode requires its 4xx. The app tier's warm scenario fails when the priming launch errors (its samples would be cold starts labeled warm) and accepts only the not_modified outcome on measured warm launches, matching the CLI warm validation. The runner script validates the row count before emitting, so a partial scenario can no longer leak rows into the captured JSONL. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkRunner.swift | 45 +++++++++++++++-- .../BenchmarkRunnerValidationTests.swift | 48 +++++++++++++++++++ .../UITests/AppLaunchBenchmarkUITests.swift | 16 +++++-- .../SDKConfigBenchmarkApp/run-app-launch.sh | 10 ++-- 4 files changed, 106 insertions(+), 13 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index b4be9d0a5c..7e10372f03 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -214,12 +214,19 @@ private extension BenchmarkRunner { return Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000 } - /// Warm runs must prove that EVERY measured iteration hit the revalidation paths; a single - /// iteration silently falling back to full 200 responses or re-downloading blobs would mix - /// cold behavior into the warm distribution. + /// Every measured iteration must prove it exercised the path its row claims: warm runs + /// must hit the revalidation paths, and cold config runs must complete a successful + /// config request (a failed refresh silently delivers offerings via the legacy fallback, + /// which would let a "config" row measure the wrong system). Skipped under packet loss, + /// where transient failures are the thing being measured. func validateScenario(_ measurement: IterationMeasurement, iteration: Int) throws { - guard self.command.scenario == .warm, self.command.lossPercent == 0 else { return } - try Self.validateWarmMeasurement(measurement, mode: self.command.mode, iteration: iteration) + guard self.command.lossPercent == 0 else { return } + switch self.command.scenario { + case .warm: + try Self.validateWarmMeasurement(measurement, mode: self.command.mode, iteration: iteration) + case .cold: + try Self.validateColdMeasurement(measurement, mode: self.command.mode, iteration: iteration) + } } } @@ -250,6 +257,34 @@ extension BenchmarkRunner { return BlobAccounting(newRefSizes: newRefSizes, downloadedRefs: downloadedRefs) } + /// Cold config iterations must have actually completed the config exchange their mode + /// claims: a success for plain config mode, the disabling 4xx for the kill switch. + static func validateColdMeasurement( + _ measurement: IterationMeasurement, + mode: BenchmarkMode, + iteration: Int + ) throws { + guard mode.usesRemoteConfig else { return } + + if mode.forcesConfigFailure { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ (400...499).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "cold kill-switch iteration \(iteration) did not hit the config 4xx " + + "(statuses: \(measurement.configStatusCodes))" + ) + } + return + } + + guard measurement.configStatusCodes.contains(where: { (200...299).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "cold config iteration \(iteration) never completed a successful config request " + + "(statuses: \(measurement.configStatusCodes)); the row would measure legacy fallback" + ) + } + } + static func validateWarmMeasurement( _ measurement: IterationMeasurement, mode: BenchmarkMode, diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift index 8f93281d19..be80b0f4c4 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -96,6 +96,54 @@ final class BenchmarkRunnerValidationTests: BenchmarkTestCase { )) } + // MARK: - Cold validation + + /// A cold config iteration whose config refresh failed still delivers offerings through + /// the legacy fallback, so without this check the row would pass as a config measurement. + func testColdConfigModeRequiresASuccessfulConfigRequest() throws { + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .config, iteration: 1 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [500, 503]), mode: .config, iteration: 1 + )) + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [500, 200]), mode: .config, iteration: 1 + )) + } + + func testColdKillswitchModeRequiresThe4xx() throws { + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [400]), mode: .configKillswitch, iteration: 0 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [200]), mode: .configKillswitch, iteration: 0 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .configKillswitch, iteration: 0 + )) + } + + func testColdLegacyModeNeedsNoConfigRequest() throws { + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .legacy, iteration: 0 + )) + } + + private static let configURL = URL(string: "https://api.revenuecat.com/v1/config/app") + + private func measurement(configStatusCodes: [Int]) -> IterationMeasurement { + guard let url = Self.configURL else { + preconditionFailure("static config URL failed to parse") + } + let events: [TransportEvent] = configStatusCodes.map { status in + TransportEvent.success( + url: url, iteration: 0, statusCode: status, bytesReceived: 10, startedAt: .now() + ) + } + return IterationMeasurement(totalMs: 10, events: events) + } + // MARK: - Blob accounting func testBlobAccountingCountsOnlyRefsStoredThisIterationAndAttributesByRequest() throws { diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift index 32fe53e1a1..e87890e33c 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift @@ -62,8 +62,14 @@ final class AppLaunchBenchmarkUITests: XCTestCase { let appUserID = "bench-app-warm-\(configuration.runNonce)" // One uncounted priming launch populates the disk caches; measured launches then - // relaunch with retained state and the same user. - _ = self.launchAndCollect(configuration: configuration, appUserID: appUserID, wipeState: true) + // relaunch with retained state and the same user. The priming launch must succeed: + // if it left the caches empty, every "warm" sample would actually be a cold start. + let priming = self.launchAndCollect(configuration: configuration, appUserID: appUserID, wipeState: true) + if let primingError = AppLaunchMetrics.errorMessage(for: priming) { + XCTFail("Warm priming launch failed (\(primingError)); samples would measure cold starts") + return + } + let samples = (0..&2 - FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + fail_variant "Variant $variant produced $ROW_COUNT row(s), expected 2 (cold + warm); refusing them" + continue fi + printf '%s\n' "$ROWS" + TOTAL_ROWS=$((TOTAL_ROWS + ROW_COUNT)) rm -f "$LOG" done From 1478c09c77883c9aa3687081f785c5d597ee726b Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 14:24:36 +0200 Subject: [PATCH 30/34] fix(benchmarks): require an explicit project label with externally supplied keys Final review-pass finding: SDK_CONFIG_BENCHMARK_API_KEY (or --api-key) with no project id let rows carry the pinned default project label with another project's key, so live rows from different projects could compare as equivalents. The shared script helper and the CLI binary now both require the explicit label. Also corrects the observer-overhead comment: the verbose-log observer is not perfectly even across variants (config logs more lines), but its per-line cost is microseconds against hundreds-of-ms deltas. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- Tests/Benchmarks/SDKConfigBenchmark/README.md | 5 +++-- .../Sources/BenchmarkCommand.swift | 20 +++++++++++-------- .../Tests/BenchmarkCommandTests.swift | 17 +++++++++++----- .../SDKConfigBenchmark/resolve-api-key.sh | 16 +++++++++++++++ .../SDKConfigBenchmark/run-matrix.sh | 5 +++-- .../App/BenchmarkAppMain.swift | 6 ++++-- .../SDKConfigBenchmarkApp/run-app-launch.sh | 11 +++++----- 7 files changed, 56 insertions(+), 24 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index 58273ff0ce..e8c0858ec8 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -12,7 +12,8 @@ Two transports: prepared stress-test project ([`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3)). No keys live in source: the matrix script resolves the key via mafdet; direct binary runs take `--api-key` or the - `SDK_CONFIG_BENCHMARK_API_KEY` environment variable. Same recorded per-request metrics; real + `SDK_CONFIG_BENCHMARK_API_KEY` environment variable, either of which also requires + `--project-id` so rows are labeled with the key's real project. Same recorded per-request metrics; real CDN/TLS/latency behavior. Kill-switch, profiles, and loss are simulated-only (you cannot force a 4xx or packet loss on production). @@ -103,7 +104,7 @@ python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py app-launch.jsonl Rows are `compare.py`-compatible, with `mode` = `app-launch-legacy` / `app-launch-config` and `profile` = `simulator` / `device`. Knobs: `ITERATIONS`, `WARMUP`, `PROJECT_ID`, -`SDK_CONFIG_BENCHMARK_API_KEY` (skips mafdet), and `DESTINATION` (pass +`SDK_CONFIG_BENCHMARK_API_KEY` (skips mafdet; requires an explicit `PROJECT_ID`), and `DESTINATION` (pass `DESTINATION="platform=iOS,id="` to measure a physical device's real radio). Live only: there is no simulated transport, no kill-switch mode, and no loss model in this tier. diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 0a5eb8ccb9..b999136eaa 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -224,21 +224,25 @@ struct BenchmarkCommand { if self.apiKey == Self.defaultSimulatedAPIKey { // No keys live in source; resolve from the environment (run-matrix.sh populates - // it via mafdet) and assume the pinned project unless labeled otherwise. + // it via mafdet). guard let environmentKey = environment[BenchmarkProject.apiKeyEnvironmentVariable], !environmentKey.isEmpty else { throw BenchmarkError.invalidArgument( "live runs need an API key: pass --api-key with --project-id, set " + - "\(BenchmarkProject.apiKeyEnvironmentVariable), or use run-matrix.sh " + - "(resolves it via mafdet)" + "\(BenchmarkProject.apiKeyEnvironmentVariable) with --project-id, or use " + + "run-matrix.sh (resolves it via mafdet)" ) } self.apiKey = environmentKey - self.projectID = self.projectID ?? BenchmarkProject.projectID - } else if self.projectID == nil { - // A custom key without a project label would let rows from different projects - // collide in comparisons. - throw BenchmarkError.invalidArgument("--api-key with --transport live also requires --project-id") + } + // ANY externally supplied key requires a project label: nothing ties a key to the + // pinned default project, and rows from different projects must never compare as + // equivalents. + guard self.projectID != nil else { + throw BenchmarkError.invalidArgument( + "a live API key (--api-key or \(BenchmarkProject.apiKeyEnvironmentVariable)) " + + "also requires --project-id, so rows are labeled with the key's real project" + ) } // Fixture-size knobs don't shape live payloads (the pinned project's real content diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift index d0f2a9ebb1..49adc91469 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -84,15 +84,22 @@ final class BenchmarkCommandTests: BenchmarkTestCase { XCTAssertEqual(try BenchmarkCommand.parse([]).transport, .simulated) } - func testLiveTransportResolvesKeyFromEnvironment() throws { - let command = try BenchmarkCommand.parse( + func testLiveTransportResolvesKeyFromEnvironmentButRequiresAProjectLabel() throws { + // An environment key without a project label could mislabel rows: nothing ties the + // key to the pinned default project, and rows from different projects must never + // compare as equivalents. + XCTAssertThrowsError(try BenchmarkCommand.parse( ["--transport", "live"], environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] - ) + )) + let command = try BenchmarkCommand.parse( + ["--transport", "live", "--project-id", "abc123"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + ) XCTAssertEqual(command.transport, .live) XCTAssertEqual(command.apiKey, "test_fromEnv") - XCTAssertEqual(command.projectID, BenchmarkProject.projectID) + XCTAssertEqual(command.projectID, "abc123") } func testLiveTransportWithoutAnyKeyFails() { @@ -115,7 +122,7 @@ final class BenchmarkCommandTests: BenchmarkTestCase { func testLiveTransportZeroesFixtureSizeKnobs() throws { let command = try BenchmarkCommand.parse( - ["--transport", "live", "--paywalls", "500", "--workflows", "500"], + ["--transport", "live", "--project-id", "abc123", "--paywalls", "500", "--workflows", "500"], environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] ) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh index afd8205d39..f512a6589e 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh @@ -12,6 +12,22 @@ # (preferring the project's test-store app, whose products actually exist). Exits nonzero # with a message on stderr when no key can be resolved. +# Resolves the project id BEFORE any default applies. A custom key via +# SDK_CONFIG_BENCHMARK_API_KEY requires an explicit PROJECT_ID: otherwise rows would carry +# the pinned default project label with another project's key, and live rows from different +# projects would compare as equivalents. +default_benchmark_project_id() { + if [[ -z "${PROJECT_ID:-}" ]]; then + if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then + echo "SDK_CONFIG_BENCHMARK_API_KEY requires an explicit PROJECT_ID so rows are labeled with the key's real project" >&2 + return 1 + fi + printf '5f07e7e3' + else + printf '%s' "$PROJECT_ID" + fi +} + resolve_benchmark_api_key() { local project_id="$1" local key diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh index 6e58547d91..25e833b78c 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -68,10 +68,11 @@ FAILED_ROWS=0 PROJECT_ARGS=() if [[ "$TRANSPORT" == "live" ]]; then # No keys live in source: resolve the target project's key at run time (shared with - # the app-launch tier so both tiers always measure the same key for the same project). - PROJECT_ID="${PROJECT_ID:-5f07e7e3}" + # the app-launch tier so both tiers always measure the same key for the same project; + # an env-key override requires an explicit PROJECT_ID so rows are labeled correctly). # shellcheck source=resolve-api-key.sh disable=SC1091 source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" + PROJECT_ID="$(default_benchmark_project_id)" RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" echo "Live target: project $PROJECT_ID" >&2 PROJECT_ARGS=(--api-key "$RESOLVED_KEY" --project-id "$PROJECT_ID") diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift index a82f439c5c..98946bd7fd 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift @@ -48,8 +48,10 @@ struct SDKConfigBenchmarkApp: App { } // The config path (config persisted, each blob stored inline vs downloaded) is only - // observable through the SDK's log stream, so run verbose and parse. The logging - // overhead is inside the measured window but identical across variants. + // observable through the SDK's log stream, so run verbose and parse. The observer + // sits inside the measured window and is NOT perfectly even across variants (the + // config path emits more log lines), but its per-line cost is a few substring scans + // (microseconds), orders of magnitude below the hundreds-of-ms deltas measured. let observer = model.blobObserver Purchases.logLevel = .verbose Purchases.verboseLogHandler = { _, message, _, _, _ in diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh index e6542fa013..e4cbd3c78c 100755 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -27,7 +27,12 @@ DERIVED_DATA_ROOT="${DERIVED_DATA_ROOT:-$REPO_ROOT/.build/sdk-config-benchmark-a VARIANTS="${VARIANTS:-legacy config}" ITERATIONS="${ITERATIONS:-10}" WARMUP="${WARMUP:-2}" -PROJECT_ID="${PROJECT_ID:-5f07e7e3}" + +# Key + project resolution shared with the CLI tier's run-matrix.sh (env override requires +# an explicit PROJECT_ID; else mafdet against the pinned project; no keys live in source). +# shellcheck source=../../Benchmarks/SDKConfigBenchmark/resolve-api-key.sh disable=SC1091 +source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" +PROJECT_ID="$(default_benchmark_project_id)" # Package.swift prefers CI.xcconfig over Local.xcconfig, so the variant switch must edit # whichever file actually wins. @@ -56,10 +61,6 @@ restore_xcconfig() { } trap restore_xcconfig EXIT -# Resolve the API key (shared with the CLI tier's run-matrix.sh: env override, else mafdet; -# no keys live in source). -# shellcheck source=../../Benchmarks/SDKConfigBenchmark/resolve-api-key.sh disable=SC1091 -source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" echo "Live target: project $PROJECT_ID" >&2 From fe0e434d1f3eeb9e3b54f15925a734810ead7e9f Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 14:34:17 +0200 Subject: [PATCH 31/34] feat(benchmarks): make configure + getOfferings the app-launch headline Per team sync, the metric that gates release decisions is Purchases.configure + getOfferings with and without workflows compiled in. The row's percentile statistics now aggregate the offerings phase, matching the CLI tier's total (offerings delivered), so both tiers answer the same question. The paywall render marks (wrapper, impression) stay in the row as secondary phase means, and the launch flow is unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../CONFIG_ENDPOINT_BENCHMARKS.md | 38 +++++++++---------- Tests/Benchmarks/SDKConfigBenchmark/README.md | 11 +++--- .../Tests/AppLaunchMetricsTests.swift | 17 +++++---- .../UITests/AppLaunchMetrics.swift | 8 ++-- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md index 43581bfd8f..ae2cda7c1a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -236,9 +236,11 @@ the time until a `PaywallView` actually appears on screen. delivered), `offerings` (getOfferings completed), `paywall_appeared` (the PaywallView wrapper mounted), and `paywall_impression` (the SDK tracked that paywall CONTENT actually appeared: `paywall_impression` for classic paywalls, `workflows_step_started` for workflow - paywalls, observed via the SDK's internal events-listener SPI). The row's percentile - statistics are over `paywall_impression`; a launch missing any phase counts as an error and - fails the run. + paywalls, observed via the SDK's internal events-listener SPI). **The row's percentile + statistics are over `offerings`: the measurement is `Purchases.configure` + `getOfferings`, + with or without workflows compiled in, matching the CLI tier's total.** The paywall marks + stay in the row as secondary phase means; a launch missing any phase still counts as an + error and fails the run. - Config-path phases and blob registration, observed from the stock SDK's log stream (the parser is pinned to `RemoteConfigStrings` by unit tests): `config_persisted_ms_mean`, `last_blob_stored_ms_mean`, `blobs_inline_mean` vs `blobs_downloaded_mean`, `blob_bytes_mean`, @@ -266,24 +268,22 @@ Because there is no network control, treat app-tier numbers as end-to-end monito real network, not as a controlled A/B; the CLI's simulated transport remains the tool for isolating variables. -First Release sample (simulator, live network against the stress project, 3 iterations, -1 warmup discarded; p50 over time-to-paywall-impression, phase columns are means in ms): +First Release sample (simulator, live network against the stress project, 4 iterations, +1 warmup discarded; p50 over configure + getOfferings, phase columns are means in ms): -| mode | scenario | p50 | impression | wrapper | offerings | customer info | config persisted | blobs | +| mode | scenario | p50 | offerings | config persisted | customer info | wrapper | impression | blobs | |---|---|---|---|---|---|---|---|---| -| app-launch-legacy | cold | 1174 | 1244 | 1095 | 1030 | 706 | - | - | -| app-launch-config | cold | 1607 | 1674 | 1347 | 1327 | 922 | 716 | 2 inline + 4 CDN, 161KB | -| app-launch-legacy | warm | 753 | 783 | 762 | 669 | 480 | - | - | -| app-launch-config | warm | 934 | 973 | 840 | 821 | 577 | - (204) | 0 (already on disk) | - -Notable in this sample: config cold runs ~35% over legacy end to end; the impression mark -trails the wrapper mount by ~330ms on config cold (vs ~90ms on legacy warm), which is the -loading-state gap the wrapper-based timing used to hide; and the blob registration caught a -real signal on its first Release run: `min_downloaded_blob_bytes` = **65** — a 65-byte blob -was CDN-fetched despite being far under the inline budget (worth raising with the config -team: either those blobs should be inlined, or the inline policy is scoped more narrowly -than "size under budget"). Iteration counts this small are smoke-level; use `ITERATIONS=25` -or more for numbers worth acting on. +| app-launch-legacy | cold | 1221 | 1298 | - | 839 | 1405 | 1971 | - | +| app-launch-config | cold | 1354 | 1551 | 977 | 1009 | 1590 | 2361 | 2 inline + 4 CDN, 161KB | +| app-launch-legacy | warm | 888 | 921 | - | 690 | 1104 | 1188 | - | +| app-launch-config | warm | 1094 | 1267 | - (204) | 1008 | 1316 | 2149 | 0 (already on disk) | + +Notable in this sample: configure + getOfferings runs ~11% over legacy on cold and ~23% on +warm with workflows compiled in; the blob registration keeps catching the same real signal: +`min_downloaded_blob_bytes` = **65** — a 65-byte blob CDN-fetched despite being far under +the inline budget (worth raising with the config team: either those blobs should be inlined, +or the inline policy is scoped more narrowly than "size under budget"). Iteration counts +this small are smoke-level; use `ITERATIONS=25` or more for numbers worth acting on. ## Known limitations diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md index e8c0858ec8..c7c6ce84d1 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/README.md +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -110,11 +110,12 @@ there is no simulated transport, no kill-switch mode, and no loss model in this Each launch also registers the config path when it runs: config persisted time, blob counts split inline vs CDN-downloaded with byte totals, and size extremes that bracket the backend's -inline-size budget. The headline percentile is time to `paywall_impression` (content actually -on screen); tests run under Release and fail if a launch's observed SDK variant contradicts -the row's label. The intent is that any automation able to drive `xcodebuild test` (CI, the -baguette CLI, a cron box) can run the app N times and collect the same gateable JSONL as the -CLI matrix. +inline-size budget. The headline percentile is `Purchases.configure` + `getOfferings` +completed, with or without workflows compiled in (matching the CLI tier's total); paywall +render marks stay in the row as secondary phases. Tests run under Release and fail if a +launch's observed SDK variant contradicts the row's label. The intent is that any automation +able to drive `xcodebuild test` (CI, the baguette CLI, a cron box) can run the app N times +and collect the same gateable JSONL as the CLI matrix. ## Unit tests diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift index 70393b0ae2..9952636933 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -9,9 +9,9 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { var sample = LaunchSample() sample.configuredMs = total * 0.1 sample.customerInfoMs = total * 0.4 - sample.offeringsMs = total * 0.8 - sample.paywallAppearedMs = total * 0.9 - sample.paywallImpressionMs = total + sample.offeringsMs = total + sample.paywallAppearedMs = total * 1.1 + sample.paywallImpressionMs = total * 1.2 return sample } @@ -58,9 +58,10 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertEqual(row["project_id"] as? String, "5f07e7e3") } - func testRowStatisticsUsePaywallImpressionOfPostWarmupSamples() throws { + func testRowStatisticsUseOfferingsCompletionOfPostWarmupSamples() throws { // Warmup discards by index: the 500ms first launch never enters the stats. The - // headline percentiles use the impression mark (content appeared), not the wrapper. + // headline percentiles measure configure + getOfferings (matching the CLI tier); + // the paywall marks remain secondary phase means. let row = try self.decodedRow( samples: [Self.sample(500), Self.sample(100), Self.sample(300), Self.sample(200)] ) @@ -72,11 +73,11 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertEqual(row["p50_ms"] as? Double, 200) XCTAssertEqual(row["p95_ms"] as? Double, 300) XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) - XCTAssertEqual(row["paywall_impression_ms_mean"] as? Double, 200) - XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 180) + XCTAssertEqual(row["offerings_ms_mean"] as? Double, 200) + XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 220) + XCTAssertEqual(row["paywall_impression_ms_mean"] as? Double, 240) XCTAssertEqual(row["configured_ms_mean"] as? Double, 20) XCTAssertEqual(row["customer_info_ms_mean"] as? Double, 80) - XCTAssertEqual(row["offerings_ms_mean"] as? Double, 160) } func testSampleMissingImpressionCountsAsError() { diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift index 6fae4a1088..c3bb6fb9bf 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift @@ -64,9 +64,11 @@ enum AppLaunchMetrics { "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count ] - // Headline statistic: time until paywall CONTENT appeared (the SDK's impression - // mark), not the wrapper mount, which can precede content by a loading state. - let totals = measured.compactMap(\.paywallImpressionMs).sorted() + // Headline statistic: `Purchases.configure` + `getOfferings` completed, with or + // without workflows compiled in. This matches the CLI tier's total (offerings + // delivered), so the two tiers answer the same question; the paywall marks stay in + // the row as secondary phase means. + let totals = measured.compactMap(\.offeringsMs).sorted() if !totals.isEmpty { row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) row["min_ms"] = Self.rounded(totals[0]) From 9aa21699485baa69f9444b54b22240b31e7a6829 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 14:37:04 +0200 Subject: [PATCH 32/34] other(benchmarks): ignore python bytecode from compare.py runs Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 77482667db..eb9dcd018f 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,7 @@ Tests/TestingApps/PaywallsTester/PaywallsTester/PaywallsTester.storekit # Claude Code .claude/ + +# Python bytecode (e.g. from running the benchmark compare.py) +__pycache__/ +*.pyc From b66b56227f14f5ebad0fd6fc232bfe258c6cc543 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 16:18:53 +0200 Subject: [PATCH 33/34] feat(benchmarks): attribute stored blobs to their config topics Rows now say WHAT was fetched, not just how much: the CLI runner joins each iteration's newly stored blob refs against the persisted topic index and emits blobs_by_topic (per-topic count and byte means). First live run attributes all 3 inlined blobs in the macOS request to the workflows topic, 159.5KB total. Groundwork for phase 2 of the measurement campaign, where the stress project gains more workflows and offerings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkCommand.swift | 2 +- .../Sources/BenchmarkMetrics.swift | 62 ++++++++++++------- .../Sources/BenchmarkRunner.swift | 31 ++++++++-- .../Sources/BenchmarkSDKStack.swift | 12 +++- .../Tests/BenchmarkMetricsTests.swift | 25 +++++++- 5 files changed, 101 insertions(+), 31 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index b999136eaa..6b01732dde 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -78,7 +78,7 @@ struct BenchmarkCommand { "post_warmup_error_count", "mean_ms", "min_ms", "max_ms", "p50_ms", "p90_ms", "p95_ms", "p99_ms", "request_count_mean", "bytes_received_mean", "failed_requests_total", "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", - "blobs_inline_mean", "blobs_downloaded_mean", "blob_bytes_mean", "max_inline_blob_bytes", + "blobs_inline_mean", "blobs_downloaded_mean", "blob_bytes_mean", "blobs_by_topic", "max_inline_blob_bytes", "min_downloaded_blob_bytes", "config_persisted_ms_mean", "last_blob_stored_ms_mean", "first_error", "project_id" ] diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift index 80e0873e79..74591f0325 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -6,28 +6,37 @@ import Foundation /// smallest blob that needed a download. struct BlobAccounting { + /// Label for stored refs the persisted topic index does not reference. Should be empty + /// in practice (`retainOnly` prunes them); nonzero is itself a signal worth seeing. + static let unreferencedTopic = "unreferenced" + let inlineCount: Int let downloadedCount: Int let totalBytes: Int let maxInlineBytes: Int? let minDownloadedBytes: Int? + /// Newly stored blobs grouped by the topic that references them (e.g. "workflows", + /// "ui_config"), so rows say WHAT was fetched, not just how much. + let countsByTopic: [String: Int] + let bytesByTopic: [String: Int] - static let empty = BlobAccounting( - inlineCount: 0, - downloadedCount: 0, - totalBytes: 0, - maxInlineBytes: nil, - minDownloadedBytes: nil - ) + static let empty = BlobAccounting(newRefSizes: [:], downloadedRefs: []) /// Attributes newly stored refs: a new ref that has a successful blob request this /// iteration was downloaded; any other new ref can only have come from the container. - init(newRefSizes: [String: Int], downloadedRefs: Set) { + /// `topicByRef` (from the persisted topic index) labels each ref with its topic. + init( + newRefSizes: [String: Int], + downloadedRefs: Set, + topicByRef: [String: String] = [:] + ) { var inlineCount = 0 var downloadedCount = 0 var totalBytes = 0 var maxInlineBytes: Int? var minDownloadedBytes: Int? + var countsByTopic: [String: Int] = [:] + var bytesByTopic: [String: Int] = [:] for (ref, byteCount) in newRefSizes { totalBytes += byteCount @@ -38,6 +47,9 @@ struct BlobAccounting { inlineCount += 1 maxInlineBytes = max(maxInlineBytes ?? .min, byteCount) } + let topic = topicByRef[ref] ?? Self.unreferencedTopic + countsByTopic[topic, default: 0] += 1 + bytesByTopic[topic, default: 0] += byteCount } self.inlineCount = inlineCount @@ -45,20 +57,8 @@ struct BlobAccounting { self.totalBytes = totalBytes self.maxInlineBytes = maxInlineBytes self.minDownloadedBytes = minDownloadedBytes - } - - private init( - inlineCount: Int, - downloadedCount: Int, - totalBytes: Int, - maxInlineBytes: Int?, - minDownloadedBytes: Int? - ) { - self.inlineCount = inlineCount - self.downloadedCount = downloadedCount - self.totalBytes = totalBytes - self.maxInlineBytes = maxInlineBytes - self.minDownloadedBytes = minDownloadedBytes + self.countsByTopic = countsByTopic + self.bytesByTopic = bytesByTopic } } @@ -222,6 +222,24 @@ struct BenchmarkMetrics { aggregates["min_downloaded_blob_bytes"] = minDownloaded } + // Per-topic attribution: what was fetched, not just how much. Mean per iteration. + var topicCounts: [String: Int] = [:] + var topicBytes: [String: Int] = [:] + for iteration in measured { + topicCounts.merge(iteration.blobs.countsByTopic, uniquingKeysWith: +) + topicBytes.merge(iteration.blobs.bytesByTopic, uniquingKeysWith: +) + } + if !topicCounts.isEmpty { + var byTopic: [String: [String: Double]] = [:] + for topic in topicCounts.keys { + byTopic[topic] = [ + "count_mean": Self.rounded(Double(topicCounts[topic] ?? 0) / count), + "bytes_mean": Self.rounded(Double(topicBytes[topic] ?? 0) / count) + ] + } + aggregates["blobs_by_topic"] = byTopic + } + return aggregates } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift index 7e10372f03..3301a78a70 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -154,7 +154,8 @@ private extension BenchmarkRunner { blobAccounting: Self.blobAccounting( blobStore: stack.blobStore, refsBeforeLaunch: blobRefsBeforeLaunch, - events: events + events: events, + topics: stack.remoteConfigDiskCache?.read()?.topics ) ) } @@ -235,11 +236,13 @@ extension BenchmarkRunner { /// Attributes this iteration's newly stored blobs (all disk reads happen after the timed /// window): a new ref with a successful blob request was downloaded; any other new ref can - /// only have arrived inline in the config container. + /// only have arrived inline in the config container. `topics` (the persisted topic index) + /// labels each ref with the topic that references it. static func blobAccounting( blobStore: RemoteConfigBlobStoreType?, refsBeforeLaunch: Set, - events: [TransportEvent] + events: [TransportEvent], + topics: RemoteConfiguration.Topics? = nil ) -> BlobAccounting { guard let blobStore else { return .empty } @@ -254,7 +257,27 @@ extension BenchmarkRunner { let newRefSizes = newRefs.reduce(into: [String: Int]()) { sizes, ref in sizes[ref] = blobStore.read(ref: ref)?.count ?? 0 } - return BlobAccounting(newRefSizes: newRefSizes, downloadedRefs: downloadedRefs) + return BlobAccounting( + newRefSizes: newRefSizes, + downloadedRefs: downloadedRefs, + topicByRef: Self.topicByRef(from: topics) + ) + } + + /// Inverts the persisted topic index into ref → topic name. A ref referenced by several + /// topics keeps the alphabetically first, deterministically. + static func topicByRef(from topics: RemoteConfiguration.Topics?) -> [String: String] { + guard let topics else { return [:] } + + var topicByRef: [String: String] = [:] + for (topicName, items) in topics.entries { + for item in items.values { + guard let ref = item.blobRef else { continue } + if let existing = topicByRef[ref], existing <= topicName { continue } + topicByRef[ref] = topicName + } + } + return topicByRef } /// Cold config iterations must have actually completed the config exchange their mode diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift index 74c3a114d2..94f9ecf397 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift @@ -15,6 +15,9 @@ final class BenchmarkSDKStack { /// The manager's content-addressed blob store, exposed so the runner can attribute /// per-iteration blob storage (inline vs downloaded) and sizes. Nil in legacy mode. let blobStore: RemoteConfigBlobStoreType? + /// The manager's disk cache, exposed so the runner can join stored blob refs against + /// the persisted topic index (which topic each blob belongs to). Nil in legacy mode. + let remoteConfigDiskCache: RemoteConfigDiskCacheType? let httpClient: HTTPClient let deviceCache: DeviceCache @@ -64,6 +67,7 @@ final class BenchmarkSDKStack { let remoteConfigManager = remoteConfigStack?.manager self.remoteConfigManager = remoteConfigManager self.blobStore = remoteConfigStack?.blobStore + self.remoteConfigDiskCache = remoteConfigStack?.diskCache self.offeringsManager = OfferingsManager( deviceCache: deviceCache, @@ -114,7 +118,11 @@ private extension BenchmarkSDKStack { static func makeRemoteConfigManager( backendConfiguration: BackendConfiguration, appUserID: String - ) -> (manager: RemoteConfigManagerType, blobStore: RemoteConfigBlobStoreType) { + ) -> ( + manager: RemoteConfigManagerType, + blobStore: RemoteConfigBlobStoreType, + diskCache: RemoteConfigDiskCacheType + ) { let diskCache = RemoteConfigDiskCache() let blobStore = RemoteConfigBlobStore() let sourceProvider = RemoteConfigSourceProvider(topicStore: diskCache) @@ -132,7 +140,7 @@ private extension BenchmarkSDKStack { blobFetcher: blobFetcher, currentUserProvider: BenchmarkCurrentUserProvider(appUserID: appUserID) ) - return (manager, blobStore) + return (manager, blobStore, diskCache) } } diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift index 1a6ed6a889..126380af0f 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -163,6 +163,19 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { XCTAssertEqual(accounting.minDownloadedBytes, 9_000) } + func testBlobAccountingAttributesRefsToTopics() { + // Which topic each stored blob belongs to, from the persisted topic index; a stored + // ref no topic references is labeled explicitly rather than dropped. + let accounting = BlobAccounting( + newRefSizes: ["wf-1": 40_000, "wf-2": 30_000, "ui-app": 5_000, "mystery": 100], + downloadedRefs: ["ui-app"], + topicByRef: ["wf-1": "workflows", "wf-2": "workflows", "ui-app": "ui_config"] + ) + + XCTAssertEqual(accounting.countsByTopic, ["workflows": 2, "ui_config": 1, "unreferenced": 1]) + XCTAssertEqual(accounting.bytesByTopic, ["workflows": 70_000, "ui_config": 5_000, "unreferenced": 100]) + } + func testJSONLRowCarriesBlobAccountingAggregates() throws { var command = BenchmarkCommand() command.iterations = 2 @@ -170,10 +183,12 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { var metrics = BenchmarkMetrics() metrics.record(self.measurement(totalMs: 10, blobAccounting: BlobAccounting( - newRefSizes: ["i1": 1_000, "d1": 4_000], downloadedRefs: ["d1"] + newRefSizes: ["i1": 1_000, "d1": 4_000], downloadedRefs: ["d1"], + topicByRef: ["i1": "workflows", "d1": "ui_config"] )), iteration: 0) metrics.record(self.measurement(totalMs: 20, blobAccounting: BlobAccounting( - newRefSizes: ["i2": 3_000, "d2": 2_000], downloadedRefs: ["d2"] + newRefSizes: ["i2": 3_000, "d2": 2_000], downloadedRefs: ["d2"], + topicByRef: ["i2": "workflows", "d2": "ui_config"] )), iteration: 1) let row = try self.decodeRow(metrics, command: command) @@ -183,6 +198,12 @@ final class BenchmarkMetricsTests: BenchmarkTestCase { XCTAssertEqual(row["blob_bytes_mean"] as? Double, 5_000) XCTAssertEqual(row["max_inline_blob_bytes"] as? Int, 3_000) XCTAssertEqual(row["min_downloaded_blob_bytes"] as? Int, 2_000) + + let byTopic = try XCTUnwrap(row["blobs_by_topic"] as? [String: [String: Double]]) + XCTAssertEqual(byTopic["workflows"]?["count_mean"], 1) + XCTAssertEqual(byTopic["workflows"]?["bytes_mean"], 2_000) + XCTAssertEqual(byTopic["ui_config"]?["count_mean"], 1) + XCTAssertEqual(byTopic["ui_config"]?["bytes_mean"], 3_000) } func testJSONLRowOmitsBlobExtremesWhenNoBlobsWereStored() throws { From fff4dccd18f376f84cfe929233245bfe59d15c97 Mon Sep 17 00:00:00 2001 From: Facundo Menzella Date: Thu, 9 Jul 2026 16:37:52 +0200 Subject: [PATCH 34/34] fix(benchmarks): retry launches that produce nothing, once, with a visible budget At hundreds of iterations, a ~1% harness flake (the result element never appearing within 90s) is expected, and the zero-error gate was voiding whole 35-minute rounds over 2 censored launches out of 250. A launch that produces nothing is now relaunched once (a fresh, unbiased sample); the retry count lands in the row as launch_retries so censored launches stay visible, and the runner fails the round when retries exceed ~2% of iterations or a retry also fails. Samples that report a real error are still never retried. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UXR3hncr2daCqrnXdU5N4H --- .../Sources/BenchmarkCommand.swift | 1 + .../Tests/AppLaunchMetricsTests.swift | 10 ++++ .../UITests/AppLaunchBenchmarkUITests.swift | 58 ++++++++++++++++--- .../UITests/AppLaunchMetrics.swift | 8 ++- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift index 6b01732dde..5213039a63 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -80,6 +80,7 @@ struct BenchmarkCommand { "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", "blobs_inline_mean", "blobs_downloaded_mean", "blob_bytes_mean", "blobs_by_topic", "max_inline_blob_bytes", "min_downloaded_blob_bytes", "config_persisted_ms_mean", "last_blob_stored_ms_mean", + "launch_retries", "first_error", "project_id" ] diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift index 9952636933..cbbd3bdb1a 100644 --- a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -107,6 +107,16 @@ final class AppLaunchMetricsTests: BenchmarkTestCase { XCTAssertTrue(firstError.contains("BENCH_API_KEY missing")) } + func testRowCarriesLaunchRetries() throws { + let row = AppLaunchMetrics.row( + mode: "app-launch-config", scenario: "cold", profile: "simulator", + projectID: "p", warmupDiscarded: 0, + samples: [Self.sample(100)], launchRetries: 2 + ) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(row.utf8)) as? [String: Any]) + XCTAssertEqual(object["launch_retries"] as? Int, 2) + } + func testWarmupWindowErrorsAreCountedButNotPostWarmup() throws { var failed = Self.sample(50) failed.error = "boom" diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift index e87890e33c..0f98c38ef3 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift @@ -46,15 +46,17 @@ final class AppLaunchBenchmarkUITests: XCTestCase { let configuration = try self.configuration() // A fresh user and wiped state every launch: each iteration is a true cold start. + var retries = 0 let samples = (0..90s stall). At hundreds of iterations a ~1% harness flake + /// is expected; a retry is a fresh, unbiased sample, and the retry COUNT lands in the + /// row so censored launches stay visible. A sample that reports an error is NOT + /// retried: that is a real measured failure. + private func launchCollectingWithRetry( + configuration: Configuration, + appUserID: String, + wipeState: Bool, + retries: inout Int + ) -> LaunchSample? { + if let sample = self.launchAndCollect( + configuration: configuration, appUserID: appUserID, wipeState: wipeState + ) { + return sample + } + retries += 1 + return self.launchAndCollect( + configuration: configuration, appUserID: appUserID, wipeState: wipeState + ) + } + private func launchAndCollect( configuration: Configuration, appUserID: String, @@ -126,14 +156,28 @@ final class AppLaunchBenchmarkUITests: XCTestCase { return LaunchSample.decode(from: result.label) } - private func report(samples: [LaunchSample?], scenario: String, configuration: Configuration) { + private func report( + samples: [LaunchSample?], + scenario: String, + configuration: Configuration, + retries: Int + ) { + // A retried launch is a fresh sample, but the retry budget stays small: pervasive + // retries mean the environment is unstable and the whole round is suspect. + let retryBudget = max(1, configuration.iterations / 50) + XCTAssertLessThanOrEqual( + retries, retryBudget, + "\(retries) launches produced nothing and were retried (budget \(retryBudget)); environment unstable" + ) + let row = AppLaunchMetrics.row( mode: configuration.modeLabel, scenario: scenario, profile: Self.profile, projectID: configuration.projectID, warmupDiscarded: configuration.warmup, - samples: samples + samples: samples, + launchRetries: retries ) // The script greps this prefix out of the xcodebuild log to build the JSONL file. diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift index c3bb6fb9bf..a17f5606f5 100644 --- a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift @@ -29,7 +29,8 @@ enum AppLaunchMetrics { profile: String, projectID: String, warmupDiscarded: Int, - samples: [LaunchSample?] + samples: [LaunchSample?], + launchRetries: Int = 0 ) -> String { let indexed = samples.enumerated() let errors: [(index: Int, message: String)] = indexed.compactMap { index, sample in @@ -61,7 +62,10 @@ enum AppLaunchMetrics { "project_id": projectID, "measured_iterations": measured.count, "error_count": errors.count, - "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count + "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count, + // Launches that produced nothing and were relaunched once (censored samples); + // visible so a round's stability can be judged, bounded by the runner. + "launch_retries": launchRetries ] // Headline statistic: `Purchases.configure` + `getOfferings` completed, with or