diff --git a/.gitignore b/.gitignore index 77482667db..eb9dcd018f 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,7 @@ Tests/TestingApps/PaywallsTester/PaywallsTester/PaywallsTester.storekit # Claude Code .claude/ + +# Python bytecode (e.g. from running the benchmark compare.py) +__pycache__/ +*.pyc diff --git a/Dangerfile b/Dangerfile index 01ec6063b8..19e7e1928e 100644 --- a/Dangerfile +++ b/Dangerfile @@ -4,7 +4,11 @@ require 'pathname' require 'set' EXCLUDED_SWIFT_PATH_PREFIXES = [ - 'Tests/APITesters/' + 'Tests/APITesters/', + # Tuist-only benchmark targets (Projects/SDKConfigBenchmark and + # Projects/SDKConfigBenchmarkApp); intentionally absent from RevenueCat.xcodeproj. + 'Tests/Benchmarks/', + 'Tests/TestingApps/SDKConfigBenchmarkApp/' ].freeze COMMENT_MARKER = "" diff --git a/Projects/SDKConfigBenchmark/Project.swift b/Projects/SDKConfigBenchmark/Project.swift new file mode 100644 index 0000000000..112b148e48 --- /dev/null +++ b/Projects/SDKConfigBenchmark/Project.swift @@ -0,0 +1,104 @@ +import ProjectDescription +import ProjectDescriptionHelpers + +// Benchmark harness for comparing the legacy offerings flow against the remote config +// endpoint flow. Compiles the SDK sources directly (like BinarySizeTest does for its +// local-source mode) so the benchmark can drive internal manager-level APIs without +// exposing new public SDK API. `SDK_CONFIG_BENCHMARK` installs a simulated transport +// in `HTTPClient`; `ENABLE_REMOTE_CONFIG` turns on the remote config gate. +// +// TUIST_SWIFT_CONDITIONS is folded in by hand instead of via +// `appendingTuistSwiftConditions()`, because that helper replaces the whole +// SWIFT_ACTIVE_COMPILATION_CONDITIONS value and would silently drop the benchmark flags. +let benchmarkSwiftConditions: SettingsDictionary = [ + "SWIFT_ACTIVE_COMPILATION_CONDITIONS": SettingValue(stringLiteral: ( + ["$(inherited)", "SDK_CONFIG_BENCHMARK", "ENABLE_REMOTE_CONFIG"] + Environment.extraSwiftConditions + ).joined(separator: " ")) +] + +let project = Project( + name: "SDKConfigBenchmark", + organizationName: .revenueCatOrgName, + settings: .framework, + targets: [ + .target( + name: "SDKConfigBenchmarkCore", + destinations: [.mac], + product: .staticLibrary, + bundleId: "com.revenuecat.SDKConfigBenchmarkCore", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Sources/**/*.swift", + .glob( + "../../Sources/**/*.swift", + excluding: [ + "../../Sources/LocalReceiptParsing/ReceiptParser-only-files/**/*.swift" + ] + ) + ], + dependencies: [ + .storeKit + ], + settings: .settings( + base: benchmarkSwiftConditions + ) + ), + .target( + name: "SDKConfigBenchmark", + destinations: [.mac], + product: .commandLineTool, + bundleId: "com.revenuecat.SDKConfigBenchmark", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Main/**/*.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkCore") + ], + settings: .settings( + // The SDK's disk caches derive their directory from Bundle.main.bundleIdentifier, + // which a bare command-line binary does not have. Embedding the Info.plist into + // the binary gives it one, so etags/offerings/remote-config persistence works. + base: benchmarkSwiftConditions + .merging(["CREATE_INFOPLIST_SECTION_IN_BINARY": "YES"]) + ) + ), + .target( + name: "SDKConfigBenchmarkTests", + destinations: [.mac], + product: .unitTests, + bundleId: "com.revenuecat.SDKConfigBenchmarkTests", + deploymentTargets: .macOS("13.0"), + infoPlist: .default, + sources: [ + "../../Tests/Benchmarks/SDKConfigBenchmark/Tests/**/*.swift", + // App-host tier helpers, shared into this target so their aggregation + // logic is unit-testable without booting a simulator. + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift", + "../../Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkCore") + ], + settings: .settings( + base: benchmarkSwiftConditions + ) + ) + ], + schemes: [ + .scheme( + name: "SDKConfigBenchmark", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmark"]), + runAction: .runAction(configuration: "Release") + ), + .scheme( + name: "SDKConfigBenchmarkTests", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmarkTests"]), + testAction: .targets(["SDKConfigBenchmarkTests"]) + ) + ] +) diff --git a/Projects/SDKConfigBenchmarkApp/Project.swift b/Projects/SDKConfigBenchmarkApp/Project.swift new file mode 100644 index 0000000000..7b02bf043f --- /dev/null +++ b/Projects/SDKConfigBenchmarkApp/Project.swift @@ -0,0 +1,85 @@ +import ProjectDescription +import ProjectDescriptionHelpers + +// App-host tier of the SDK config benchmark: a minimal iOS app that links the stock +// RevenueCat/RevenueCatUI products (unlike Projects/SDKConfigBenchmark, which compiles the +// SDK sources directly) and measures a real `Purchases.configure` launch end to end. +// Local/CI only; driven by Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh. +// +// The legacy-vs-config variant switch is NOT set here: the SPM-built RevenueCat reads +// `SWIFT_ACTIVE_COMPILATION_CONDITIONS` from Local.xcconfig (see Package.swift), which the +// runner script rewrites per variant. + +// Injected into the scheme's run environment so the app can be launched straight from Xcode: +// TUIST_BENCH_API_KEY= tuist generate SDKConfigBenchmarkApp +// Without it, a direct run reports "BENCH_API_KEY missing" by design (keys never live in +// source or in committed scheme files). +let runEnvironment: [String: EnvironmentVariable] = { + var environment: [String: EnvironmentVariable] = [:] + if let apiKey = Environment.benchApiKey { + environment["BENCH_API_KEY"] = .environmentVariable(value: apiKey, isEnabled: true) + environment["BENCH_APP_USER_ID"] = .environmentVariable(value: "bench-xcode-run", isEnabled: true) + } + return environment +}() + +let project = Project( + name: "SDKConfigBenchmarkApp", + organizationName: .revenueCatOrgName, + packages: .projectPackages, + settings: .appProject, + targets: [ + .target( + name: "SDKConfigBenchmarkApp", + destinations: [.iPhone], + product: .app, + bundleId: "com.revenuecat.SDKConfigBenchmarkApp", + deploymentTargets: .iOS("16.0"), + infoPlist: "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist", + sources: [ + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/**/*.swift" + ], + dependencies: [ + .revenueCat, + .revenueCatUI + ], + settings: .appTarget(including: ([:] as SettingsDictionary).appendingTuistSwiftConditions()) + ), + .target( + name: "SDKConfigBenchmarkAppUITests", + destinations: [.iPhone], + product: .uiTests, + bundleId: "com.revenuecat.SDKConfigBenchmarkAppUITests", + deploymentTargets: .iOS("16.0"), + infoPlist: .default, + sources: [ + "../../Tests/TestingApps/SDKConfigBenchmarkApp/UITests/**/*.swift", + // Shared with the app target: the runner decodes the same LaunchSample + // the app encodes. + "../../Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift" + ], + dependencies: [ + .target(name: "SDKConfigBenchmarkApp") + ], + settings: .appTarget + ) + ], + schemes: [ + .scheme( + name: "SDKConfigBenchmarkApp", + shared: true, + buildAction: .buildAction(targets: ["SDKConfigBenchmarkApp"]), + // Release: the rows claim to measure the shipping SDK, so Debug-built + // (unoptimized, assertion-enabled) frameworks would misrepresent it. + testAction: .targets( + ["SDKConfigBenchmarkAppUITests"], + configuration: "Release" + ), + runAction: .runAction( + configuration: "Debug", + executable: "SDKConfigBenchmarkApp", + arguments: .arguments(environmentVariables: runEnvironment) + ) + ) + ] +) diff --git a/Sources/Caching/DeviceCache.swift b/Sources/Caching/DeviceCache.swift index 4911da3bd8..ff1b15e7bf 100644 --- a/Sources/Caching/DeviceCache.swift +++ b/Sources/Caching/DeviceCache.swift @@ -856,6 +856,9 @@ private extension DeviceCache { // swiftlint:disable avoid_using_directory_apis_directly private func oldDocumentsDirectoryURL() -> URL? { + #if SDK_CONFIG_BENCHMARK + guard !DirectoryHelper.skipsLegacyDocumentsMigrations else { return nil } + #endif let documentsDirectoryURL: URL? if #available(iOS 16.0, macOS 13.0, tvOS 16.0, watchOS 9.0, *) { documentsDirectoryURL = URL.documentsDirectory diff --git a/Sources/Caching/DirectoryHelper.swift b/Sources/Caching/DirectoryHelper.swift index 2b4a0981c9..b80123ffe4 100644 --- a/Sources/Caching/DirectoryHelper.swift +++ b/Sources/Caching/DirectoryHelper.swift @@ -37,7 +37,35 @@ enum DirectoryHelper { return Self.baseUrl(for: directoryType, inAppSpecificDirectory: false) } + #if SDK_CONFIG_BENCHMARK + /// Benchmark-only: when set, every SDK disk cache resolves under this root. The standard + /// container directories ignore $HOME, so without this override concurrent benchmark + /// processes would share (and corrupt) one another's caches in the real user Library. + /// Set once at process startup, before any disk access. + static var benchmarkBaseDirectoryOverride: URL? + + /// While the override is active, legacy Documents-directory migrations must not run: + /// they read, move, and delete files under the real user Documents folder, outside the + /// benchmark's sandbox. + static var skipsLegacyDocumentsMigrations: Bool { + return self.benchmarkBaseDirectoryOverride != nil + } + #endif + static func baseUrl(for type: DirectoryType, inAppSpecificDirectory: Bool = true) -> URL? { + #if SDK_CONFIG_BENCHMARK + if let overrideRoot = Self.benchmarkBaseDirectoryOverride { + switch type { + case .cache: + return overrideRoot.appendingPathComponent("caches") + #if !os(tvOS) + case .applicationSupport: + return overrideRoot.appendingPathComponent("application-support") + #endif + } + } + #endif + guard let baseDirectory = type.url else { return nil } diff --git a/Sources/Networking/HTTPClient/ETagManager.swift b/Sources/Networking/HTTPClient/ETagManager.swift index bc799a4e7c..1bfb8a52c8 100644 --- a/Sources/Networking/HTTPClient/ETagManager.swift +++ b/Sources/Networking/HTTPClient/ETagManager.swift @@ -205,6 +205,9 @@ private extension ETagManager { } private func oldETagDirectoryURL() -> URL? { + #if SDK_CONFIG_BENCHMARK + guard !DirectoryHelper.skipsLegacyDocumentsMigrations else { return nil } + #endif // swiftlint:disable:next avoid_using_directory_apis_directly guard let documentsURL = Self.fileManager.urls( for: .documentDirectory, diff --git a/Sources/Networking/HTTPClient/HTTPClient.swift b/Sources/Networking/HTTPClient/HTTPClient.swift index 6a67d5709a..ad6e33f111 100644 --- a/Sources/Networking/HTTPClient/HTTPClient.swift +++ b/Sources/Networking/HTTPClient/HTTPClient.swift @@ -58,6 +58,9 @@ class HTTPClient { config.timeoutIntervalForRequest = requestTimeout config.timeoutIntervalForResource = requestTimeout config.urlCache = nil // We implement our own caching with `ETagManager`. + #if SDK_CONFIG_BENCHMARK + config.protocolClasses = [SimulatedTransportURLProtocol.self] + #endif self.session = URLSession(configuration: config, delegate: RedirectLoggerSessionDelegate(), delegateQueue: nil) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md new file mode 100644 index 0000000000..ae2cda7c1a --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/CONFIG_ENDPOINT_BENCHMARKS.md @@ -0,0 +1,313 @@ +# SDK Config Endpoint Benchmarks + +Last updated: 2026-07-08 + +## Goal + +Produce reproducible, SDK-side numbers comparing the **current offerings system** with the +**config endpoint system** (`/v1/config/{domain}` + content-addressed blobs), so the config +endpoint plus workflows can be released with data instead of guesses. The harness also measures +the **kill switch**: what a launch costs when `/v1/config` is force-failed with a 4xx and the +SDK falls back to the legacy path. + +This is not a backend stress test. It measures the iOS SDK's fetch, cache, revalidation, and +decode behavior under controlled network conditions. + +## What the benchmark actually drives + +Each iteration is one simulated app launch through the **real manager-level SDK code**: + +- **`legacy`**: `OfferingsManager.offerings(appUserID:)` with `remoteConfigManager: nil`, the + exact wiring `Purchases` uses when the remote config gate is off. One + `GET /v1/subscribers/{id}/offerings` request. +- **`config`**: the full production stack: `RemoteConfigManager` (with `RemoteConfigDiskCache`, + `RemoteConfigBlobStore`, `RemoteConfigSourceProvider`, `RemoteConfigBlobFetcher`), refreshed + at launch like `updateAllCaches` does, plus the same `OfferingsManager` fetch. Offerings + delivery goes through `deliverWhenConfigReady`, so the measured time includes waiting for the + `workflows` topic and its prefetch-marked blobs, exactly as production would. +- **`config-killswitch`**: identical wiring to `config`, but the fixture returns `400` for + `/v1/config`, tripping `RemoteConfigManager`'s session kill switch. Measures what the switch + costs users: the wasted config round trip plus the legacy fetch. + +Timing stops when the offerings completion fires. That is the product-shaped metric ("time +until offerings are usable"), not "time for N HTTP requests". + +Payloads are generated by `BenchmarkPayloadFactory` and are validated by unit tests to decode +into the real SDK models (`OfferingsResponse`, `RemoteConfiguration` inside a binary +RC-Container, `PublishedWorkflow` blobs, `ui_config` blobs) with content-addressed blob refs +that pass the SDK's checksum validation. StoreKit is replaced by a `ProductsManagerType` fake; +everything else is production code. + +## Transports + +- **`--transport simulated`** (default): the in-process network model below. Deterministic, + seedable, and the only way to measure loss, degraded profiles, and the forced kill-switch + 4xx. Use it for controlled A/B comparisons of SDK behavior and for CI regression tracking. +- **`--transport live`**: real requests against production, defaulting to the prepared + stress-test project [`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3) ("Stress Test + Config Endpoint"). No keys live in source: `run-matrix.sh` resolves the project's key via + mafdet (Test Store app preferred, since the project's packages live there); direct binary + runs take `--api-key` or `SDK_CONFIG_BENCHMARK_API_KEY`. Requests flow through a recording + passthrough, so live rows carry the same per-request metrics: API traffic re-issues through a + single-connection pool mirroring `HTTPClient`, blob traffic through a default pool mirroring + the production blob downloader. Use it for real-world numbers (CDN, TLS, actual backend + latency) and for monitoring the two systems against real project content. + +## Network model + +All requests are intercepted in-process by `SimulatedTransportURLProtocol` (every host, +including the SDK's fallback hosts, so nothing can leak to the real network). Responses are +delivered asynchronously with: + +- **Per-class round-trip times**: dynamic API endpoints and CDN blob endpoints get separate RTT + ranges, because the config system's premise is that blobs are cheap CDN fetches while + `/offerings` and `/v1/config` are dynamic backend calls. A flat per-request latency would + structurally bias against whichever mode makes more requests. +- **Bandwidth-paced chunked bodies**, so payload size costs transfer time. +- **A seeded loss model**: per body chunk, `loss%` chance of one retransmission delay (1x-2x + RTT); per request, `(loss/100)^3` chance of failing with a timeout after one RTO, which + exercises the SDK's retry and fallback-host logic. + +Profiles: `ideal` (0ms, unlimited), `wifi` (25-40ms API / 10-20ms CDN, ~50 Mbit/s), `lte` +(55-110ms API / 30-70ms CDN, ~12 Mbit/s). + +**Determinism contract.** Every request's network plan (RTT, failure, chunk delays) is derived +from a stable key: seed, iteration, URL, and per-URL attempt. Request arrival order, which is +scheduler-dependent, cannot influence sampling. Measured on this machine: + +- Loss-free rows are exactly reproducible across processes for the same seed: identical + request counts, bytes, and failure counts; timings differ only by wall-clock jitter + (a few percent, which the warmup discard and percentiles absorb). +- Rows with loss keep deterministic per-request plans (identical failure and fallback-host + counts across processes), but the SDK's concurrent reaction to failures (blob-fetcher + failover and retry scheduling) is real thread-scheduling behavior, so request counts can + drift by about one request and timings by a few percent. That residual variation is the + system under test, not the harness, and faking it away would distort the measurement. + +Each benchmark process isolates all SDK disk caches under its own temporary root (removed on +exit); concurrent runs cannot corrupt each other and nothing touches the real user Library. +One caveat: the scratch `UserDefaults` suite is per-user, not per-process, but it only carries +cache timestamps and identity bookkeeping, never response bodies or ETags. + +**Honest caveat:** the loss model approximates TCP behavior; it does not simulate real packet +loss (retransmission dynamics, congestion control, TLS handshakes). It is good enough to +compare the two systems under identical degraded conditions. For absolute numbers under real +loss, use a local HTTP server plus the OS Network Link Conditioner. + +## Scenarios + +- **`cold`**: every iteration wipes ETags, the offerings disk cache, and the persisted remote + config + blob store, and uses a fresh app user ID. First-launch behavior. +- **`warm`**: disk state is primed once, then every iteration builds a fresh in-memory stack + against the retained disk state, simulating an app relaunch. The fixture honors the SDK's + actual revalidation mechanisms: offerings revalidate via `X-RevenueCat-ETag` -> `304`, config + revalidates via its manifest token -> `204`, and blobs are content-addressed so they are never + re-downloaded. The runner **fails loudly** unless every measured warm iteration revalidates + (offerings all-304, config all-204 in config mode, zero blob re-downloads), so the scenario + can't silently mix cold behavior into the warm distribution. + +The first `--warmup-iterations` (default 3) measured iterations are discarded from statistics +so one-time process costs don't pollute the distribution. + +## Running + +```sh +bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py results.jsonl +``` + +Run the same command on a baseline branch and a candidate branch, then: + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py baseline.jsonl candidate.jsonl +``` + +Every JSONL row carries the full configuration (mode, scenario, profile, loss, counts, seed) +plus `sdk_commit`, so result files are self-describing. + +## Sample numbers + +Default matrix (25 iterations, 3 warmup discarded, 50 paywalls, 100 workflows, seed 42), +macOS Release build: + +| mode | scenario | profile | loss % | p50 ms | p95 ms | requests (mean) | bytes (mean) | +|---|---|---|---:|---:|---:|---:|---:| +| legacy | cold | ideal | 0 | 6.6 | 7.3 | 1 | 64,972 | +| config | cold | ideal | 0 | 30.8 | 35.1 | 102 | 132,550 | +| config-killswitch | cold | ideal | 0 | 6.9 | 7.7 | 2 | 65,022 | +| legacy | cold | lte | 0 | 146.5 | 176.5 | 1 | 64,972 | +| config | cold | lte | 0 | 1,653.2 | 1,735.7 | 102 | 132,550 | +| config-killswitch | cold | lte | 0 | 239.8 | 280.0 | 2 | 65,022 | +| legacy | warm | ideal | 0 | 7.0 | 7.2 | 1 | 0 | +| config | warm | ideal | 0 | 8.1 | 8.5 | 2 | 0 | +| config-killswitch | warm | ideal | 0 | 7.0 | 7.8 | 2 | 50 | +| legacy | warm | lte | 0 | 102.8 | 131.4 | 1 | 0 | +| config | warm | lte | 0 | 108.4 | 133.2 | 2 | 0 | +| config-killswitch | warm | lte | 0 | 198.9 | 237.8 | 2 | 50 | +| legacy | cold | lte | 10 | 158.3 | 296.2 | 1 | 64,972 | +| config | cold | lte | 10 | 1,948.1 | 2,577.2 | 102 | 132,503 | +| legacy | cold | lte | 20 | 169.5 | 300.7 | 1 | 64,972 | +| config | cold | lte | 20 | 2,352.9 | 3,128.8 | 96 | 128,778 | +| legacy | cold | lte | 30 | 222.5 | 630.2 | 1 | 64,972 | +| config | cold | lte | 30 | 2,828.5 | 3,802.0 | 74 | 116,852 | + +Every warm row above is verified per iteration: offerings revalidated via 304, config via +manifest 204 with the full prefetched-blob proof (or the expected 4xx in kill-switch mode), +and zero blob re-downloads. Launches fire the exact `updateAllCaches` pair in production +order (offerings enqueued before the config refresh), and each iteration's accounting +includes trailing requests that finish after offerings delivery (like the warm config 204). + +### Live sample numbers + +`TRANSPORT=live` matrix (25 iterations, 3 warmup discarded) against the stress-test project, +run 2026-07-09 from a residential connection: + +| mode | scenario | p50 ms | p95 ms | requests (mean) | bytes (mean) | +|---|---|---:|---:|---:|---:| +| legacy | cold | 133.4 | 331.4 | 1 | 32,124 | +| config | cold | 280.0 | 564.9 | 2 | 36,052 | +| legacy | warm | 135.1 | 329.0 | 1 | 0 | +| config | warm | 142.2 | 398.9 | 2 | 0 | + +Live observations (as of this run): + +- The project currently publishes **one workflow**, and khepri delivers its blob **inline in + the RC-Container** (config response ~3.9KB vs ~1.2KB of config JSON), so cold launches make + 2 requests and no CDN fetches. The full chain still runs: inline extraction -> blob store -> + offerings delivery gated on the prefetched workflow -> warm 204 with the blob proof. The + **CDN download path** (`config.revenuecat-static.com` + failover) only engages once enough + workflows/paywalls are published that the response stops inlining; live runs pick that up + automatically with no harness changes. +- **Warm config launches cost almost nothing extra** (142ms vs 135ms p50): offerings delivery + does not wait for the config revalidation when the topics are already on disk, so the 204 + trails after delivery, exactly like production. Cold config pays the full extra round trip + (280ms vs 133ms) because first delivery waits for the config sync. +- Warm revalidation works end to end against production: offerings answers `304` to + `X-RevenueCat-ETag` and config answers `204` to a matching manifest + prefetched-blob set, + both with zero content bytes. The runner verifies every measured iteration and fails the + run if any falls back to full responses. + +## Interpretation + +- **Prefetch gating dominates config cold starts.** With 100 workflows all marked + `prefetch: true`, first offerings delivery waits for ~100 blob downloads through the + fetcher's 4-concurrent cap: ~25 sequential batches at LTE CDN RTTs is the entire gap to + legacy (1,653ms vs 147ms p50). This is a **fixture policy worst case**, not an inherent + cost of the config system; the real product decision is prefetch scope (current offering + only? top N?), and this harness can now measure each option. +- **Warm launches are where the config design pays off.** A warm config relaunch delivers + offerings essentially as fast as legacy (108ms vs 103ms p50 on LTE): the topics are already + on disk, so delivery doesn't wait for the manifest 204, which revalidates in the background. + Content bytes are zero and blobs are never re-downloaded. +- **The kill switch is cheap.** Force-failing `/v1/config` with a 4xx costs roughly one API + round trip over pure legacy on a cold LTE launch (240ms vs 147ms p50), after which the + session behaves like legacy. No cascading retries, no blob traffic. Warm kill-switch + launches keep paying that round trip each launch (199ms vs 103ms), which is the argument + for flipping the switch server-side only briefly. +- **Loss hurts config more in absolute terms** because it multiplies across ~100 requests + (and at 30% loss, failed blob downloads start shrinking the request/byte counts as fetches + give up), while legacy's single request degrades only its own tail. Per request the two + systems degrade the same way; the difference is request count, which again points at + prefetch scope and blob layout as the levers that matter. +- **Bytes double on cold config** (133KB vs 65KB) in this fixture because offerings still + carries full `paywall_components` while the config path also downloads workflow blobs. The + "stripped offerings" lever below is the measurement that matters for deciding whether to + remove paywall data from `/offerings`. + +## App-launch tier + +The CLI measures the manager-level fetch flows in isolation on macOS. The app-launch tier +(`Tests/TestingApps/SDKConfigBenchmarkApp/`) measures what the CLI cannot: a real iOS app +that calls `Purchases.configure` like a customer app does, on iOS hardware or simulator, +including process cold start, customer-info and offerings contention on the launch path, and +the time until a `PaywallView` actually appears on screen. + +- The app links the **stock** `RevenueCat`/`RevenueCatUI` products (no benchmark hooks, no + simulated transport), so it exercises exactly the binary that ships. It is live-only, + against the same pinned stress-test project. +- An XCUITest terminates and relaunches the app once per iteration, so every sample is a + true process cold start. `cold` wipes all SDK disk state and uses a fresh app user per + launch; `warm` primes once and relaunches with retained disk state and the same user. +- The legacy/config variant switch is the `ENABLE_REMOTE_CONFIG` compile condition, which + the SPM-built SDK reads from `Local.xcconfig` (`Package.swift` parses it). + `run-app-launch.sh` rewrites that file per variant, forces a manifest re-evaluation, and + verifies from the build log that the flag actually reached the SDK compile. +- Phases per launch: `configured` (configure returned), `customer_info` (first CustomerInfo + delivered), `offerings` (getOfferings completed), `paywall_appeared` (the PaywallView + wrapper mounted), and `paywall_impression` (the SDK tracked that paywall CONTENT actually + appeared: `paywall_impression` for classic paywalls, `workflows_step_started` for workflow + paywalls, observed via the SDK's internal events-listener SPI). **The row's percentile + statistics are over `offerings`: the measurement is `Purchases.configure` + `getOfferings`, + with or without workflows compiled in, matching the CLI tier's total.** The paywall marks + stay in the row as secondary phase means; a launch missing any phase still counts as an + error and fails the run. +- Config-path phases and blob registration, observed from the stock SDK's log stream (the + parser is pinned to `RemoteConfigStrings` by unit tests): `config_persisted_ms_mean`, + `last_blob_stored_ms_mean`, `blobs_inline_mean` vs `blobs_downloaded_mean`, `blob_bytes_mean`, + and the size extremes `max_inline_blob_bytes` / `min_downloaded_blob_bytes`. The extremes + bracket the backend's inline-size budget empirically (blobs under the budget, currently + ~3MB, should arrive inline; a small blob arriving via CDN is a regression signal). The CLI + tier registers the same fields via its blob store and transport events. +- Tests build and run under **Release** (the rows claim shipping-SDK numbers), and every + measured launch must prove at runtime which SDK variant is inside the binary: the config + path's refresh log fires on every config-variant launch, and the runner fails on any launch + whose `configPathActive` contradicts the row's label, cached builds included. Because live + runs use the project's Test Store key, which the SDK refuses (by crashing) in Release + builds, both variants also compile the SDK's designed opt-out for that guard, + `BYPASS_SIMULATED_STORE_RELEASE_CHECK`. +- Rows carry `mode` = `app-launch-legacy` / `app-launch-config` and `profile` = `simulator` / + `device`, so `compare.py` never mixes app-tier rows with CLI rows or simulator with device. + +The goal of this tier is to run the real app N times, via any automation that can drive +`xcodebuild test` (locally, CI, or tooling like the baguette CLI), and get the same kind of +comparable, gateable JSONL the CLI benchmark produces, measuring both systems end to end +exactly as a customer app experiences them. + +Run it with `bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl`. +Because there is no network control, treat app-tier numbers as end-to-end monitoring on a +real network, not as a controlled A/B; the CLI's simulated transport remains the tool for +isolating variables. + +First Release sample (simulator, live network against the stress project, 4 iterations, +1 warmup discarded; p50 over configure + getOfferings, phase columns are means in ms): + +| mode | scenario | p50 | offerings | config persisted | customer info | wrapper | impression | blobs | +|---|---|---|---|---|---|---|---|---| +| app-launch-legacy | cold | 1221 | 1298 | - | 839 | 1405 | 1971 | - | +| app-launch-config | cold | 1354 | 1551 | 977 | 1009 | 1590 | 2361 | 2 inline + 4 CDN, 161KB | +| app-launch-legacy | warm | 888 | 921 | - | 690 | 1104 | 1188 | - | +| app-launch-config | warm | 1094 | 1267 | - (204) | 1008 | 1316 | 2149 | 0 (already on disk) | + +Notable in this sample: configure + getOfferings runs ~11% over legacy on cold and ~23% on +warm with workflows compiled in; the blob registration keeps catching the same real signal: +`min_downloaded_blob_bytes` = **65** — a 65-byte blob CDN-fetched despite being far under +the inline budget (worth raising with the config team: either those blobs should be inlined, +or the inline policy is scoped more narrowly than "size under budget"). Iteration counts +this small are smoke-level; use `ITERATIONS=25` or more for numbers worth acting on. + +## Known limitations + +- **macOS CPU, not device CPU** for the CLI tier. Decode and CPU costs are measured on + desktop hardware. The comparison between modes is valid; absolute numbers on device will + differ. The app-launch tier above covers absolute, on-device launch timings. +- **Loss is approximated**, see the network model section above. +- **Fixture payloads are simpler than production.** Paywall component trees are minimal (one + stack + one text component per paywall). Byte counts scale with `--paywalls`/`--workflows`, + but production component trees are larger and heavier to decode. Grow the factory shapes + before making final decisions that hinge on decode cost. +- **No identity or purchase flows.** The benchmark measures the launch-path fetch and cache + system only. + +## Future levers to measure + +These slot in as new fixture/factory knobs rather than new machinery: + +- **Blob layout**: bundled vs topic-split vs per-paywall vs selected-paywalls blobs. +- **Prefetch scope**: which workflow blobs are marked `prefetch: true` (today the fixture marks + all of them, which is the worst case for `config` cold starts, since offerings delivery waits + on every prefetch-marked blob). +- **Stripped offerings**: offerings payloads without `paywall_components`, with paywalls served + from config blobs instead. +- **Parallelism**: `HTTPClient` pins `httpMaximumConnectionsPerHost = 1` and the blob fetcher + caps at 4 concurrent downloads; both are worth sweeping once the transport models connection + reuse. diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift b/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift new file mode 100644 index 0000000000..ea59f7ac5b --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Main/main.swift @@ -0,0 +1,3 @@ +import SDKConfigBenchmarkCore + +BenchmarkMain.run() diff --git a/Tests/Benchmarks/SDKConfigBenchmark/README.md b/Tests/Benchmarks/SDKConfigBenchmark/README.md new file mode 100644 index 0000000000..c7c6ce84d1 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/README.md @@ -0,0 +1,130 @@ +# SDKConfigBenchmark + +A macOS command-line benchmark that measures the SDK's **legacy offerings flow** against the +**remote config endpoint flow** (and its kill switch) through the real manager-level code paths. + +Two transports: + +- **`--transport simulated`** (default): in-process fixtures with seeded network profiles + (ideal/wifi/lte), packet-loss modeling, and a forced kill-switch 4xx. Deterministic and + CI-friendly. +- **`--transport live`**: real requests against the production backend, defaulting to the + prepared stress-test project + ([`5f07e7e3`](https://app.revenuecat.com/projects/5f07e7e3)). No keys live in source: the + matrix script resolves the key via mafdet; direct binary runs take `--api-key` or the + `SDK_CONFIG_BENCHMARK_API_KEY` environment variable, either of which also requires + `--project-id` so rows are labeled with the key's real project. Same recorded per-request metrics; real + CDN/TLS/latency behavior. Kill-switch, profiles, and loss are simulated-only (you cannot + force a 4xx or packet loss on production). + +See `CONFIG_ENDPOINT_BENCHMARKS.md` for methodology, scenario definitions, sample numbers, and +known limitations. + +## Setup + +```sh +tuist install +tuist generate SDKConfigBenchmark SDKConfigBenchmarkTests +``` + +## Run the matrix + +```sh +bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > results.jsonl # simulated +TRANSPORT=live bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > live.jsonl +``` + +Live runs default to the pinned stress-test project. To measure a different project, pass its +id; the key is resolved via the mafdet CLI (test-store app preferred) and every row is labeled +with `project_id`, so results from different projects can never be compared as equivalents: + +```sh +PROJECT_ID= TRANSPORT=live \ + bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > other-project.jsonl +``` + +(The target project's keyed app needs at least one package with a product attached, or the +offerings fetch fails with a configuration error.) + +Compare two runs (e.g. baseline branch vs candidate branch): + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py baseline.jsonl candidate.jsonl +``` + +Render one run as a table: + +```sh +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py results.jsonl +``` + +## Run a single configuration + +```sh +xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmark \ + -configuration Release -destination platform=macOS build + +# --transport: simulated | live +# --mode: legacy | config | config-killswitch (kill switch is simulated-only) +# --profile (ideal | wifi | lte) and --loss-percent are simulated-only +SDKConfigBenchmark \ + --transport simulated \ + --mode config \ + --scenario cold \ + --profile lte \ + --loss-percent 20 \ + --iterations 25 \ + --warmup-iterations 3 \ + --paywalls 50 \ + --workflows 100 \ + --seed 42 \ + --annotation sdk_commit=$(git rev-parse --short HEAD) +``` + +Output is one JSON object (a JSONL row) on stdout. + +Every run isolates the SDK's disk caches (ETags, offerings, remote config, blobs) under a +fresh temporary directory that is removed on exit, so runs never touch the real user Library +and concurrent runs cannot corrupt each other. + +## App-launch tier (simulator or device) + +The CLI above measures the manager-level flows in isolation. The app-host tier measures what +a customer actually experiences: a real iOS app (`SDKConfigBenchmarkApp`, linking the stock +`RevenueCat`/`RevenueCatUI` products) that runs `Purchases.configure` and reports the time to +configure, first customer info, offerings, and paywall appeared. An XCUITest relaunches the +app once per iteration, so every sample is a true process cold start; the runner script +builds the app twice, once per SDK variant, by rewriting `SWIFT_ACTIVE_COMPILATION_CONDITIONS` +in `Local.xcconfig` (restored on exit): + +```sh +bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl +python3 Tests/Benchmarks/SDKConfigBenchmark/compare.py app-launch.jsonl +``` + +Rows are `compare.py`-compatible, with `mode` = `app-launch-legacy` / `app-launch-config` and +`profile` = `simulator` / `device`. Knobs: `ITERATIONS`, `WARMUP`, `PROJECT_ID`, +`SDK_CONFIG_BENCHMARK_API_KEY` (skips mafdet; requires an explicit `PROJECT_ID`), and `DESTINATION` (pass +`DESTINATION="platform=iOS,id="` to measure a physical device's real radio). Live only: +there is no simulated transport, no kill-switch mode, and no loss model in this tier. + +Each launch also registers the config path when it runs: config persisted time, blob counts +split inline vs CDN-downloaded with byte totals, and size extremes that bracket the backend's +inline-size budget. The headline percentile is `Purchases.configure` + `getOfferings` +completed, with or without workflows compiled in (matching the CLI tier's total); paywall +render marks stay in the row as secondary phases. Tests run under Release and fail if a +launch's observed SDK variant contradicts the row's label. The intent is that any automation +able to drive `xcodebuild test` (CI, the baguette CLI, a cron box) can run the app N times +and collect the same gateable JSONL as the CLI matrix. + +## Unit tests + +```sh +xcodebuild -workspace RevenueCat-Tuist.xcworkspace -scheme SDKConfigBenchmarkTests \ + -destination platform=macOS test +``` + +The tests validate that fixtures decode into the real SDK response models, that the RC-Container +encoder round-trips through the SDK parser, that the transport is deterministic per seed, and +that each benchmark mode drives the expected SDK flow end to end (including warm 304/204 +revalidation and the kill-switch fallback). diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift new file mode 100644 index 0000000000..5213039a63 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkCommand.swift @@ -0,0 +1,255 @@ +import Foundation + +enum BenchmarkMode: String, CaseIterable { + + case legacy + case config + case configKillswitch = "config-killswitch" + + /// Whether a `RemoteConfigManager` stack is wired into the offerings flow. + var usesRemoteConfig: Bool { + switch self { + case .legacy: return false + case .config, .configKillswitch: return true + } + } + + /// Whether the fixture forces `/v1/config` to fail with a 4xx (the kill switch). + var forcesConfigFailure: Bool { + return self == .configKillswitch + } + + /// Whether warm iterations must revalidate config via manifest 204s. Kill-switch mode + /// pays a 4xx on every launch instead, by design. + var expectsWarmConfigRevalidation: Bool { + return self == .config + } + +} + +enum BenchmarkTransport: String, CaseIterable { + + /// In-process fixture transport: deterministic, seedable, supports profiles, loss, and the + /// forced kill-switch 4xx. + case simulated + + /// Real requests against the production backend, recorded for metrics. Pinned to the + /// prepared stress-test project (`BenchmarkProject`). + case live + +} + +/// The canonical RevenueCat project live runs measure against: +/// https://app.revenuecat.com/projects/5f07e7e3 ("Stress Test Config Endpoint"), prepared with +/// a large number of paywalls and workflows. No keys live in source: live runs read the key +/// from `--api-key`, the `SDK_CONFIG_BENCHMARK_API_KEY` environment variable, or (via +/// `run-matrix.sh`) mafdet resolution. Use the project's Test Store app key: its packages live +/// there; the App Store app has no products registered, which makes `OfferingsManager` fail +/// with a configuration error. +enum BenchmarkProject { + + static let projectID = "5f07e7e3" + static let dashboardURL = "https://app.revenuecat.com/projects/5f07e7e3" + static let apiKeyEnvironmentVariable = "SDK_CONFIG_BENCHMARK_API_KEY" + +} + +enum BenchmarkScenario: String, CaseIterable { + + /// Every iteration starts with empty disk caches and a fresh app user ID. + case cold + + /// Disk caches are primed once, then every iteration simulates an app relaunch: + /// fresh in-memory state, retained disk state, same app user ID. The fixture + /// serves 304 (offerings) and 204 (config) when the SDK proves it has current data. + case warm + +} + +struct BenchmarkCommand { + + static let defaultSimulatedAPIKey = "appl_benchmark" + + /// Keys `jsonlRow` writes itself; annotations must never overwrite them, or a row could + /// lie about what was measured (e.g. `--annotation mode=legacy` on a config run). + static let reservedRowKeys: Set = [ + "mode", "transport", "scenario", "profile", "loss_percent", "paywalls", "workflows", + "seed", "iterations", "warmup_discarded", "measured_iterations", "error_count", + "post_warmup_error_count", "mean_ms", "min_ms", "max_ms", "p50_ms", "p90_ms", "p95_ms", + "p99_ms", "request_count_mean", "bytes_received_mean", "failed_requests_total", + "fallback_host_requests_total", "offerings_ms_mean", "config_ms_mean", "blob_ms_mean", + "blobs_inline_mean", "blobs_downloaded_mean", "blob_bytes_mean", "blobs_by_topic", "max_inline_blob_bytes", + "min_downloaded_blob_bytes", "config_persisted_ms_mean", "last_blob_stored_ms_mean", + "launch_retries", + "first_error", "project_id" + ] + + var mode: BenchmarkMode = .legacy + var transport: BenchmarkTransport = .simulated + var scenario: BenchmarkScenario = .cold + var profileName: String = "ideal" + var lossPercent: Int = 0 + var iterations: Int = 25 + var warmupIterations: Int = 3 + var paywallCount: Int = 50 + var workflowCount: Int = 100 + var seed: UInt64 = 42 + var appUserID: String = "benchmark-user" + var apiKey: String = BenchmarkCommand.defaultSimulatedAPIKey + /// Which RevenueCat project a live run measures; labels the row so results from different + /// projects can never be compared as equivalents. Nil for simulated runs. + var projectID: String? + /// Extra key=value pairs echoed verbatim into the JSONL row (e.g. sdk_commit=abc123). + var annotations: [String: String] = [:] + + // swiftlint:disable:next cyclomatic_complexity function_body_length + static func parse( + _ args: [String], + environment: [String: String] = ProcessInfo.processInfo.environment + ) throws -> BenchmarkCommand { + var command = BenchmarkCommand() + var index = 0 + + func value(for flag: String) throws -> String { + index += 1 + guard index < args.count else { + throw BenchmarkError.invalidArgument("\(flag) requires a value") + } + return args[index] + } + + func intValue(for flag: String, in range: ClosedRange) throws -> Int { + let raw = try value(for: flag) + guard let parsed = Int(raw), range.contains(parsed) else { + throw BenchmarkError.invalidArgument("\(flag) must be an integer in \(range), got \(raw)") + } + return parsed + } + + while index < args.count { + let flag = args[index] + switch flag { + case "--mode": + let raw = try value(for: flag) + guard let mode = BenchmarkMode(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown mode \(raw)") + } + command.mode = mode + case "--transport": + let raw = try value(for: flag) + guard let transport = BenchmarkTransport(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown transport \(raw)") + } + command.transport = transport + case "--scenario": + let raw = try value(for: flag) + guard let scenario = BenchmarkScenario(rawValue: raw) else { + throw BenchmarkError.invalidArgument("unknown scenario \(raw)") + } + command.scenario = scenario + case "--profile": + command.profileName = try value(for: flag) + case "--loss-percent": + command.lossPercent = try intValue(for: flag, in: 0...100) + case "--iterations": + command.iterations = try intValue(for: flag, in: 1...100_000) + case "--warmup-iterations": + command.warmupIterations = try intValue(for: flag, in: 0...1_000) + case "--paywalls": + command.paywallCount = try intValue(for: flag, in: 1...10_000) + case "--workflows": + command.workflowCount = try intValue(for: flag, in: 0...10_000) + case "--seed": + let raw = try value(for: flag) + guard let seed = UInt64(raw) else { + throw BenchmarkError.invalidArgument("--seed must be an unsigned integer, got \(raw)") + } + command.seed = seed + case "--app-user-id": + command.appUserID = try value(for: flag) + case "--api-key": + command.apiKey = try value(for: flag) + case "--project-id": + command.projectID = try value(for: flag) + case "--annotation": + let raw = try value(for: flag) + let parts = raw.split(separator: "=", maxSplits: 1).map(String.init) + guard parts.count == 2, !parts[0].isEmpty else { + throw BenchmarkError.invalidArgument("--annotation expects key=value, got \(raw)") + } + guard !Self.reservedRowKeys.contains(parts[0]) else { + throw BenchmarkError.invalidArgument("--annotation key \(parts[0]) is a reserved row field") + } + command.annotations[parts[0]] = parts[1] + default: + throw BenchmarkError.invalidArgument("unknown flag \(flag)") + } + index += 1 + } + + guard command.warmupIterations < command.iterations else { + throw BenchmarkError.invalidArgument( + "--warmup-iterations (\(command.warmupIterations)) must be below --iterations (\(command.iterations))" + ) + } + + try command.validateAndDefaultTransport(environment: environment) + + return command + } + + /// Live runs hit the real backend: the knobs that shape the simulated network (and the + /// forced kill-switch 4xx) cannot apply there, and the API key defaults to the pinned + /// stress-test project. + private mutating func validateAndDefaultTransport(environment: [String: String]) throws { + guard self.transport == .live else { + guard self.projectID == nil else { + throw BenchmarkError.invalidArgument("--project-id only applies to --transport live") + } + return + } + + guard self.lossPercent == 0 else { + throw BenchmarkError.invalidArgument("--loss-percent requires --transport simulated") + } + guard self.profileName == "ideal" else { + throw BenchmarkError.invalidArgument( + "--profile requires --transport simulated; live runs use the real network" + ) + } + guard !self.mode.forcesConfigFailure else { + throw BenchmarkError.invalidArgument( + "config-killswitch cannot force a 4xx on the real backend; use --transport simulated" + ) + } + + if self.apiKey == Self.defaultSimulatedAPIKey { + // No keys live in source; resolve from the environment (run-matrix.sh populates + // it via mafdet). + guard let environmentKey = environment[BenchmarkProject.apiKeyEnvironmentVariable], + !environmentKey.isEmpty else { + throw BenchmarkError.invalidArgument( + "live runs need an API key: pass --api-key with --project-id, set " + + "\(BenchmarkProject.apiKeyEnvironmentVariable) with --project-id, or use " + + "run-matrix.sh (resolves it via mafdet)" + ) + } + self.apiKey = environmentKey + } + // ANY externally supplied key requires a project label: nothing ties a key to the + // pinned default project, and rows from different projects must never compare as + // equivalents. + guard self.projectID != nil else { + throw BenchmarkError.invalidArgument( + "a live API key (--api-key or \(BenchmarkProject.apiKeyEnvironmentVariable)) " + + "also requires --project-id, so rows are labeled with the key's real project" + ) + } + + // Fixture-size knobs don't shape live payloads (the pinned project's real content + // does), so rows carry 0 rather than a fixture size that was never measured. + self.paywallCount = 0 + self.workflowCount = 0 + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift new file mode 100644 index 0000000000..5fbf6b149c --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkError.swift @@ -0,0 +1,21 @@ +import Foundation + +enum BenchmarkError: Error, CustomStringConvertible { + + case invalidArgument(String) + case invalidFixture(String) + case timeout(String) + case backendFailure(String) + case scenarioViolation(String) + + var description: String { + switch self { + case let .invalidArgument(message): return "Invalid argument: \(message)" + case let .invalidFixture(message): return "Invalid fixture: \(message)" + case let .timeout(operation): return "Timed out waiting for \(operation)" + case let .backendFailure(message): return "Backend failure: \(message)" + case let .scenarioViolation(message): return "Scenario violation: \(message)" + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift new file mode 100644 index 0000000000..b583dd6d3e --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMain.swift @@ -0,0 +1,58 @@ +import Foundation + +// BenchmarkMain hosts only static members, but it cannot be the caseless enum that +// `convenience_type` wants because the repo bans new public enums outright. +// swiftlint:disable convenience_type + +/// The single public entry point for the `SDKConfigBenchmark` executable target. +/// Everything else in this module stays internal; the executable is one call into here. +public struct BenchmarkMain { + + /// Parses `CommandLine.arguments`, runs the requested benchmark, prints one JSONL row + /// to stdout, and terminates the process (0 on success, 1 on error). + /// + /// The benchmark itself runs on a background queue while the main thread sits in + /// `dispatchMain()`: `OfferingsManager` delivers its completion on the main queue, so + /// blocking the main thread on the run would deadlock every iteration. + public static func run() -> Never { + let command: BenchmarkCommand + do { + command = try BenchmarkCommand.parse(Array(CommandLine.arguments.dropFirst())) + } catch { + FileHandle.standardError.write(Data("\(error)\n".utf8)) + exit(1) + } + + // Every run gets its own disk-cache root: the SDK's container directories ignore + // $HOME, so this is the only way concurrent runs stay isolated (and the user's real + // Library stays clean). + let diskRoot = FileManager.default.temporaryDirectory + .appendingPathComponent("SDKConfigBenchmark-\(UUID().uuidString)", isDirectory: true) + DirectoryHelper.benchmarkBaseDirectoryOverride = diskRoot + + func finish(exitCode: Int32) -> Never { + try? FileManager.default.removeItem(at: diskRoot) + exit(exitCode) + } + + DispatchQueue.global(qos: .userInitiated).async { + do { + let result = try BenchmarkRunner(command: command).run() + print(result.jsonlRow) + if result.postWarmupErrorCount > 0 { + let message = "\(result.postWarmupErrorCount) post-warmup iteration(s) failed; " + + "timings are not valid comparison input\n" + FileHandle.standardError.write(Data(message.utf8)) + finish(exitCode: 2) + } + finish(exitCode: 0) + } catch { + FileHandle.standardError.write(Data("\(error)\n".utf8)) + finish(exitCode: 1) + } + } + + dispatchMain() + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift new file mode 100644 index 0000000000..74591f0325 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkMetrics.swift @@ -0,0 +1,257 @@ +import Foundation + +/// Blobs stored during one iteration, split by how they arrived: written from the config +/// container (inline) vs fetched from a blob source (downloaded). The size extremes bracket +/// the backend's inline-size budget empirically: the largest blob seen inline and the +/// smallest blob that needed a download. +struct BlobAccounting { + + /// Label for stored refs the persisted topic index does not reference. Should be empty + /// in practice (`retainOnly` prunes them); nonzero is itself a signal worth seeing. + static let unreferencedTopic = "unreferenced" + + let inlineCount: Int + let downloadedCount: Int + let totalBytes: Int + let maxInlineBytes: Int? + let minDownloadedBytes: Int? + /// Newly stored blobs grouped by the topic that references them (e.g. "workflows", + /// "ui_config"), so rows say WHAT was fetched, not just how much. + let countsByTopic: [String: Int] + let bytesByTopic: [String: Int] + + static let empty = BlobAccounting(newRefSizes: [:], downloadedRefs: []) + + /// Attributes newly stored refs: a new ref that has a successful blob request this + /// iteration was downloaded; any other new ref can only have come from the container. + /// `topicByRef` (from the persisted topic index) labels each ref with its topic. + init( + newRefSizes: [String: Int], + downloadedRefs: Set, + topicByRef: [String: String] = [:] + ) { + var inlineCount = 0 + var downloadedCount = 0 + var totalBytes = 0 + var maxInlineBytes: Int? + var minDownloadedBytes: Int? + var countsByTopic: [String: Int] = [:] + var bytesByTopic: [String: Int] = [:] + + for (ref, byteCount) in newRefSizes { + totalBytes += byteCount + if downloadedRefs.contains(ref) { + downloadedCount += 1 + minDownloadedBytes = min(minDownloadedBytes ?? .max, byteCount) + } else { + inlineCount += 1 + maxInlineBytes = max(maxInlineBytes ?? .min, byteCount) + } + let topic = topicByRef[ref] ?? Self.unreferencedTopic + countsByTopic[topic, default: 0] += 1 + bytesByTopic[topic, default: 0] += byteCount + } + + self.inlineCount = inlineCount + self.downloadedCount = downloadedCount + self.totalBytes = totalBytes + self.maxInlineBytes = maxInlineBytes + self.minDownloadedBytes = minDownloadedBytes + self.countsByTopic = countsByTopic + self.bytesByTopic = bytesByTopic + } + +} + +/// One measured simulated launch, with phases attributed from the transport event log. +struct IterationMeasurement { + + /// Start of the iteration to delivery of the offerings completion. + let totalMs: Double + /// Span of the offerings request(s), nil if none were made. + let offeringsMs: Double? + /// Span of the `/v1/config` request(s), nil for legacy mode. + let configMs: Double? + /// First blob request start to last blob request end, nil if no blobs were fetched. + let blobMs: Double? + let requestCount: Int + let bytesReceived: Int + let failedRequestCount: Int + let fallbackHostRequestCount: Int + /// Per-kind statuses and blob traffic, for strict warm-scenario verification. + let offeringsStatusCodes: [Int] + let configStatusCodes: [Int] + let blobRequestCount: Int + /// Blobs stored this iteration, attributed inline vs downloaded via the blob store. + let blobs: BlobAccounting + + init(totalMs: Double, events: [TransportEvent], blobAccounting: BlobAccounting = .empty) { + self.totalMs = totalMs + let byKind = Dictionary(grouping: events, by: \.kind) + let offerings = byKind[.offerings] ?? [] + let config = byKind[.config] ?? [] + let blobs = byKind[.blob] ?? [] + self.offeringsMs = Self.span(of: offerings) + self.configMs = Self.span(of: config) + self.blobMs = Self.span(of: blobs) + self.requestCount = events.count + self.bytesReceived = events.reduce(0) { $0 + $1.bytesReceived } + self.failedRequestCount = events.filter(\.failed).count + self.fallbackHostRequestCount = events.filter(\.isFallbackHostRequest).count + self.offeringsStatusCodes = offerings.map(\.statusCode) + self.configStatusCodes = config.map(\.statusCode) + self.blobRequestCount = blobs.count + self.blobs = blobAccounting + } + + private static func span(of events: [TransportEvent]) -> Double? { + guard let first = events.map(\.startedAt.uptimeNanoseconds).min(), + let last = events.map(\.endedAt.uptimeNanoseconds).max() else { + return nil + } + return Double(last - first) / 1_000_000 + } + +} + +/// Aggregates measured iterations into one JSONL row. Iterations with index below +/// `warmupIterations` are excluded from the statistics (by index, so a failed warmup iteration +/// never shifts a slow measured one into the discarded window), and errors are reported both in +/// total and for the post-warmup window: a run whose candidate "wins" by erroring out of its +/// slowest iterations must be visibly invalid, not quietly faster. +struct BenchmarkMetrics { + + private var measurementsByIteration: [Int: IterationMeasurement] = [:] + private var errorsByIteration: [Int: String] = [:] + + mutating func record(_ measurement: IterationMeasurement, iteration: Int) { + self.measurementsByIteration[iteration] = measurement + } + + mutating func record(error: Error, iteration: Int) { + self.errorsByIteration[iteration] = "\(error)" + } + + /// Errors in the measured (post-warmup) window. Nonzero means the row's timing statistics + /// are not valid comparison input, and the process must exit nonzero so automation notices. + func postWarmupErrorCount(warmupIterations: Int) -> Int { + return self.errorsByIteration.filter { $0.key >= warmupIterations }.count + } + + func jsonlRow(for command: BenchmarkCommand) -> String { + let measured = self.measurementsByIteration + .filter { $0.key >= command.warmupIterations } + .sorted { $0.key < $1.key } + .map(\.value) + let totals = measured.map(\.totalMs).sorted() + + var row: [String: Any] = [ + "mode": command.mode.rawValue, + "transport": command.transport.rawValue, + "scenario": command.scenario.rawValue, + "profile": command.profileName, + "loss_percent": command.lossPercent, + "paywalls": command.paywallCount, + "workflows": command.workflowCount, + "seed": command.seed, + "iterations": command.iterations, + "warmup_discarded": command.warmupIterations, + "measured_iterations": measured.count, + "error_count": self.errorsByIteration.count, + "post_warmup_error_count": self.postWarmupErrorCount(warmupIterations: command.warmupIterations) + ] + + if !totals.isEmpty { + row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) + row["min_ms"] = Self.rounded(totals[0]) + row["max_ms"] = Self.rounded(totals[totals.count - 1]) + for percentile in [50, 90, 95, 99] { + row["p\(percentile)_ms"] = Self.rounded(Self.percentile(percentile, of: totals)) + } + } + + row += Self.aggregates(of: measured) + + if let projectID = command.projectID { + row["project_id"] = projectID + } + + if let firstError = self.errorsByIteration.min(by: { $0.key < $1.key }) { + row["first_error"] = "iteration \(firstError.key): \(firstError.value)" + } + + for (key, value) in command.annotations { + row[key] = value + } + + do { + let data = try JSONSerialization.data(withJSONObject: row, options: [.sortedKeys]) + return String(bytes: data, encoding: .utf8) ?? "{}" + } catch { + preconditionFailure("Could not serialize benchmark row: \(error)") + } + } + + private static func aggregates(of measured: [IterationMeasurement]) -> [String: Any] { + guard !measured.isEmpty else { return [:] } + + let count = Double(measured.count) + var aggregates: [String: Any] = [ + "request_count_mean": Self.rounded(Double(measured.reduce(0) { $0 + $1.requestCount }) / count), + "bytes_received_mean": Self.rounded(Double(measured.reduce(0) { $0 + $1.bytesReceived }) / count), + "failed_requests_total": measured.reduce(0) { $0 + $1.failedRequestCount }, + "fallback_host_requests_total": measured.reduce(0) { $0 + $1.fallbackHostRequestCount } + ] + + for (key, values) in [ + ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), + ("config_ms_mean", measured.compactMap(\.configMs)), + ("blob_ms_mean", measured.compactMap(\.blobMs)), + ("blobs_inline_mean", measured.map { Double($0.blobs.inlineCount) }), + ("blobs_downloaded_mean", measured.map { Double($0.blobs.downloadedCount) }), + ("blob_bytes_mean", measured.map { Double($0.blobs.totalBytes) }) + ] where !values.isEmpty { + aggregates[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) + } + + // Size extremes across the run bracket the backend's inline-size budget. + if let maxInline = measured.compactMap(\.blobs.maxInlineBytes).max() { + aggregates["max_inline_blob_bytes"] = maxInline + } + if let minDownloaded = measured.compactMap(\.blobs.minDownloadedBytes).min() { + aggregates["min_downloaded_blob_bytes"] = minDownloaded + } + + // Per-topic attribution: what was fetched, not just how much. Mean per iteration. + var topicCounts: [String: Int] = [:] + var topicBytes: [String: Int] = [:] + for iteration in measured { + topicCounts.merge(iteration.blobs.countsByTopic, uniquingKeysWith: +) + topicBytes.merge(iteration.blobs.bytesByTopic, uniquingKeysWith: +) + } + if !topicCounts.isEmpty { + var byTopic: [String: [String: Double]] = [:] + for topic in topicCounts.keys { + byTopic[topic] = [ + "count_mean": Self.rounded(Double(topicCounts[topic] ?? 0) / count), + "bytes_mean": Self.rounded(Double(topicBytes[topic] ?? 0) / count) + ] + } + aggregates["blobs_by_topic"] = byTopic + } + + return aggregates + } + + /// Nearest-rank percentile over an already-sorted array. + static func percentile(_ percentile: Int, of sorted: [Double]) -> Double { + precondition(!sorted.isEmpty) + let rank = Int((Double(percentile) / 100 * Double(sorted.count)).rounded(.up)) + return sorted[max(0, min(sorted.count - 1, rank - 1))] + } + + private static func rounded(_ value: Double) -> Double { + return (value * 1_000).rounded() / 1_000 + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift new file mode 100644 index 0000000000..350296397e --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkPayloadFactory.swift @@ -0,0 +1,268 @@ +import Foundation + +/// Generates the wire payloads the fixture server responds with: the legacy offerings JSON, +/// the `/v1/config/{domain}` RC-Container, and the content-addressed blobs it references. +/// All payloads are deterministic for a given (paywallCount, workflowCount), so byte counts +/// are comparable across runs and branches. +struct BenchmarkPayloadFactory { + + static let blobURLFormat = "https://cdn.revenuecat.local/blobs/{blob_ref}" + + let configManifest = "benchmark-manifest-v1" + + let offeringsData: Data + let configContainerData: Data + /// The refs the config marks for prefetch; a warm relaunch must report exactly these back + /// as its locally-cached blobs. + let workflowPrefetchRefs: Set + + private let blobsByRef: [String: Data] + + init(paywallCount: Int, workflowCount: Int) { + let builder = PayloadBuilder(paywallCount: paywallCount, workflowCount: workflowCount) + self.offeringsData = builder.offeringsData() + + var blobs: [String: Data] = [:] + var workflowRefs: [String: String] = [:] + for index in 0.. Data? { + return self.blobsByRef[ref] + } + + var allBlobRefs: [String] { + return Array(self.blobsByRef.keys) + } + +} + +/// Raw JSON dictionary builders, kept separate so `BenchmarkPayloadFactory.init` reads as the +/// wiring of refs into the config payload. +private struct PayloadBuilder { + + let paywallCount: Int + let workflowCount: Int + + static func productIdentifier(index: Int) -> String { + return "com.revenuecat.benchmark.product_\(index)" + } + + func offeringsData() -> Data { + return self.data([ + "current_offering_id": "offering_0", + "offerings": (0.. Data { + return self.data([ + "id": "workflow_\(index)", + "display_name": "Benchmark Workflow \(index)", + "initial_step_id": "step-1", + "single_step_fallback_id": "step-1", + "steps": [ + "step-1": [ + "id": "step-1", + "type": "paywall", + "screen_id": "screen-1" + ] + ], + "screens": [:], + "metadata": [ + "benchmark_index": index, + "benchmark_payload": String(repeating: "w", count: 256) + ] + ]) + } + + func uiConfigAppBlobData() -> Data { + return self.data([ + "colors": [:], + "fonts": [:] + ]) + } + + func uiConfigLocalizationsBlobData() -> Data { + return self.data([ + "en_US": [ + "benchmark.title": "Benchmark title", + "benchmark.subtitle": "Benchmark subtitle" + ] + ]) + } + + func configJSONData( + manifest: String, + workflowRefsById: [String: String], + uiConfigAppRef: String, + uiConfigLocalizationsRef: String + ) -> Data { + var workflowsTopic: [String: Any] = [:] + for (workflowId, ref) in workflowRefsById { + let index = Int(workflowId.split(separator: "_").last.map(String.init) ?? "0") ?? 0 + workflowsTopic[workflowId] = [ + "offering_identifier": "offering_\(self.paywallCount == 0 ? 0 : index % self.paywallCount)", + "blob_ref": ref, + "prefetch": true + ] + } + + // Only workflow blobs are prefetched: those are what offerings delivery awaits + // (`awaitTopicAndPrefetchBlobsReady(.workflows)`). Prefetching ui_config blobs too + // would leave downloads running past the measured completion, corrupting the + // per-iteration request/byte accounting; they stay fetchable on demand via blob_ref. + // Sorted so the payload bytes are stable across processes (JSON arrays keep their order). + let prefetchBlobs = workflowRefsById.values.sorted() + + return self.data([ + "domain": "app", + "manifest": manifest, + "active_topics": ["sources", "workflows", "ui_config"], + "prefetch_blobs": prefetchBlobs, + "topics": [ + "sources": [ + "api": [ + "sources": [ + ["url": "https://api.revenuecat.com/", "priority": 0, "weight": 100] + ] + ], + "blob": [ + "sources": [ + ["url_format": BenchmarkPayloadFactory.blobURLFormat, "priority": 0, "weight": 100] + ] + ] + ], + "workflows": workflowsTopic, + "ui_config": [ + "app": ["blob_ref": uiConfigAppRef], + "localizations": ["blob_ref": uiConfigLocalizationsRef] + ] + ] + ]) + } + +} + +private extension PayloadBuilder { + + func offering(index: Int) -> [String: Any] { + let offeringIdentifier = "offering_\(index)" + return [ + "identifier": offeringIdentifier, + "description": "Benchmark offering \(index)", + "packages": [ + [ + "identifier": "$rc_monthly", + "platform_product_identifier": Self.productIdentifier(index: index) + ] + ], + "metadata": [ + "benchmark_index": index, + "benchmark_payload": String(repeating: "m", count: 64) + ], + "paywall_components": self.paywallComponents(id: "paywall_\(index)", offeringIdentifier: offeringIdentifier) + ] + } + + func paywallComponents(id: String, offeringIdentifier: String) -> [String: Any] { + return [ + "id": id, + "default_locale": "en_US", + "revision": 1, + "template_name": "benchmark_components", + "asset_base_url": "https://assets.revenuecat.com", + "components_localizations": [ + "en_US": [ + "title": "Benchmark title for \(offeringIdentifier)", + "cta": "Continue" + ] + ], + "components_config": [ + "base": [ + "stack": self.paywallStack(), + "background": [ + "type": "color", + "value": [ + "light": ["type": "hex", "value": "#FFFFFF"] + ] + ] + ] + ] + ] + } + + func paywallStack() -> [String: Any] { + return [ + "type": "stack", + "components": [self.paywallTitleComponent()], + "dimension": [ + "type": "vertical", + "alignment": "center", + "distribution": "center" + ], + "size": [ + "width": ["type": "fill"], + "height": ["type": "fill"] + ], + "padding": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "margin": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "spacing": 16 + ] + } + + func paywallTitleComponent() -> [String: Any] { + return [ + "type": "text", + "text_lid": "title", + "font_weight": "bold", + "font_size": 24, + "color": [ + "light": ["type": "hex", "value": "#111111"] + ], + "size": [ + "width": ["type": "fill"], + "height": ["type": "fit"] + ], + "padding": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "margin": ["top": 0, "bottom": 0, "leading": 0, "trailing": 0], + "horizontal_alignment": "center" + ] + } + + func data(_ object: Any) -> Data { + do { + return try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + } catch { + preconditionFailure("Invalid benchmark JSON fixture: \(error)") + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift new file mode 100644 index 0000000000..519898a4a7 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkProductsManager.swift @@ -0,0 +1,41 @@ +import Foundation + +/// StoreKit-free `ProductsManagerType` so the benchmark measures the SDK's fetch and decode +/// paths, not App Store product lookups. Every requested identifier resolves immediately to a +/// synthesized monthly subscription, the same way UI preview mode fabricates products. +final class BenchmarkProductsManager: ProductsManagerType { + + let requestTimeout: TimeInterval = 30 + + func products( + withIdentifiers identifiers: Set, + completion: @escaping ProductsManagerType.Completion + ) { + let products = identifiers.map { identifier in + TestStoreProduct( + localizedTitle: "Benchmark Monthly", + price: 9.99, + localizedPriceString: "$9.99", + productIdentifier: identifier, + productType: .autoRenewableSubscription, + localizedDescription: "Benchmark subscription", + subscriptionGroupIdentifier: "benchmark.group", + subscriptionPeriod: SubscriptionPeriod(value: 1, unit: .month) + ).toStoreProduct() + } + completion(.success(Set(products))) + } + + @available(iOS 15.0, tvOS 15.0, watchOS 8.0, macOS 12.0, *) + func sk2Products( + withIdentifiers identifiers: Set, + completion: @escaping ProductsManagerType.SK2Completion + ) { + completion(.success([])) + } + + func cache(_ product: StoreProductType) {} + + func clearCache() {} + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift new file mode 100644 index 0000000000..3301a78a70 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkRunner.swift @@ -0,0 +1,353 @@ +import Foundation + +struct BenchmarkRunResult { + + let jsonlRow: String + /// Nonzero means the row is visible but its timings are not valid comparison input. + let postWarmupErrorCount: Int + +} + +/// Drives one benchmark configuration: N simulated app launches through the real SDK stack, +/// against the simulated transport, producing a single JSONL row. +/// +/// An iteration is one launch: build a fresh stack (fresh in-memory state, like a new process), +/// kick the remote config refresh when wired, fetch offerings, and stop the clock when the +/// offerings completion fires. `cold` wipes disk state before every iteration and uses a +/// per-iteration app user ID; `warm` primes disk once, then relaunches against retained disk +/// state, which must revalidate via 304 (offerings) and 204 (config) or the run fails loudly +/// rather than silently measuring cold behavior. +final class BenchmarkRunner { + + private let command: BenchmarkCommand + + init(command: BenchmarkCommand) { + self.command = command + } + + func run() throws -> BenchmarkRunResult { + switch self.command.transport { + case .simulated: + guard let profile = NetworkProfile.named(self.command.profileName) else { + throw BenchmarkError.invalidArgument("unknown profile \(self.command.profileName)") + } + let factory = BenchmarkPayloadFactory( + paywallCount: self.command.paywallCount, + workflowCount: self.command.workflowCount + ) + let server = FixtureServer( + factory: factory, + killSwitchConfig: self.command.mode.forcesConfigFailure + ) + SimulatedTransportURLProtocol.install( + server: server, + profile: profile, + loss: LossModel(lossPercent: self.command.lossPercent), + seed: self.command.seed + ) + case .live: + // Requests hit the real backend (the pinned stress-test project) through a + // recording passthrough, so live rows carry the same per-request metrics. + SimulatedTransportURLProtocol.installPassthrough() + } + defer { SimulatedTransportURLProtocol.uninstall() } + + var metrics = BenchmarkMetrics() + + if self.command.scenario == .warm { + try self.primeDiskState() + } + + for iteration in 0..= self.command.warmupIterations { + try self.validateScenario(measurement, iteration: iteration) + } + metrics.record(measurement, iteration: iteration) + } catch { + if case BenchmarkError.scenarioViolation = error { throw error } + metrics.record(error: error, iteration: iteration) + } + } + + return BenchmarkRunResult( + jsonlRow: metrics.jsonlRow(for: self.command), + postWarmupErrorCount: metrics.postWarmupErrorCount(warmupIterations: self.command.warmupIterations) + ) + } + +} + +private extension BenchmarkRunner { + + /// Live runs get a per-run nonce so reruns (even parallel or same-second ones) never share + /// server-side per-user state; simulated runs stay fully deterministic. + private static let liveRunNonce = String(UUID().uuidString.prefix(8)) + + var baseAppUserID: String { + switch self.command.transport { + case .simulated: + return self.command.appUserID + case .live: + return "\(self.command.appUserID)-\(Self.liveRunNonce)" + } + } + + func appUserID(forIteration iteration: Int) -> String { + switch self.command.scenario { + case .cold: + return "\(self.baseAppUserID)-\(iteration)" + case .warm: + return self.baseAppUserID + } + } + + /// Uncounted launch that fills the disk caches the warm iterations relaunch against. + func primeDiskState() throws { + SimulatedTransportURLProtocol.beginIteration(-1) + let stack = BenchmarkSDKStack( + mode: self.command.mode, + apiKey: self.command.apiKey, + appUserID: self.baseAppUserID + ) + stack.clearAllDiskState() + _ = try self.launch(stack, appUserID: self.baseAppUserID) + SimulatedTransportURLProtocol.waitUntilIdle() + _ = SimulatedTransportURLProtocol.drainEvents() + } + + func runIteration(_ iteration: Int) throws -> IterationMeasurement { + let appUserID = self.appUserID(forIteration: iteration) + let stack = BenchmarkSDKStack( + mode: self.command.mode, + apiKey: self.command.apiKey, + appUserID: appUserID + ) + if self.command.scenario == .cold { + stack.clearAllDiskState() + } + _ = SimulatedTransportURLProtocol.drainEvents() + let blobRefsBeforeLaunch = stack.blobStore?.cachedRefs() ?? [] + + let totalMs: Double + do { + totalMs = try self.launch(stack, appUserID: appUserID) + } catch { + // Quiet the failed stack so its in-flight work stops churning; any straggler + // events it still produces carry this iteration's stamp and are filtered out of + // later measurements below. + stack.remoteConfigManager?.close() + throw error + } + + // The clock stopped at offerings delivery, but trailing requests from this launch + // (e.g. the warm config 204 that finishes after offerings, matching production order) + // still belong to this iteration's accounting: collect until the transport is idle + // AND the config request (when wired) has landed, then keep only this iteration's + // events so stragglers from earlier launches can't inflate the row. + let events = self.collectEvents(forIteration: iteration) + return IterationMeasurement( + totalMs: totalMs, + events: events, + blobAccounting: Self.blobAccounting( + blobStore: stack.blobStore, + refsBeforeLaunch: blobRefsBeforeLaunch, + events: events, + topics: stack.remoteConfigDiskCache?.read()?.topics + ) + ) + } + + /// Drains transport events until quiescence, additionally waiting (bounded) for this + /// iteration's config request in config modes: transport idleness alone can race the gap + /// between kicking the refresh and its request reaching the transport. + func collectEvents(forIteration iteration: Int, timeoutMs: Int = 15_000) -> [TransportEvent] { + var events: [TransportEvent] = [] + let deadline = DispatchTime.now() + .milliseconds(timeoutMs) + + while true { + SimulatedTransportURLProtocol.waitUntilIdle() + events += SimulatedTransportURLProtocol.drainEvents() + + let configLanded = events.contains { $0.iteration == iteration && $0.kind == .config } + if !self.command.mode.usesRemoteConfig || configLanded || DispatchTime.now() >= deadline { + break + } + usleep(5_000) + } + + return events.filter { $0.iteration == iteration } + } + + /// One simulated launch; returns start-to-offerings-delivered wall time in milliseconds. + /// Runs off the main thread; the offerings completion is delivered on the main queue, which + /// `BenchmarkMain` keeps pumping via `dispatchMain()`. + /// + /// This calls the exact pair `Purchases.updateAllCaches` calls, in the same order: + /// `updateOfferingsCache` (which enqueues the offerings operation synchronously, before + /// this method moves on) and then `refreshRemoteConfig`. Both land on the same serial + /// backend queue and single-connection host pool, so enqueue order shapes the measured + /// serialization; using `offerings(appUserID:)` instead would hop through the main actor + /// first and let the config request race ahead. + func launch(_ stack: BenchmarkSDKStack, appUserID: String) throws -> Double { + let start = DispatchTime.now() + + let semaphore = DispatchSemaphore(value: 0) + let failure = Atomic(nil) + stack.offeringsManager.updateOfferingsCache(appUserID: appUserID, isAppBackgrounded: false) { result in + if case let .failure(error) = result { + failure.value = error + } + semaphore.signal() + } + stack.refreshRemoteConfigIfWired() + + guard semaphore.wait(timeout: .now() + 120) == .success else { + throw BenchmarkError.timeout("offerings fetch") + } + if let error = failure.value { + throw BenchmarkError.backendFailure("offerings fetch failed: \(error)") + } + + let end = DispatchTime.now() + return Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000 + } + + /// Every measured iteration must prove it exercised the path its row claims: warm runs + /// must hit the revalidation paths, and cold config runs must complete a successful + /// config request (a failed refresh silently delivers offerings via the legacy fallback, + /// which would let a "config" row measure the wrong system). Skipped under packet loss, + /// where transient failures are the thing being measured. + func validateScenario(_ measurement: IterationMeasurement, iteration: Int) throws { + guard self.command.lossPercent == 0 else { return } + switch self.command.scenario { + case .warm: + try Self.validateWarmMeasurement(measurement, mode: self.command.mode, iteration: iteration) + case .cold: + try Self.validateColdMeasurement(measurement, mode: self.command.mode, iteration: iteration) + } + } + +} + +extension BenchmarkRunner { + + /// Attributes this iteration's newly stored blobs (all disk reads happen after the timed + /// window): a new ref with a successful blob request was downloaded; any other new ref can + /// only have arrived inline in the config container. `topics` (the persisted topic index) + /// labels each ref with the topic that references it. + static func blobAccounting( + blobStore: RemoteConfigBlobStoreType?, + refsBeforeLaunch: Set, + events: [TransportEvent], + topics: RemoteConfiguration.Topics? = nil + ) -> BlobAccounting { + guard let blobStore else { return .empty } + + let newRefs = blobStore.cachedRefs().subtracting(refsBeforeLaunch) + guard !newRefs.isEmpty else { return .empty } + + let downloadedRefs = Set( + events + .filter { $0.kind == .blob && !$0.failed } + .map { ($0.path as NSString).lastPathComponent } + ) + let newRefSizes = newRefs.reduce(into: [String: Int]()) { sizes, ref in + sizes[ref] = blobStore.read(ref: ref)?.count ?? 0 + } + return BlobAccounting( + newRefSizes: newRefSizes, + downloadedRefs: downloadedRefs, + topicByRef: Self.topicByRef(from: topics) + ) + } + + /// Inverts the persisted topic index into ref → topic name. A ref referenced by several + /// topics keeps the alphabetically first, deterministically. + static func topicByRef(from topics: RemoteConfiguration.Topics?) -> [String: String] { + guard let topics else { return [:] } + + var topicByRef: [String: String] = [:] + for (topicName, items) in topics.entries { + for item in items.values { + guard let ref = item.blobRef else { continue } + if let existing = topicByRef[ref], existing <= topicName { continue } + topicByRef[ref] = topicName + } + } + return topicByRef + } + + /// Cold config iterations must have actually completed the config exchange their mode + /// claims: a success for plain config mode, the disabling 4xx for the kill switch. + static func validateColdMeasurement( + _ measurement: IterationMeasurement, + mode: BenchmarkMode, + iteration: Int + ) throws { + guard mode.usesRemoteConfig else { return } + + if mode.forcesConfigFailure { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ (400...499).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "cold kill-switch iteration \(iteration) did not hit the config 4xx " + + "(statuses: \(measurement.configStatusCodes))" + ) + } + return + } + + guard measurement.configStatusCodes.contains(where: { (200...299).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "cold config iteration \(iteration) never completed a successful config request " + + "(statuses: \(measurement.configStatusCodes)); the row would measure legacy fallback" + ) + } + } + + static func validateWarmMeasurement( + _ measurement: IterationMeasurement, + mode: BenchmarkMode, + iteration: Int + ) throws { + guard !measurement.offeringsStatusCodes.isEmpty, + measurement.offeringsStatusCodes.allSatisfy({ $0 == 304 }) else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) did not revalidate offerings via 304 " + + "(statuses: \(measurement.offeringsStatusCodes))" + ) + } + + // Plain config mode must revalidate via manifest 204s; kill-switch mode must pay the + // disabling 4xx on every launch (each iteration is a fresh manager, so the switch + // trips again). Anything else means the mode isn't measuring what it claims. + if mode.expectsWarmConfigRevalidation { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ $0 == 204 }) else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) did not revalidate config via manifest 204 " + + "(statuses: \(measurement.configStatusCodes))" + ) + } + } + if mode.forcesConfigFailure { + guard !measurement.configStatusCodes.isEmpty, + measurement.configStatusCodes.allSatisfy({ (400...499).contains($0) }) else { + throw BenchmarkError.scenarioViolation( + "warm kill-switch iteration \(iteration) did not hit the config 4xx " + + "(statuses: \(measurement.configStatusCodes))" + ) + } + } + + guard measurement.blobRequestCount == 0 else { + throw BenchmarkError.scenarioViolation( + "warm iteration \(iteration) re-downloaded \(measurement.blobRequestCount) blob(s)" + ) + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift new file mode 100644 index 0000000000..94f9ecf397 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/BenchmarkSDKStack.swift @@ -0,0 +1,155 @@ +import Foundation + +/// Builds the same manager-level object graph `Purchases` wires at configure time, minus +/// StoreKit and identity, so one simulated app launch is: construct a stack, kick the remote +/// config refresh (config modes), and fetch offerings through `OfferingsManager`. +/// +/// Mode selection mirrors production's `SystemInfo.remoteConfigEnabled` gate: `legacy` passes +/// `remoteConfigManager: nil` to `OfferingsManager` (gate off), the config modes wire a real +/// `RemoteConfigManager` stack (gate on). The kill-switch mode uses identical wiring; the +/// fixture server's 4xx is what trips the manager's session kill switch. +final class BenchmarkSDKStack { + + let offeringsManager: OfferingsManager + let remoteConfigManager: RemoteConfigManagerType? + /// The manager's content-addressed blob store, exposed so the runner can attribute + /// per-iteration blob storage (inline vs downloaded) and sizes. Nil in legacy mode. + let blobStore: RemoteConfigBlobStoreType? + /// The manager's disk cache, exposed so the runner can join stored blob refs against + /// the persisted topic index (which topic each blob belongs to). Nil in legacy mode. + let remoteConfigDiskCache: RemoteConfigDiskCacheType? + let httpClient: HTTPClient + let deviceCache: DeviceCache + + private let appUserID: String + + init(mode: BenchmarkMode, apiKey: String, appUserID: String) { + self.appUserID = appUserID + + let operationDispatcher = OperationDispatcher.default + let systemInfo = Self.makeSystemInfo(apiKey: apiKey, operationDispatcher: operationDispatcher) + + let httpClient = HTTPClient( + systemInfo: systemInfo, + eTagManager: ETagManager(), + signing: Signing(apiKey: apiKey), + diagnosticsTracker: nil, + requestTimeout: 15, + operationDispatcher: operationDispatcher + ) + self.httpClient = httpClient + + let backendConfiguration = BackendConfiguration( + httpClient: httpClient, + operationDispatcher: operationDispatcher, + operationQueue: Backend.QueueProvider.createBackendQueue(), + diagnosticsQueue: Backend.QueueProvider.createDiagnosticsQueue(), + systemInfo: systemInfo, + offlineCustomerInfoCreator: nil + ) + let backend = Backend( + backendConfig: backendConfiguration, + attributionFetcher: AttributionFetcher( + attributionFactory: AttributionTypeFactory(), + systemInfo: systemInfo + ) + ) + + guard let userDefaults = UserDefaults(suiteName: "com.revenuecat.SDKConfigBenchmark.scratch") else { + preconditionFailure("Could not create benchmark UserDefaults suite") + } + let deviceCache = DeviceCache(systemInfo: systemInfo, userDefaults: userDefaults) + self.deviceCache = deviceCache + + let remoteConfigStack = mode.usesRemoteConfig + ? Self.makeRemoteConfigManager(backendConfiguration: backendConfiguration, appUserID: appUserID) + : nil + let remoteConfigManager = remoteConfigStack?.manager + self.remoteConfigManager = remoteConfigManager + self.blobStore = remoteConfigStack?.blobStore + self.remoteConfigDiskCache = remoteConfigStack?.diskCache + + self.offeringsManager = OfferingsManager( + deviceCache: deviceCache, + operationDispatcher: operationDispatcher, + systemInfo: systemInfo, + backend: backend, + offeringsFactory: OfferingsFactory(systemInfo: systemInfo), + productsManager: BenchmarkProductsManager(), + diagnosticsTracker: nil, + remoteConfigManager: remoteConfigManager + ) + } + + /// Kicks the remote config sync the way `Purchases.updateAllCaches` does on launch. + func refreshRemoteConfigIfWired() { + self.remoteConfigManager?.refreshRemoteConfig(isAppBackgrounded: false) + } + + /// Erases every piece of disk state a previous launch could have left behind, so a `cold` + /// iteration starts from nothing: ETags, the offerings disk cache, and the persisted remote + /// configuration with its blob store. + func clearAllDiskState() { + self.httpClient.clearCaches() + self.deviceCache.clearOfferingsCache(appUserID: self.appUserID) + self.remoteConfigManager?.clearCache() + } + +} + +private extension BenchmarkSDKStack { + + static func makeSystemInfo(apiKey: String, operationDispatcher: OperationDispatcher) -> SystemInfo { + return SystemInfo( + platformInfo: nil, + finishTransactions: true, + operationDispatcher: operationDispatcher, + storeKitVersion: .storeKit1, + apiKey: apiKey, + responseVerificationMode: .disabled, + isAppBackgrounded: false, + preferredLocalesProvider: PreferredLocalesProvider( + preferredLocaleOverride: "en_US", + systemPreferredLocalesGetter: { ["en_US"] } + ) + ) + } + + static func makeRemoteConfigManager( + backendConfiguration: BackendConfiguration, + appUserID: String + ) -> ( + manager: RemoteConfigManagerType, + blobStore: RemoteConfigBlobStoreType, + diskCache: RemoteConfigDiskCacheType + ) { + let diskCache = RemoteConfigDiskCache() + let blobStore = RemoteConfigBlobStore() + let sourceProvider = RemoteConfigSourceProvider(topicStore: diskCache) + let blobFetcher = RemoteConfigBlobFetcher( + blobStore: blobStore, + sourceProvider: sourceProvider, + downloader: URLSessionRemoteConfigBlobDownloader( + session: SimulatedTransportURLProtocol.makeSession() + ) + ) + let manager = RemoteConfigManager( + remoteConfigAPI: RemoteConfigAPI(backendConfig: backendConfiguration), + diskCache: diskCache, + blobStore: blobStore, + blobFetcher: blobFetcher, + currentUserProvider: BenchmarkCurrentUserProvider(appUserID: appUserID) + ) + return (manager, blobStore, diskCache) + } + +} + +private struct BenchmarkCurrentUserProvider: CurrentUserProvider { + + let appUserID: String + + var currentAppUserID: String { return self.appUserID } + var currentUserIsAnonymous: Bool { return false } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift new file mode 100644 index 0000000000..d6641521fa --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/FixtureServer.swift @@ -0,0 +1,96 @@ +import Foundation + +/// Routes simulated requests to deterministic fixture responses. +/// +/// Routing uses the shared `RequestKind` classifier, so the server, the metrics attribution, +/// and the warm-scenario validation can never disagree about what a request was. Warm paths +/// are supported the same way production is: offerings replies 304 to a matching +/// `X-RevenueCat-ETag`, config replies 204 to a matching manifest token in the POST body. +/// `killSwitchConfig` makes the config endpoint return a 4xx, which trips +/// `RemoteConfigManager`'s session kill switch. +final class FixtureServer { + + struct Response { + let statusCode: Int + let headers: [String: String] + let body: Data + } + + static let offeringsETag = "benchmark-offerings-v1" + + private let factory: BenchmarkPayloadFactory + private let killSwitchConfig: Bool + + // All responses are precomputed: the fixture path runs on every simulated request of + // every iteration. + private let offeringsResponse: Response + private let offeringsNotModifiedResponse: Response + private let configResponse: Response + private let configNotModifiedResponse = Response(statusCode: 204, headers: [:], body: Data()) + private let killSwitchResponse: Response + private let notFoundResponse: Response + + init(factory: BenchmarkPayloadFactory, killSwitchConfig: Bool = false) { + self.factory = factory + self.killSwitchConfig = killSwitchConfig + + let jsonHeadersWithETag = [ + "Content-Type": "application/json", + HTTPClient.ResponseHeader.eTag.rawValue: Self.offeringsETag + ] + self.offeringsResponse = Response(statusCode: 200, headers: jsonHeadersWithETag, body: factory.offeringsData) + self.offeringsNotModifiedResponse = Response(statusCode: 304, headers: jsonHeadersWithETag, body: Data()) + self.configResponse = Response( + statusCode: 200, + headers: ["Content-Type": "application/octet-stream"], + body: factory.configContainerData + ) + self.killSwitchResponse = Response( + statusCode: 400, + headers: ["Content-Type": "application/json"], + body: Data(#"{"code": 7000, "message": "benchmark kill switch"}"#.utf8) + ) + self.notFoundResponse = Response( + statusCode: 404, + headers: ["Content-Type": "application/json"], + body: Data(#"{"code": 7259, "message": "benchmark fixture not found"}"#.utf8) + ) + } + + func response(for request: URLRequest, bodyData: Data?) -> Response { + guard let url = request.url else { + return self.notFoundResponse + } + + switch RequestKind(url: url) { + case .offerings: + let requestETag = request.value(forHTTPHeaderField: HTTPClient.RequestHeader.eTag.rawValue) + return requestETag == Self.offeringsETag ? self.offeringsNotModifiedResponse : self.offeringsResponse + + case .config: + if self.killSwitchConfig { + return self.killSwitchResponse + } + // 204 requires the client to prove BOTH that its config is current (manifest) and + // that it still holds every prefetched blob (`prefetched_blobs`). A client that + // stops replaying its cached refs gets a full response, so a regression in that + // reporting shows up as warm 200s instead of silently passing. + if let bodyData, + let body = try? JSONSerialization.jsonObject(with: bodyData) as? [String: Any], + body["manifest"] as? String == self.factory.configManifest, + let prefetchedBlobs = body["prefetched_blobs"] as? [String], + Set(prefetchedBlobs).isSuperset(of: self.factory.workflowPrefetchRefs) { + return self.configNotModifiedResponse + } + return self.configResponse + + case .blob: + guard let blobRange = url.path.range(of: "/blobs/"), + let blob = self.factory.blobData(forRef: String(url.path[blobRange.upperBound...])) else { + return self.notFoundResponse + } + return Response(statusCode: 200, headers: ["Content-Type": "application/json"], body: blob) + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift new file mode 100644 index 0000000000..79330a4382 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/NetworkProfile.swift @@ -0,0 +1,109 @@ +import Foundation + +/// A named network condition applied by `SimulatedTransportURLProtocol`. +/// +/// API endpoints (dynamic backend calls) and CDN endpoints (static blob downloads) get separate +/// round-trip ranges because the config system's premise is that blobs are served from cheap, +/// cacheable CDN edges while `/v1/config` and `/v1/subscribers/.../offerings` hit the backend. +/// A flat per-request latency would structurally bias results against whichever mode issues +/// more requests, regardless of what those requests cost in production. +struct NetworkProfile { + + let name: String + /// Round-trip time range for dynamic API requests, in milliseconds. Sampled uniformly. + let apiRTTMs: ClosedRange + /// Round-trip time range for CDN blob requests, in milliseconds. Sampled uniformly. + let cdnRTTMs: ClosedRange + /// Downlink throughput; `nil` means body transfer time is not modeled. + let bandwidthBytesPerSec: Double? + + static let ideal = NetworkProfile( + name: "ideal", + apiRTTMs: 0...0, + cdnRTTMs: 0...0, + bandwidthBytesPerSec: nil + ) + + /// Good home wifi: low RTT, ~50 Mbit/s downlink. + static let wifi = NetworkProfile( + name: "wifi", + apiRTTMs: 25...40, + cdnRTTMs: 10...20, + bandwidthBytesPerSec: 50_000_000 / 8 + ) + + /// Typical LTE: higher and jitterier RTT, ~12 Mbit/s downlink. + static let lte = NetworkProfile( + name: "lte", + apiRTTMs: 55...110, + cdnRTTMs: 30...70, + bandwidthBytesPerSec: 12_000_000 / 8 + ) + + static func named(_ name: String) -> NetworkProfile? { + switch name { + case Self.ideal.name: return .ideal + case Self.wifi.name: return .wifi + case Self.lte.name: return .lte + default: return nil + } + } + + /// RTT class follows the request kind, the same classification everything else uses: + /// offerings and config are dynamic API calls, blobs are CDN downloads. + func rttMs(for kind: RequestKind, rng: inout SeededRandom) -> Double { + let range: ClosedRange + switch kind { + case .offerings, .config: range = self.apiRTTMs + case .blob: range = self.cdnRTTMs + } + guard range.lowerBound < range.upperBound else { return range.lowerBound } + return Double.random(in: range, using: &rng) + } + + /// Transfer time for `byteCount` at the profile's bandwidth, in milliseconds. + func transferTimeMs(forByteCount byteCount: Int) -> Double { + guard let bandwidth = self.bandwidthBytesPerSec, bandwidth > 0 else { return 0 } + return Double(byteCount) / bandwidth * 1_000 + } + +} + +/// Approximates packet loss for the simulated transport. +/// +/// This deliberately does NOT model real TCP behavior. Loss on a real connection shows up as +/// retransmission delays, stalled transfers, and occasional timeouts, not as a proportional +/// request failure rate. The approximation here: +/// - per delivered chunk, with probability `lossPercent`, one retransmission delay of 1x-2x RTT +/// is added before the chunk arrives; +/// - per request, with probability `(lossPercent/100)^3` (all retries of a critical packet +/// lost), the whole request fails with `URLError(.timedOut)` after one RTO so the SDK's +/// retry and fallback-host paths are exercised at a plausible rate. +/// Good enough to compare the two fetch systems under identical degraded conditions; not a +/// substitute for link-conditioner testing when absolute numbers matter. +struct LossModel { + + let lossPercent: Int + + var lossProbability: Double { + return Double(self.lossPercent) / 100 + } + + var requestFailureProbability: Double { + let loss = self.lossProbability + return loss * loss * loss + } + + /// Extra delay (ms) to add before a chunk of the response body, given the connection RTT. + func chunkRetransmitDelayMs(rttMs: Double, rng: inout SeededRandom) -> Double { + guard self.lossPercent > 0 else { return 0 } + guard Double.random(in: 0..<1, using: &rng) < self.lossProbability else { return 0 } + return rttMs * Double.random(in: 1...2, using: &rng) + } + + func shouldFailRequest(rng: inout SeededRandom) -> Bool { + guard self.lossPercent > 0 else { return false } + return Double.random(in: 0..<1, using: &rng) < self.requestFailureProbability + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift new file mode 100644 index 0000000000..8e403ff6de --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/RCContainerEncoder.swift @@ -0,0 +1,54 @@ +import Foundation + +/// Serializes RC-Container payloads for the fixture server, mirroring the wire format that +/// `RCContainer.Parser` reads: an 8-byte header followed by elements, each with a 24-byte +/// SHA-256-prefix checksum, little-endian wire size, encoding byte, 3 reserved bytes, the +/// payload, and zero padding to an 8-byte boundary. Only the `none` encoding is produced; +/// the benchmark measures transport and decode behavior, not compression codecs. +enum RCContainerEncoder { + + private static let version: UInt8 = 1 + private static let checksumSize = 24 + private static let paddingBoundary = 8 + + /// Builds a container whose first element is the config payload followed by the given + /// inline content elements, matching what `RemoteConfigContainer` expects. + static func container(config: Data, contentElements: [Data]) -> Data { + var data = Data([UInt8(ascii: "R"), UInt8(ascii: "C"), self.version, 0]) + data.append(contentsOf: [0, 0, 0, 0]) + + for element in [config] + contentElements { + self.appendElement(element, to: &data) + } + + return data + } + + /// The externally-referenced blob ref for a payload, computed by the same helper the SDK + /// uses to validate refs, so fixtures can never drift from the production format. + static func blobRef(for data: Data) -> String { + return data.withUnsafeBytes { RemoteConfigBlobRefHelpers.ref(for: $0) } + } + +} + +private extension RCContainerEncoder { + + static func checksum(for data: Data) -> Data { + return data.sha256.prefix(self.checksumSize) + } + + static func appendElement(_ payload: Data, to data: inout Data) { + data.append(self.checksum(for: payload)) + data.append(UInt32(payload.count).littleEndianData) + data.append(0) // ContentEncoding.none + data.append(contentsOf: [0, 0, 0]) + data.append(payload) + + let remainder = payload.count % self.paddingBoundary + if remainder != 0 { + data.append(Data(repeating: 0, count: self.paddingBoundary - remainder)) + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift new file mode 100644 index 0000000000..2044efd749 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SeededRandom.swift @@ -0,0 +1,20 @@ +import Foundation + +/// Deterministic SplitMix64 generator so benchmark runs are reproducible for a given `--seed`. +struct SeededRandom: RandomNumberGenerator { + + private var state: UInt64 + + init(seed: UInt64) { + self.state = seed + } + + mutating func next() -> UInt64 { + self.state &+= 0x9E3779B97F4A7C15 + var value = self.state + value = (value ^ (value >> 30)) &* 0xBF58476D1CE4E5B9 + value = (value ^ (value >> 27)) &* 0x94D049BB133111EB + return value ^ (value >> 31) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift new file mode 100644 index 0000000000..4e5721046a --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Passthrough.swift @@ -0,0 +1,63 @@ +import Foundation + +// MARK: - Passthrough recording (live runs) + +extension SimulatedTransportURLProtocol { + + /// Sessions the passthrough re-issues requests on. API traffic mirrors `HTTPClient`'s + /// pool: ephemeral, no URL cache (the SDK does its own ETag caching), one connection per + /// host. Blob traffic uses `URLSession.shared`, exactly what the production + /// `URLSessionRemoteConfigBlobDownloader` defaults to. Injectable for tests. + static var passthroughAPISession: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.urlCache = nil + configuration.httpMaximumConnectionsPerHost = 1 + return URLSession(configuration: configuration) + }() + static var passthroughBlobSession: URLSession = .shared + + func startPassthrough(url: URL, iteration: Int, startedAt: DispatchTime) { + let session: URLSession + switch RequestKind(url: url) { + case .offerings, .config: session = Self.passthroughAPISession + case .blob: session = Self.passthroughBlobSession + } + + let task = session.dataTask(with: self.request) { [weak self] data, response, error in + guard let self else { return } + + if error != nil { + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.client?.urlProtocol(self, didFailWithError: error ?? URLError(.unknown)) + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.backendFailure("non-HTTP response from \(url.host ?? "")") + ) + return + } + + self.recordTerminal(.success( + url: url, + iteration: iteration, + statusCode: httpResponse.statusCode, + bytesReceived: data?.count ?? 0, + startedAt: startedAt + )) + self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + if let data, !data.isEmpty { + self.client?.urlProtocol(self, didLoad: data) + } + self.client?.urlProtocolDidFinishLoading(self) + } + self.stateLock.withLock { + self.passthroughTask = task + } + task.resume() + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift new file mode 100644 index 0000000000..50acaba199 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol+Plans.swift @@ -0,0 +1,69 @@ +import Foundation + +// MARK: - Deterministic request plans + +extension SimulatedTransportURLProtocol { + + /// Identity of one simulated request. Plans derive from this key alone, so the same key + /// always produces the same plan, in any process, in any request order. + struct PlanKey { + + let seed: UInt64 + let iteration: Int + let url: URL + let attempt: Int + + /// FNV-1a over the request identity, mixed with the run seed. + var hashValue64: UInt64 { + var hash: UInt64 = 0xCBF29CE484222325 + func mix(_ byte: UInt8) { + hash ^= UInt64(byte) + hash = hash &* 0x100000001B3 + } + func mix(_ value: UInt64) { + for shift in stride(from: 0, to: 64, by: 8) { + mix(UInt8(truncatingIfNeeded: value >> shift)) + } + } + mix(self.seed) + mix(UInt64(bitPattern: Int64(self.iteration))) + for byte in self.url.absoluteString.utf8 { + mix(byte) + } + mix(UInt64(bitPattern: Int64(self.attempt))) + return hash + } + + } + + struct RequestPlan { + let rttMs: Double + let fails: Bool + let chunkDelaysMs: [Double] + } + + /// Computes the full network timeline of one request from its stable key. + static func requestPlan( + key: PlanKey, + bodyCount: Int, + profile: NetworkProfile, + loss: LossModel + ) -> RequestPlan { + var rng = SeededRandom(seed: key.hashValue64) + + let rttMs = profile.rttMs(for: RequestKind(url: key.url), rng: &rng) + let fails = loss.shouldFailRequest(rng: &rng) + var chunkDelaysMs: [Double] = [] + // Loss-free plans skip the per-chunk delay array entirely (delivery treats an empty + // array as zero delay everywhere). + if !fails && loss.lossPercent > 0 { + var offset = 0 + while offset < bodyCount { + chunkDelaysMs.append(loss.chunkRetransmitDelayMs(rttMs: rttMs, rng: &rng)) + offset += Self.chunkSize + } + } + return RequestPlan(rttMs: rttMs, fails: fails, chunkDelaysMs: chunkDelaysMs) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift new file mode 100644 index 0000000000..22fdb4acfd --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/SimulatedTransportURLProtocol.swift @@ -0,0 +1,358 @@ +import Foundation + +/// The transport used by every URLSession in the benchmark. Two modes: +/// +/// **Simulated**: requests resolve against an in-process `FixtureServer`, so no run can ever +/// touch the real network; fallback-host retries (`api-production.8-lives-cat.io`) resolve +/// against the same fixtures as the primary host. Responses are delivered asynchronously on a +/// private queue: headers after one sampled RTT, then the body in chunks paced by the profile's +/// bandwidth and the loss model's retransmission delays. Nothing here ever blocks a thread, so +/// concurrent requests overlap the way they would on a real connection pool. +/// +/// **Passthrough** (live runs): requests are re-issued unmodified against the real network and +/// recorded, so live runs get the same per-request metrics as simulated ones. API hosts go +/// through a single-connection session mirroring `HTTPClient`'s pool; every other host (blob +/// CDNs) goes through a default pool like the production blob downloader's `URLSession.shared`. +final class SimulatedTransportURLProtocol: URLProtocol { + + private enum Mode { + case simulated(server: FixtureServer, profile: NetworkProfile, loss: LossModel) + case passthrough + } + + private static let lock = NSLock() + private static var mode: Mode? + private static var seed: UInt64 = 0 + private static var iterationIndex: Int = 0 + /// Requests already planned this iteration, keyed by URL, so retries of the same URL get + /// distinct (but stable) plans. + private static var attemptCountsByURL: [String: Int] = [:] + private static var events: [TransportEvent] = [] + /// Requests started but not yet recorded. Every `startLoading` is balanced by exactly one + /// `record`, so zero means the transport is quiescent. + private static var inFlightCount = 0 + + static let chunkSize = 16 * 1024 + + /// Serial per request so header, chunks, and completion arrive in order; separate requests + /// each get their own queue and overlap freely. + private let deliveryQueue = DispatchQueue(label: "com.revenuecat.benchmark.transport.request") + private var pendingWorkItems: [DispatchWorkItem] = [] + var passthroughTask: URLSessionDataTask? + /// Captured at `startLoading` so a cancellation can still record this request's terminal + /// event (URLSession may cancel before any scheduled delivery block runs). + private var requestContext: (url: URL, iteration: Int, startedAt: DispatchTime)? + private var hasRecordedTerminalEvent = false + let stateLock = NSLock() + + /// Records the request's single terminal event. Every started request must record exactly + /// once (that's what balances the in-flight counter), whether it completes, fails, or is + /// cancelled mid-delivery. + func recordTerminal(_ event: TransportEvent) { + let isFirst = self.stateLock.withLock { () -> Bool in + guard !self.hasRecordedTerminalEvent else { return false } + self.hasRecordedTerminalEvent = true + return true + } + if isFirst { + Self.record(event) + } + } + + static func install(server: FixtureServer, profile: NetworkProfile, loss: LossModel, seed: UInt64) { + self.lock.withLock { + self.mode = .simulated(server: server, profile: profile, loss: loss) + self.seed = seed + self.iterationIndex = 0 + self.attemptCountsByURL = [:] + self.events = [] + } + } + + /// Marks the start of an iteration. Request plans are keyed by (seed, iteration, URL, + /// attempt), so samples stay stable across processes regardless of the order concurrent + /// requests happen to reach the transport, while still varying between iterations. + static func beginIteration(_ index: Int) { + self.lock.withLock { + self.iterationIndex = index + self.attemptCountsByURL = [:] + } + } + + static func installPassthrough() { + self.lock.withLock { + self.mode = .passthrough + self.events = [] + } + } + + static func uninstall() { + self.lock.withLock { + self.mode = nil + self.events = [] + } + } + + /// Blocks until every started request has recorded its event (or the timeout passes). + /// Launch measurements stop at offerings delivery, but trailing requests from the same + /// launch (e.g. the warm config 204 finishing after offerings) still belong to the + /// iteration's accounting, so callers wait for quiescence before draining. + @discardableResult + static func waitUntilIdle(timeoutMs: Int = 10_000) -> Bool { + let deadline = DispatchTime.now() + .milliseconds(timeoutMs) + while DispatchTime.now() < deadline { + if self.lock.withLock({ self.inFlightCount }) == 0 { + return true + } + usleep(2_000) + } + return self.lock.withLock { self.inFlightCount } == 0 + } + + /// Returns all events recorded since the last drain, oldest first. + static func drainEvents() -> [TransportEvent] { + return self.lock.withLock { + let drained = self.events + self.events = [] + return drained + } + } + + /// A URLSession whose requests go through this transport, for injecting into SDK seams + /// that build their own sessions (e.g. the remote config blob downloader). + static func makeSession() -> URLSession { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [Self.self] + configuration.urlCache = nil + return URLSession(configuration: configuration) + } + + // MARK: - URLProtocol + + override class func canInit(with request: URLRequest) -> Bool { + guard self.lock.withLock({ self.mode }) != nil, + let scheme = request.url?.scheme else { + return false + } + return scheme == "http" || scheme == "https" + } + + override class func canonicalRequest(for request: URLRequest) -> URLRequest { + return request + } + + override func startLoading() { + let started = DispatchTime.now() + guard let url = self.request.url, + let mode = Self.lock.withLock({ Self.mode }) else { + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.invalidFixture("Benchmark transport not installed") + ) + return + } + + let iteration = Self.lock.withLock { () -> Int in + Self.inFlightCount += 1 + return Self.iterationIndex + } + self.stateLock.withLock { + self.requestContext = (url, iteration, started) + } + switch mode { + case let .simulated(server, profile, loss): + self.startSimulated( + url: url, + server: server, + profile: profile, + loss: loss, + iteration: iteration, + startedAt: started + ) + case .passthrough: + self.startPassthrough(url: url, iteration: iteration, startedAt: started) + } + } + + override func stopLoading() { + let context = self.stateLock.withLock { () -> (url: URL, iteration: Int, startedAt: DispatchTime)? in + for item in self.pendingWorkItems { + item.cancel() + } + self.pendingWorkItems = [] + self.passthroughTask?.cancel() + self.passthroughTask = nil + return self.requestContext + } + // Cancellation can kill the scheduled delivery blocks before they record; backfill the + // terminal event so the in-flight counter always balances and cancelled requests show + // up as failures instead of disappearing. + if let context { + self.recordTerminal(.failure(url: context.url, iteration: context.iteration, startedAt: context.startedAt)) + } + } + +} + +// MARK: - Simulated delivery + +private extension SimulatedTransportURLProtocol { + + // swiftlint:disable:next function_parameter_count + func startSimulated( + url: URL, + server: FixtureServer, + profile: NetworkProfile, + loss: LossModel, + iteration: Int, + startedAt: DispatchTime + ) { + let bodyData = Self.bodyData(of: self.request) + let response = server.response(for: self.request, bodyData: bodyData) + + // The plan is derived from a stable per-request key, not drawn from a shared RNG in + // arrival order: concurrent requests reach the transport in a scheduler-dependent + // order, and order-dependent sampling would make same-seed runs disagree across + // processes. Only the attempt counter needs the lock; the plan itself is pure. + let key = Self.lock.withLock { () -> PlanKey in + let attempt = Self.attemptCountsByURL[url.absoluteString, default: 0] + Self.attemptCountsByURL[url.absoluteString] = attempt + 1 + return PlanKey(seed: Self.seed, iteration: Self.iterationIndex, url: url, attempt: attempt) + } + let plan = Self.requestPlan(key: key, bodyCount: response.body.count, profile: profile, loss: loss) + + if plan.fails { + self.deliverFailure(url: url, iteration: iteration, rttMs: plan.rttMs, startedAt: startedAt) + } else { + self.deliverResponse( + response, + url: url, + profile: profile, + plan: (plan.rttMs, plan.chunkDelaysMs), + iteration: iteration, + startedAt: startedAt + ) + } + } + +} + +// MARK: - Delivery helpers + +private extension SimulatedTransportURLProtocol { + + /// One retransmission-timeout's worth of waiting before the failure surfaces, so failed + /// attempts cost time like they do on a real link. + func deliverFailure(url: URL, iteration: Int, rttMs: Double, startedAt: DispatchTime) { + let rtoMs = max(1_000, rttMs * 2) + self.schedule(afterMs: rtoMs) { [weak self] in + guard let self else { return } + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.client?.urlProtocol(self, didFailWithError: URLError(.timedOut)) + } + } + + // swiftlint:disable:next function_parameter_count + func deliverResponse( + _ response: FixtureServer.Response, + url: URL, + profile: NetworkProfile, + plan: (rttMs: Double, chunkDelaysMs: [Double]), + iteration: Int, + startedAt: DispatchTime + ) { + guard let httpResponse = HTTPURLResponse( + url: url, + statusCode: response.statusCode, + httpVersion: "HTTP/1.1", + headerFields: response.headers + ) else { + self.recordTerminal(.failure(url: url, iteration: iteration, startedAt: startedAt)) + self.client?.urlProtocol( + self, + didFailWithError: BenchmarkError.invalidFixture("Could not create fixture HTTP response") + ) + return + } + + var deliveryOffsetMs = plan.rttMs + self.schedule(afterMs: deliveryOffsetMs) { [weak self] in + guard let self else { return } + self.client?.urlProtocol(self, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + } + + var offset = 0 + var chunkIndex = 0 + while offset < response.body.count { + let end = min(offset + Self.chunkSize, response.body.count) + // A slice, not a copy: the factory retains the parent body for the whole run. + let chunk = response.body[offset.. Void) { + let item = DispatchWorkItem(block: work) + self.stateLock.withLock { + self.pendingWorkItems.append(item) + } + self.deliveryQueue.asyncAfter(deadline: .now() + delayMs / 1_000, execute: item) + } + +} + +extension SimulatedTransportURLProtocol { + + static func record(_ event: TransportEvent) { + self.lock.withLock { + self.events.append(event) + self.inFlightCount -= 1 + } + } + + /// URLProtocol surfaces POST bodies as a stream; drain it so the fixture server can + /// inspect request payloads (the config manifest check needs this). + static func bodyData(of request: URLRequest) -> Data? { + if let body = request.httpBody { + return body + } + guard let stream = request.httpBodyStream else { + return nil + } + + stream.open() + defer { stream.close() } + + var data = Data() + let bufferSize = 16 * 1024 + var buffer = [UInt8](repeating: 0, count: bufferSize) + while stream.hasBytesAvailable { + let read = stream.read(&buffer, maxLength: bufferSize) + guard read > 0 else { break } + data.append(buffer, count: read) + } + return data + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift b/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift new file mode 100644 index 0000000000..cb2b6a5088 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Sources/TransportEvent.swift @@ -0,0 +1,82 @@ +import Foundation + +/// What a benchmark request is for. Classified once, from the URL, and stamped on every +/// `TransportEvent`, so routing, RTT class, connection pooling, phase attribution, and warm +/// validation all share one rule instead of re-deriving it from path strings. +enum RequestKind { + + case offerings + case config + /// Anything that is neither offerings nor config: blob downloads, whose URLs come from the + /// (real or fixture) config's `url_format` and so have no fixed shape. + case blob + + init(url: URL) { + let path = url.path + if path.hasSuffix("/offerings") { + self = .offerings + } else if path.hasSuffix("/config/app") { + self = .config + } else { + self = .blob + } + } + +} + +/// One completed (or failed) simulated request, for phase attribution and byte accounting. +struct TransportEvent { + + let kind: RequestKind + /// The iteration active when the request STARTED; measurements only aggregate events + /// stamped with their own iteration, so stragglers from failed or slow earlier launches + /// can never contaminate a measured row. + let iteration: Int + let host: String + let path: String + let statusCode: Int + let bytesReceived: Int + let startedAt: DispatchTime + let endedAt: DispatchTime + let failed: Bool + + /// The SDK's backup hosts; requests here mean the primary host failed over. + var isFallbackHostRequest: Bool { + return self.host.contains("8-lives-cat") || self.host.contains("rc-backup") + } + + static func failure(url: URL, iteration: Int, startedAt: DispatchTime) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + iteration: iteration, + host: url.host ?? "", + path: url.path, + statusCode: 0, + bytesReceived: 0, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: true + ) + } + + static func success( + url: URL, + iteration: Int, + statusCode: Int, + bytesReceived: Int, + startedAt: DispatchTime + ) -> TransportEvent { + return TransportEvent( + kind: RequestKind(url: url), + iteration: iteration, + host: url.host ?? "", + path: url.path, + statusCode: statusCode, + bytesReceived: bytesReceived, + startedAt: startedAt, + endedAt: DispatchTime.now(), + failed: false + ) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift new file mode 100644 index 0000000000..cbbd3bdb1a --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/AppLaunchMetricsTests.swift @@ -0,0 +1,276 @@ +import XCTest + +// swiftlint:disable:next attributes +@_spi(Internal) @testable import SDKConfigBenchmarkCore + +final class AppLaunchMetricsTests: BenchmarkTestCase { + + private static func sample(_ total: Double) -> LaunchSample { + var sample = LaunchSample() + sample.configuredMs = total * 0.1 + sample.customerInfoMs = total * 0.4 + sample.offeringsMs = total + sample.paywallAppearedMs = total * 1.1 + sample.paywallImpressionMs = total * 1.2 + return sample + } + + private func decodedRow( + warmupDiscarded: Int = 1, + samples: [LaunchSample?] + ) throws -> [String: Any] { + let row = AppLaunchMetrics.row( + mode: "app-launch-config", + scenario: "cold", + profile: "simulator", + projectID: "5f07e7e3", + warmupDiscarded: warmupDiscarded, + samples: samples + ) + let object = try JSONSerialization.jsonObject(with: Data(row.utf8)) + return try XCTUnwrap(object as? [String: Any]) + } + + func testPercentileMatchesCLIBenchmarkFormula() { + let sorted: [Double] = (1...10).map(Double.init) + for percentile in [50, 90, 95, 99] { + XCTAssertEqual( + AppLaunchMetrics.percentile(percentile, of: sorted), + BenchmarkMetrics.percentile(percentile, of: sorted), + "p\(percentile)" + ) + } + } + + func testRowCarriesEveryComparisonKeyField() throws { + let row = try self.decodedRow(samples: [Self.sample(100), Self.sample(200)]) + + XCTAssertEqual(row["mode"] as? String, "app-launch-config") + XCTAssertEqual(row["transport"] as? String, "live") + XCTAssertEqual(row["scenario"] as? String, "cold") + XCTAssertEqual(row["profile"] as? String, "simulator") + XCTAssertEqual(row["loss_percent"] as? Int, 0) + XCTAssertEqual(row["paywalls"] as? Int, 0) + XCTAssertEqual(row["workflows"] as? Int, 0) + XCTAssertEqual(row["seed"] as? Int, 0) + XCTAssertEqual(row["iterations"] as? Int, 2) + XCTAssertEqual(row["warmup_discarded"] as? Int, 1) + XCTAssertEqual(row["project_id"] as? String, "5f07e7e3") + } + + func testRowStatisticsUseOfferingsCompletionOfPostWarmupSamples() throws { + // Warmup discards by index: the 500ms first launch never enters the stats. The + // headline percentiles measure configure + getOfferings (matching the CLI tier); + // the paywall marks remain secondary phase means. + let row = try self.decodedRow( + samples: [Self.sample(500), Self.sample(100), Self.sample(300), Self.sample(200)] + ) + + XCTAssertEqual(row["measured_iterations"] as? Int, 3) + XCTAssertEqual(row["mean_ms"] as? Double, 200) + XCTAssertEqual(row["min_ms"] as? Double, 100) + XCTAssertEqual(row["max_ms"] as? Double, 300) + XCTAssertEqual(row["p50_ms"] as? Double, 200) + XCTAssertEqual(row["p95_ms"] as? Double, 300) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + XCTAssertEqual(row["offerings_ms_mean"] as? Double, 200) + XCTAssertEqual(row["paywall_appeared_ms_mean"] as? Double, 220) + XCTAssertEqual(row["paywall_impression_ms_mean"] as? Double, 240) + XCTAssertEqual(row["configured_ms_mean"] as? Double, 20) + XCTAssertEqual(row["customer_info_ms_mean"] as? Double, 80) + } + + func testSampleMissingImpressionCountsAsError() { + var missingImpression = Self.sample(100) + missingImpression.paywallImpressionMs = nil + + XCTAssertNotNil(AppLaunchMetrics.errorMessage(for: missingImpression)) + } + + func testIncompleteOrFailedSamplesCountAsErrorsAndNeverEnterStats() throws { + var failed = Self.sample(50) + failed.error = "BENCH_API_KEY missing" + var incomplete = Self.sample(75) + incomplete.paywallAppearedMs = nil + + let row = try self.decodedRow( + warmupDiscarded: 0, + samples: [Self.sample(100), failed, incomplete, nil] + ) + + XCTAssertEqual(row["iterations"] as? Int, 4) + XCTAssertEqual(row["measured_iterations"] as? Int, 1) + XCTAssertEqual(row["error_count"] as? Int, 3) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 3) + XCTAssertEqual(row["mean_ms"] as? Double, 100) + let firstError = try XCTUnwrap(row["first_error"] as? String) + XCTAssertTrue(firstError.contains("BENCH_API_KEY missing")) + } + + func testRowCarriesLaunchRetries() throws { + let row = AppLaunchMetrics.row( + mode: "app-launch-config", scenario: "cold", profile: "simulator", + projectID: "p", warmupDiscarded: 0, + samples: [Self.sample(100)], launchRetries: 2 + ) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(row.utf8)) as? [String: Any]) + XCTAssertEqual(object["launch_retries"] as? Int, 2) + } + + func testWarmupWindowErrorsAreCountedButNotPostWarmup() throws { + var failed = Self.sample(50) + failed.error = "boom" + + let row = try self.decodedRow(warmupDiscarded: 1, samples: [failed, Self.sample(100)]) + + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + XCTAssertEqual(row["measured_iterations"] as? Int, 1) + } + + func testLaunchSampleJSONRoundTrip() throws { + var sample = Self.sample(123.456) + sample.configPersistedMs = 88.5 + sample.blobsInline = 2 + sample.blobsDownloaded = 1 + sample.blobBytes = 40_000 + sample.maxInlineBlobBytes = 30_000 + sample.minDownloadedBlobBytes = 10_000 + sample.configPathActive = true + sample.configOutcome = "persisted" + let decoded = try XCTUnwrap(LaunchSample.decode(from: sample.jsonString())) + XCTAssertEqual(decoded, sample) + } + + func testRowAggregatesBlobAndConfigPathFields() throws { + var first = Self.sample(100) + first.configPersistedMs = 80 + first.lastBlobStoredMs = 90 + first.blobsInline = 2 + first.blobsDownloaded = 1 + first.blobBytes = 30_000 + first.maxInlineBlobBytes = 20_000 + first.minDownloadedBlobBytes = 9_000 + + var second = Self.sample(200) + second.configPersistedMs = 120 + second.lastBlobStoredMs = 150 + second.blobsInline = 2 + second.blobsDownloaded = 3 + second.blobBytes = 50_000 + second.maxInlineBlobBytes = 26_000 + second.minDownloadedBlobBytes = 7_000 + + let row = try self.decodedRow(warmupDiscarded: 0, samples: [first, second]) + + XCTAssertEqual(row["config_persisted_ms_mean"] as? Double, 100) + XCTAssertEqual(row["last_blob_stored_ms_mean"] as? Double, 120) + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 2) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 2) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 40_000) + // Extremes across the run expose the backend's inline-size budget empirically. + XCTAssertEqual(row["max_inline_blob_bytes"] as? Int, 26_000) + XCTAssertEqual(row["min_downloaded_blob_bytes"] as? Int, 7_000) + } + + func testRowOmitsConfigPathFieldsWhenNeverObserved() throws { + // A legacy-variant run: no config persist, no blobs. Counts stay (provably zero); + // timing means and size extremes are omitted rather than invented. + let row = try self.decodedRow(warmupDiscarded: 0, samples: [Self.sample(100)]) + + XCTAssertNil(row["config_persisted_ms_mean"]) + XCTAssertNil(row["last_blob_stored_ms_mean"]) + XCTAssertNil(row["max_inline_blob_bytes"]) + XCTAssertNil(row["min_downloaded_blob_bytes"]) + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 0) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 0) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 0) + } + + // MARK: - Blob log parsing + + /// The app tier observes the config path through the stock SDK's log stream, so the + /// parser's phrases must track `RemoteConfigStrings` exactly. These build the real SDK + /// messages; if the log copy changes, this fails instead of the app silently reporting 0. + func testBlobLogParserMatchesRealSDKLogStrings() throws { + let url = try XCTUnwrap(URL(string: "https://config.revenuecat-static.com/abc")) + let downloaded = RemoteConfigStrings.storedBlob("ref-a", byteCount: 41_213, url).description + XCTAssertEqual( + BlobLogParser.blobEvent(from: downloaded), + BlobStorageEvent(source: .downloaded, byteCount: 41_213) + ) + + let inline = RemoteConfigStrings.storedInlineBlob("ref-b", byteCount: 512).description + XCTAssertEqual( + BlobLogParser.blobEvent(from: inline), + BlobStorageEvent(source: .inline, byteCount: 512) + ) + + let persisted = RemoteConfigStrings.persistedConfiguration( + domain: "app", activeTopicCount: 1, referencedBlobCount: 2 + ).description + XCTAssertTrue(BlobLogParser.isConfigPersisted(persisted)) + XCTAssertNil(BlobLogParser.blobEvent(from: persisted)) + + // Fires on every config-variant launch (cold and warm): the runtime variant proof. + let refreshing = RemoteConfigStrings.refreshing( + domain: "app", manifestPresent: false, isAppBackgrounded: false + ).description + XCTAssertTrue(BlobLogParser.isConfigRefreshStarted(refreshing)) + XCTAssertFalse(BlobLogParser.isConfigRefreshStarted(persisted)) + + // Terminal outcomes: a failed refresh must never pass as a config-path measurement. + XCTAssertTrue(BlobLogParser.isConfigNotModified(RemoteConfigStrings.notModified.description)) + XCTAssertTrue(BlobLogParser.isConfigRefreshFailed( + RemoteConfigStrings.refreshFailed(.missingAppUserID()).description + )) + XCTAssertFalse(BlobLogParser.isConfigNotModified(refreshing)) + XCTAssertFalse(BlobLogParser.isConfigRefreshFailed(refreshing)) + } + + /// The app decides "content appeared" from event-type strings it copies out of the SDK; + /// these build the real SDK events and assert the copies still match. + func testContentAppearedEventTypesMatchRealSDKEvents() throws { + let paywallData = PaywallEvent.Data( + paywallIdentifier: nil, + offeringIdentifier: "offering", + paywallRevision: 0, + sessionID: .init(), + displayMode: .fullScreen, + localeIdentifier: "en_US", + darkMode: false, + source: nil, + presentedOfferingContext: .init(offeringIdentifier: "offering") + ) + let impression = PaywallEvent.impression(.init(), paywallData) + let impressionType = try XCTUnwrap(impression.toMap()["type"] as? String) + XCTAssertTrue(SDKObservedValues.contentAppearedEventTypes.contains(impressionType)) + + let stepStarted = WorkflowEvent.stepStarted(.init(), .init(workflowId: "w", stepId: "s")) + let stepType = try XCTUnwrap(stepStarted.toMap()["type"] as? String) + XCTAssertTrue(SDKObservedValues.contentAppearedEventTypes.contains(stepType)) + } + + /// The app wipes the SDK's UserDefaults suite by a copied name (the SDK's constant is + /// private). Behavioral pin: a value written through the SDK's own suite must be visible + /// through a suite created with the copied name. + func testRevenueCatUserDefaultsSuiteNameMatchesRealSDKSuite() throws { + let key = "benchmark-suite-pin-\(UUID().uuidString)" + UserDefaults.revenueCatSuite.set(true, forKey: key) + defer { UserDefaults.revenueCatSuite.removeObject(forKey: key) } + + let mirror = try XCTUnwrap( + UserDefaults(suiteName: SDKObservedValues.revenueCatUserDefaultsSuiteName) + ) + XCTAssertTrue(mirror.bool(forKey: key)) + } + + func testBlobLogParserToleratesLoggerPrefixesAndIgnoresOtherMessages() { + let prefixed = "[Purchases] - VERBOSE: 😻 Stored inline remote config blob 'r' with 77 bytes." + XCTAssertEqual(BlobLogParser.blobEvent(from: prefixed)?.byteCount, 77) + + XCTAssertNil(BlobLogParser.blobEvent(from: "Prefetching 3 remote config blobs requested.")) + XCTAssertFalse(BlobLogParser.isConfigPersisted("Received remote config with 1 active topics.")) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift new file mode 100644 index 0000000000..49adc91469 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkCommandTests.swift @@ -0,0 +1,140 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkCommandTests: BenchmarkTestCase { + + func testParseDefaults() throws { + let command = try BenchmarkCommand.parse([]) + + XCTAssertEqual(command.mode, .legacy) + XCTAssertEqual(command.scenario, .cold) + XCTAssertEqual(command.profileName, "ideal") + XCTAssertEqual(command.lossPercent, 0) + XCTAssertEqual(command.iterations, 25) + XCTAssertEqual(command.warmupIterations, 3) + XCTAssertEqual(command.paywallCount, 50) + XCTAssertEqual(command.workflowCount, 100) + XCTAssertEqual(command.seed, 42) + XCTAssertTrue(command.annotations.isEmpty) + } + + func testParseFullFlagSet() throws { + let command = try BenchmarkCommand.parse([ + "--mode", "config-killswitch", + "--scenario", "warm", + "--profile", "lte", + "--loss-percent", "20", + "--iterations", "100", + "--warmup-iterations", "5", + "--paywalls", "10", + "--workflows", "25", + "--seed", "7", + "--app-user-id", "user-1", + "--api-key", "appl_x", + "--annotation", "sdk_commit=abc123" + ]) + + XCTAssertEqual(command.mode, .configKillswitch) + XCTAssertTrue(command.mode.usesRemoteConfig) + XCTAssertEqual(command.scenario, .warm) + XCTAssertEqual(command.profileName, "lte") + XCTAssertEqual(command.lossPercent, 20) + XCTAssertEqual(command.iterations, 100) + XCTAssertEqual(command.warmupIterations, 5) + XCTAssertEqual(command.paywallCount, 10) + XCTAssertEqual(command.workflowCount, 25) + XCTAssertEqual(command.seed, 7) + XCTAssertEqual(command.appUserID, "user-1") + XCTAssertEqual(command.apiKey, "appl_x") + XCTAssertEqual(command.annotations["sdk_commit"], "abc123") + } + + func testParseRejectsUnknownMode() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--mode", "turbo"])) + } + + func testParseRejectsUnknownFlag() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--nope"])) + } + + func testParseRejectsMissingValue() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--iterations"])) + } + + func testParseRejectsWarmupNotBelowIterations() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--iterations", "5", "--warmup-iterations", "5"])) + } + + func testParseRejectsOutOfRangeLoss() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--loss-percent", "101"])) + } + + func testParseRejectsReservedAnnotationKeys() { + // Annotations overwriting identity or metric fields would let a row lie about what + // was measured. + for key in ["mode", "p50_ms", "post_warmup_error_count"] { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--annotation", "\(key)=x"]), key) + } + } + + // MARK: - Transport + + func testTransportDefaultsToSimulated() throws { + XCTAssertEqual(try BenchmarkCommand.parse([]).transport, .simulated) + } + + func testLiveTransportResolvesKeyFromEnvironmentButRequiresAProjectLabel() throws { + // An environment key without a project label could mislabel rows: nothing ties the + // key to the pinned default project, and rows from different projects must never + // compare as equivalents. + XCTAssertThrowsError(try BenchmarkCommand.parse( + ["--transport", "live"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + )) + + let command = try BenchmarkCommand.parse( + ["--transport", "live", "--project-id", "abc123"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + ) + XCTAssertEqual(command.transport, .live) + XCTAssertEqual(command.apiKey, "test_fromEnv") + XCTAssertEqual(command.projectID, "abc123") + } + + func testLiveTransportWithoutAnyKeyFails() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live"], environment: [:])) + } + + func testLiveTransportWithCustomKeyRequiresProjectID() throws { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--api-key", "appl_other"])) + + let labeled = try BenchmarkCommand.parse( + ["--transport", "live", "--api-key", "appl_other", "--project-id", "abc123"] + ) + XCTAssertEqual(labeled.projectID, "abc123") + XCTAssertEqual(labeled.apiKey, "appl_other") + } + + func testSimulatedTransportRejectsProjectID() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--project-id", "abc123"])) + } + + func testLiveTransportZeroesFixtureSizeKnobs() throws { + let command = try BenchmarkCommand.parse( + ["--transport", "live", "--project-id", "abc123", "--paywalls", "500", "--workflows", "500"], + environment: [BenchmarkProject.apiKeyEnvironmentVariable: "test_fromEnv"] + ) + + // Live payloads come from the pinned project, so fixture sizes must not label the row. + XCTAssertEqual(command.paywallCount, 0) + XCTAssertEqual(command.workflowCount, 0) + } + + func testLiveTransportRejectsSimulationOnlyKnobs() { + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--loss-percent", "10"])) + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--profile", "lte"])) + XCTAssertThrowsError(try BenchmarkCommand.parse(["--transport", "live", "--mode", "config-killswitch"])) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift new file mode 100644 index 0000000000..126380af0f --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkMetricsTests.swift @@ -0,0 +1,225 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkMetricsTests: BenchmarkTestCase { + + private func measurement( + totalMs: Double, + blobAccounting: BlobAccounting = .empty + ) -> IterationMeasurement { + return IterationMeasurement(totalMs: totalMs, events: [], blobAccounting: blobAccounting) + } + + func testNearestRankPercentiles() { + let sorted: [Double] = (1...100).map(Double.init) + + XCTAssertEqual(BenchmarkMetrics.percentile(50, of: sorted), 50) + XCTAssertEqual(BenchmarkMetrics.percentile(90, of: sorted), 90) + XCTAssertEqual(BenchmarkMetrics.percentile(95, of: sorted), 95) + XCTAssertEqual(BenchmarkMetrics.percentile(99, of: sorted), 99) + XCTAssertEqual(BenchmarkMetrics.percentile(99, of: [7]), 7) + } + + private func decodeRow(_ metrics: BenchmarkMetrics, command: BenchmarkCommand) throws -> [String: Any] { + let row = metrics.jsonlRow(for: command) + let object = try JSONSerialization.jsonObject(with: Data(row.utf8)) + return try XCTUnwrap(object as? [String: Any]) + } + + func testJSONLRowDiscardsWarmupIterationsByIndex() throws { + var command = BenchmarkCommand() + command.iterations = 5 + command.warmupIterations = 2 + + var metrics = BenchmarkMetrics() + // Two slow warmup iterations that must not pollute the stats. + for (index, totalMs) in [1_000.0, 900.0, 10.0, 20.0, 30.0].enumerated() { + metrics.record(self.measurement(totalMs: totalMs), iteration: index) + } + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["warmup_discarded"] as? Int, 2) + XCTAssertEqual(row["measured_iterations"] as? Int, 3) + XCTAssertEqual(row["mean_ms"] as? Double, 20) + XCTAssertEqual(row["max_ms"] as? Double, 30) + } + + func testFailedWarmupIterationDoesNotShiftDiscardWindow() throws { + var command = BenchmarkCommand() + command.iterations = 4 + command.warmupIterations = 2 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 1_000), iteration: 0) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 1) + metrics.record(self.measurement(totalMs: 10), iteration: 2) + metrics.record(self.measurement(totalMs: 30), iteration: 3) + + let row = try self.decodeRow(metrics, command: command) + + // A failed warmup iteration must not shift a measured iteration into the discard + // window: iterations 2 and 3 are measured, the warmup error is visible but separate. + XCTAssertEqual(row["measured_iterations"] as? Int, 2) + XCTAssertEqual(row["mean_ms"] as? Double, 20) + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 0) + } + + func testPostWarmupErrorsAreCalledOut() throws { + var command = BenchmarkCommand() + command.iterations = 3 + command.warmupIterations = 1 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10), iteration: 0) + metrics.record(self.measurement(totalMs: 20), iteration: 1) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 2) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["post_warmup_error_count"] as? Int, 1) + XCTAssertEqual(metrics.postWarmupErrorCount(warmupIterations: command.warmupIterations), 1) + let firstError = try XCTUnwrap(row["first_error"] as? String) + XCTAssertTrue(firstError.hasPrefix("iteration 2:")) + } + + func testJSONLRowCarriesConfigurationAndAnnotations() throws { + var command = BenchmarkCommand() + command.mode = .configKillswitch + command.scenario = .warm + command.profileName = "lte" + command.lossPercent = 20 + command.annotations = ["sdk_commit": "abc123"] + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 42), iteration: 0) + metrics.record(error: BenchmarkError.timeout("offerings fetch"), iteration: 1) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["mode"] as? String, "config-killswitch") + XCTAssertEqual(row["scenario"] as? String, "warm") + XCTAssertEqual(row["profile"] as? String, "lte") + XCTAssertEqual(row["loss_percent"] as? Int, 20) + XCTAssertEqual(row["sdk_commit"] as? String, "abc123") + XCTAssertEqual(row["error_count"] as? Int, 1) + XCTAssertNotNil(row["first_error"]) + XCTAssertEqual(row["p50_ms"] as? Double, 42) + } + + func testMeasurementDerivesPhasesFromEvents() throws { + let start = DispatchTime.now() + func time(_ offsetMs: UInt64) -> DispatchTime { + return DispatchTime(uptimeNanoseconds: start.uptimeNanoseconds + offsetMs * 1_000_000) + } + func event( + path: String, + host: String = "api.revenuecat.com", + from startMs: UInt64, + until endMs: UInt64, + status: Int = 200, + failed: Bool = false + ) throws -> TransportEvent { + let url = try XCTUnwrap(URL(string: "https://\(host)\(path)")) + return TransportEvent(kind: RequestKind(url: url), iteration: 0, host: host, path: path, + statusCode: status, bytesReceived: 100, + startedAt: time(startMs), endedAt: time(endMs), failed: failed) + } + + let measurement = IterationMeasurement(totalMs: 100, events: [ + try event(path: "/v1/config/app", from: 0, until: 10), + try event(path: "/blobs/aaa", host: "cdn.revenuecat.local", from: 10, until: 20), + try event(path: "/blobs/bbb", host: "cdn.revenuecat.local", from: 12, until: 30), + try event(path: "/v1/subscribers/u/offerings", from: 5, until: 45), + try event(path: "/v1/offerings", host: "api-production.8-lives-cat.io", + from: 46, until: 50, status: 0, failed: true) + ]) + + XCTAssertEqual(measurement.requestCount, 5) + XCTAssertEqual(measurement.bytesReceived, 500) + XCTAssertEqual(measurement.failedRequestCount, 1) + XCTAssertEqual(measurement.fallbackHostRequestCount, 1) + XCTAssertEqual(measurement.configMs ?? 0, 10, accuracy: 0.001) + XCTAssertEqual(measurement.blobMs ?? 0, 20, accuracy: 0.001) + XCTAssertEqual(measurement.offeringsMs ?? 0, 45, accuracy: 0.001) + } + + // MARK: - Blob accounting + + func testBlobAccountingAttributesInlineVersusDownloadedAndTracksExtremes() { + let accounting = BlobAccounting( + newRefSizes: ["inline-big": 30_000, "inline-small": 500, "cdn-a": 9_000, "cdn-b": 12_000], + downloadedRefs: ["cdn-a", "cdn-b", "cdn-never-stored"] + ) + + XCTAssertEqual(accounting.inlineCount, 2) + XCTAssertEqual(accounting.downloadedCount, 2) + XCTAssertEqual(accounting.totalBytes, 51_500) + // The extremes bracket the backend's inline-size budget. + XCTAssertEqual(accounting.maxInlineBytes, 30_000) + XCTAssertEqual(accounting.minDownloadedBytes, 9_000) + } + + func testBlobAccountingAttributesRefsToTopics() { + // Which topic each stored blob belongs to, from the persisted topic index; a stored + // ref no topic references is labeled explicitly rather than dropped. + let accounting = BlobAccounting( + newRefSizes: ["wf-1": 40_000, "wf-2": 30_000, "ui-app": 5_000, "mystery": 100], + downloadedRefs: ["ui-app"], + topicByRef: ["wf-1": "workflows", "wf-2": "workflows", "ui-app": "ui_config"] + ) + + XCTAssertEqual(accounting.countsByTopic, ["workflows": 2, "ui_config": 1, "unreferenced": 1]) + XCTAssertEqual(accounting.bytesByTopic, ["workflows": 70_000, "ui_config": 5_000, "unreferenced": 100]) + } + + func testJSONLRowCarriesBlobAccountingAggregates() throws { + var command = BenchmarkCommand() + command.iterations = 2 + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10, blobAccounting: BlobAccounting( + newRefSizes: ["i1": 1_000, "d1": 4_000], downloadedRefs: ["d1"], + topicByRef: ["i1": "workflows", "d1": "ui_config"] + )), iteration: 0) + metrics.record(self.measurement(totalMs: 20, blobAccounting: BlobAccounting( + newRefSizes: ["i2": 3_000, "d2": 2_000], downloadedRefs: ["d2"], + topicByRef: ["i2": "workflows", "d2": "ui_config"] + )), iteration: 1) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 1) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 1) + XCTAssertEqual(row["blob_bytes_mean"] as? Double, 5_000) + XCTAssertEqual(row["max_inline_blob_bytes"] as? Int, 3_000) + XCTAssertEqual(row["min_downloaded_blob_bytes"] as? Int, 2_000) + + let byTopic = try XCTUnwrap(row["blobs_by_topic"] as? [String: [String: Double]]) + XCTAssertEqual(byTopic["workflows"]?["count_mean"], 1) + XCTAssertEqual(byTopic["workflows"]?["bytes_mean"], 2_000) + XCTAssertEqual(byTopic["ui_config"]?["count_mean"], 1) + XCTAssertEqual(byTopic["ui_config"]?["bytes_mean"], 3_000) + } + + func testJSONLRowOmitsBlobExtremesWhenNoBlobsWereStored() throws { + var command = BenchmarkCommand() + command.iterations = 1 + command.warmupIterations = 0 + + var metrics = BenchmarkMetrics() + metrics.record(self.measurement(totalMs: 10), iteration: 0) + + let row = try self.decodeRow(metrics, command: command) + + XCTAssertEqual(row["blobs_inline_mean"] as? Double, 0) + XCTAssertEqual(row["blobs_downloaded_mean"] as? Double, 0) + XCTAssertNil(row["max_inline_blob_bytes"]) + XCTAssertNil(row["min_downloaded_blob_bytes"]) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift new file mode 100644 index 0000000000..2a7d904083 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkPayloadFactoryTests.swift @@ -0,0 +1,87 @@ +import XCTest + +@_spi(Internal) @testable import SDKConfigBenchmarkCore + +final class BenchmarkPayloadFactoryTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 5, workflowCount: 7) + + private func decodedConfiguration() throws -> RemoteConfiguration { + let container = try RemoteConfigContainer(data: self.factory.configContainerData) + let configData = try container.configElement.withDecodedPayloadBytes { Data($0) } + return try JSONDecoder.default.decode(RemoteConfiguration.self, from: configData) + } + + func testOfferingsDataDecodesIntoSDKModel() throws { + let response = try JSONDecoder.default.decode(OfferingsResponse.self, from: self.factory.offeringsData) + + XCTAssertEqual(response.offerings.count, 5) + XCTAssertEqual(response.currentOfferingId, "offering_0") + XCTAssertEqual(response.productIdentifiers.count, 5) + + for offering in response.offerings { + let components = try XCTUnwrap(offering.paywallComponents) + XCTAssertNil(components.errorInfo, "paywall components must decode without collected errors") + XCTAssertEqual(components.componentsConfig.base.stack.components.count, 1) + } + } + + func testConfigContainerDecodesIntoRemoteConfiguration() throws { + let configuration = try self.decodedConfiguration() + + XCTAssertEqual(configuration.domain, "app") + XCTAssertEqual(configuration.manifest, self.factory.configManifest) + XCTAssertEqual(Set(configuration.activeTopics), ["sources", "workflows", "ui_config"]) + + let workflowsTopic = try XCTUnwrap(configuration.topics.entries["workflows"]) + XCTAssertEqual(workflowsTopic.count, 7) + for item in workflowsTopic.values { + XCTAssertTrue(item.prefetch) + XCTAssertNotNil(item.blobRef) + } + + let sourcesTopic = try XCTUnwrap(configuration.topics.entries["sources"]) + XCTAssertNotNil(sourcesTopic["api"]) + XCTAssertNotNil(sourcesTopic["blob"]) + } + + func testAllReferencedBlobsResolveAndValidate() throws { + let configuration = try self.decodedConfiguration() + + // Prefetch covers exactly the workflow blobs (what offerings delivery awaits); + // ui_config blobs are referenced by their topic but intentionally not prefetched. + let prefetchRefs = Set(configuration.prefetchBlobs) + XCTAssertEqual(prefetchRefs.count, 7) + + let topicRefs = configuration.topics.entries.values.flatMap { topic in + topic.values.compactMap(\.blobRef) + } + XCTAssertEqual(Set(topicRefs).count, 7 + 2) + XCTAssertTrue(prefetchRefs.isSubset(of: Set(topicRefs))) + + for ref in Set(topicRefs) { + let blob = try XCTUnwrap(self.factory.blobData(forRef: ref), "missing blob for ref \(ref)") + XCTAssertEqual(RCContainerEncoder.blobRef(for: blob), ref, "blob refs must be content-addressed") + } + } + + func testWorkflowBlobDecodesIntoPublishedWorkflow() throws { + let configuration = try self.decodedConfiguration() + + let workflowsTopic = try XCTUnwrap(configuration.topics.entries["workflows"]) + let ref = try XCTUnwrap(workflowsTopic.values.first?.blobRef) + let blob = try XCTUnwrap(self.factory.blobData(forRef: ref)) + + let workflow = try JSONDecoder.default.decode(PublishedWorkflow.self, from: blob) + XCTAssertEqual(workflow.initialStepId, "step-1") + } + + func testPayloadsAreDeterministic() { + let other = BenchmarkPayloadFactory(paywallCount: 5, workflowCount: 7) + + XCTAssertEqual(other.offeringsData, self.factory.offeringsData) + XCTAssertEqual(other.configContainerData, self.factory.configContainerData) + XCTAssertEqual(Set(other.allBlobRefs), Set(self.factory.allBlobRefs)) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift new file mode 100644 index 0000000000..be80b0f4c4 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkRunnerValidationTests.swift @@ -0,0 +1,203 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class BenchmarkRunnerValidationTests: BenchmarkTestCase { + + private func events(path: String, host: String, statuses: [Int]) -> [TransportEvent] { + let start = DispatchTime.now() + return statuses.compactMap { status in + guard let url = URL(string: "https://\(host)\(path)") else { return nil } + return TransportEvent(kind: RequestKind(url: url), iteration: 0, host: host, path: path, + statusCode: status, bytesReceived: 0, + startedAt: start, endedAt: start, failed: false) + } + } + + private func measurement( + offeringsStatuses: [Int], + configStatuses: [Int] = [], + blobPaths: [String] = [] + ) -> IterationMeasurement { + var events = self.events(path: "/v1/subscribers/u/offerings", + host: "api.revenuecat.com", + statuses: offeringsStatuses) + events += self.events(path: "/v1/config/app", host: "api.revenuecat.com", statuses: configStatuses) + for path in blobPaths { + events += self.events(path: path, host: "cdn.revenuecat.local", statuses: [200]) + } + return IterationMeasurement(totalMs: 1, events: events) + } + + func testWarmLegacyIterationWithOnly304Passes() throws { + try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304]), + mode: .legacy, + iteration: 3 + ) + } + + func testWarmIterationWithFull200OfferingsFails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [200]), + mode: .legacy, + iteration: 3 + )) + } + + func testWarmIterationMixing304And200Fails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304, 200]), + mode: .legacy, + iteration: 3 + )) + } + + func testWarmConfigIterationRequires204() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [200]), + mode: .config, + iteration: 3 + )) + + XCTAssertNoThrow(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [204]), + mode: .config, + iteration: 3 + )) + } + + func testWarmKillSwitchIterationRequiresConfig4xx() throws { + try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [400]), + mode: .configKillswitch, + iteration: 3 + ) + + // Missing the config request, or getting a non-4xx, means the kill switch was not + // actually exercised and the row would measure something else. + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304]), + mode: .configKillswitch, + iteration: 3 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [200]), + mode: .configKillswitch, + iteration: 3 + )) + } + + func testWarmIterationReDownloadingBlobsFails() { + XCTAssertThrowsError(try BenchmarkRunner.validateWarmMeasurement( + self.measurement(offeringsStatuses: [304], configStatuses: [204], blobPaths: ["/blobs/aaa"]), + mode: .config, + iteration: 3 + )) + } + + // MARK: - Cold validation + + /// A cold config iteration whose config refresh failed still delivers offerings through + /// the legacy fallback, so without this check the row would pass as a config measurement. + func testColdConfigModeRequiresASuccessfulConfigRequest() throws { + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .config, iteration: 1 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [500, 503]), mode: .config, iteration: 1 + )) + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [500, 200]), mode: .config, iteration: 1 + )) + } + + func testColdKillswitchModeRequiresThe4xx() throws { + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [400]), mode: .configKillswitch, iteration: 0 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: [200]), mode: .configKillswitch, iteration: 0 + )) + XCTAssertThrowsError(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .configKillswitch, iteration: 0 + )) + } + + func testColdLegacyModeNeedsNoConfigRequest() throws { + XCTAssertNoThrow(try BenchmarkRunner.validateColdMeasurement( + self.measurement(configStatusCodes: []), mode: .legacy, iteration: 0 + )) + } + + private static let configURL = URL(string: "https://api.revenuecat.com/v1/config/app") + + private func measurement(configStatusCodes: [Int]) -> IterationMeasurement { + guard let url = Self.configURL else { + preconditionFailure("static config URL failed to parse") + } + let events: [TransportEvent] = configStatusCodes.map { status in + TransportEvent.success( + url: url, iteration: 0, statusCode: status, bytesReceived: 10, startedAt: .now() + ) + } + return IterationMeasurement(totalMs: 10, events: events) + } + + // MARK: - Blob accounting + + func testBlobAccountingCountsOnlyRefsStoredThisIterationAndAttributesByRequest() throws { + let store = InMemoryBlobStore(contents: [ + "pre-existing": Data(count: 100), + "inline-new": Data(count: 2_000), + "cdn-new": Data(count: 8_000) + ]) + let url = try XCTUnwrap(URL(string: "https://cdn.revenuecat.local/blobs/cdn-new")) + let events = [ + TransportEvent.success(url: url, iteration: 0, statusCode: 200, + bytesReceived: 8_000, startedAt: DispatchTime.now()) + ] + + let accounting = BenchmarkRunner.blobAccounting( + blobStore: store, + refsBeforeLaunch: ["pre-existing"], + events: events + ) + + XCTAssertEqual(accounting.inlineCount, 1) + XCTAssertEqual(accounting.downloadedCount, 1) + XCTAssertEqual(accounting.totalBytes, 10_000) + XCTAssertEqual(accounting.maxInlineBytes, 2_000) + XCTAssertEqual(accounting.minDownloadedBytes, 8_000) + } + + func testBlobAccountingIsEmptyForLegacyModeWithoutABlobStore() { + let accounting = BenchmarkRunner.blobAccounting(blobStore: nil, refsBeforeLaunch: [], events: []) + + XCTAssertEqual(accounting.inlineCount, 0) + XCTAssertEqual(accounting.downloadedCount, 0) + XCTAssertNil(accounting.maxInlineBytes) + XCTAssertNil(accounting.minDownloadedBytes) + } + +} + +private final class InMemoryBlobStore: RemoteConfigBlobStoreType { + + private var contents: [String: Data] + + init(contents: [String: Data]) { + self.contents = contents + } + + func contains(ref: String) -> Bool { return self.contents[ref] != nil } + func read(ref: String) -> Data? { return self.contents[ref] } + func write(ref: String, bytes: UnsafeRawBufferPointer) -> Bool { + self.contents[ref] = Data(bytes.bindMemory(to: UInt8.self)) + return true + } + func cachedRefs() -> Set { return Set(self.contents.keys) } + func retainOnly(_ refs: Set) { self.contents = self.contents.filter { refs.contains($0.key) } } + func clear() { self.contents = [:] } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift new file mode 100644 index 0000000000..c6e7f68b0b --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkSDKStackTests.swift @@ -0,0 +1,130 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +/// End-to-end: real `OfferingsManager` (and `RemoteConfigManager` for config modes) against +/// the simulated transport. These are the tests that prove the benchmark measures the actual +/// SDK flows rather than a hand-rolled imitation of them. +final class BenchmarkSDKStackTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 3, workflowCount: 4) + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + super.tearDown() + } + + private func install(killSwitch: Bool = false) { + SimulatedTransportURLProtocol.install( + server: FixtureServer(factory: self.factory, killSwitchConfig: killSwitch), + profile: .ideal, + loss: LossModel(lossPercent: 0), + seed: 42 + ) + } + + private func fetchOfferings( + _ stack: BenchmarkSDKStack, + appUserID: String, + timeout: TimeInterval = 15 + ) throws -> Offerings { + let expectation = self.expectation(description: "offerings delivered") + let result = LockedValue>() + stack.offeringsManager.offerings(appUserID: appUserID, fetchCurrent: true) { offerings in + result.set(offerings) + expectation.fulfill() + } + self.wait(for: [expectation], timeout: timeout) + return try XCTUnwrap(result.get()).get() + } + + func testLegacyModeFetchesOfferingsWithSingleRequest() throws { + self.install() + let stack = BenchmarkSDKStack(mode: .legacy, apiKey: "appl_benchmark", appUserID: "legacy-user") + stack.clearAllDiskState() + + let offerings = try self.fetchOfferings(stack, appUserID: "legacy-user") + + XCTAssertEqual(offerings.all.count, 3) + XCTAssertNotNil(offerings.current) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertTrue(events[0].path.hasSuffix("/offerings")) + } + + func testConfigModeFetchesConfigBlobsAndOfferings() throws { + self.install() + let stack = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "config-user") + stack.clearAllDiskState() + + stack.refreshRemoteConfigIfWired() + let offerings = try self.fetchOfferings(stack, appUserID: "config-user") + + XCTAssertEqual(offerings.all.count, 3) + + let events = SimulatedTransportURLProtocol.drainEvents() + let paths = events.map(\.path) + XCTAssertTrue(paths.contains { $0.hasSuffix("/config/app") }, "expected a config fetch in \(paths)") + XCTAssertTrue(paths.contains { $0.contains("/blobs/") }, "expected blob downloads in \(paths)") + XCTAssertTrue(paths.contains { $0.hasSuffix("/offerings") }, "expected an offerings fetch in \(paths)") + + // The workflows topic marks every workflow blob prefetch, so offerings delivery waits + // for all of them. ui_config blobs are not prefetched and must not be fetched here. + let blobRequests = paths.filter { $0.contains("/blobs/") } + XCTAssertEqual(Set(blobRequests).count, 4) + } + + func testKillSwitchModeStillDeliversOfferings() throws { + self.install(killSwitch: true) + let stack = BenchmarkSDKStack(mode: .configKillswitch, apiKey: "appl_benchmark", appUserID: "kill-user") + stack.clearAllDiskState() + + stack.refreshRemoteConfigIfWired() + let offerings = try self.fetchOfferings(stack, appUserID: "kill-user") + + XCTAssertEqual(offerings.all.count, 3) + XCTAssertEqual(stack.remoteConfigManager?.isDisabled, true, "4xx must trip the session kill switch") + + let events = SimulatedTransportURLProtocol.drainEvents() + let paths = events.map(\.path) + XCTAssertTrue(paths.contains { $0.hasSuffix("/config/app") }) + XCTAssertFalse(paths.contains { $0.contains("/blobs/") }, "no blobs after a config 4xx") + } + + func testWarmRelaunchServes304And204() throws { + self.install() + + // Priming launch populates etags, offerings disk cache, and the persisted config. + let priming = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "warm-user") + priming.clearAllDiskState() + priming.refreshRemoteConfigIfWired() + _ = try self.fetchOfferings(priming, appUserID: "warm-user") + _ = SimulatedTransportURLProtocol.drainEvents() + + // Simulated relaunch: fresh in-memory stack, retained disk state. + let relaunch = BenchmarkSDKStack(mode: .config, apiKey: "appl_benchmark", appUserID: "warm-user") + relaunch.refreshRemoteConfigIfWired() + _ = try self.fetchOfferings(relaunch, appUserID: "warm-user") + + let events = SimulatedTransportURLProtocol.drainEvents() + let statusesByPath = Dictionary( + events.map { ($0.path, $0.statusCode) }, + uniquingKeysWith: { first, _ in first } + ) + + XCTAssertTrue( + statusesByPath.contains { $0.key.hasSuffix("/offerings") && $0.value == 304 }, + "warm offerings must be revalidated via 304, got \(statusesByPath)" + ) + XCTAssertTrue( + statusesByPath.contains { $0.key.hasSuffix("/config/app") && $0.value == 204 }, + "warm config must be revalidated via manifest 204, got \(statusesByPath)" + ) + XCTAssertFalse( + statusesByPath.contains { $0.key.contains("/blobs/") }, + "warm relaunch must not re-download blobs, got \(statusesByPath)" + ) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift new file mode 100644 index 0000000000..8c66ff97cb --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/BenchmarkTestCase.swift @@ -0,0 +1,48 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +/// Minimal base class for benchmark harness tests. The benchmark test target does not link +/// the main UnitTests infrastructure, so it carries its own `XCTestCase` subclass to satisfy +/// the repo convention that test classes never inherit `XCTestCase` directly. +/// +/// Every test gets a fresh disk-cache root so tests never touch the real user Library and +/// never see each other's (or a previous run's) cached state. +/// Thread-safe box for values written from URLSession completion handlers and read after +/// `wait(for:)`, shared by every async transport test. +final class LockedValue: @unchecked Sendable { + + private let lock = NSLock() + private var value: Value? + + func set(_ value: Value) { + self.lock.withLock { self.value = value } + } + + func get() -> Value? { + return self.lock.withLock { self.value } + } + +} + +// swiftlint:disable:next xctestcase_superclass +class BenchmarkTestCase: XCTestCase { + + private var diskRoot: URL! + + override func setUp() { + super.setUp() + self.diskRoot = FileManager.default.temporaryDirectory + .appendingPathComponent("SDKConfigBenchmarkTests-\(UUID().uuidString)", isDirectory: true) + DirectoryHelper.benchmarkBaseDirectoryOverride = self.diskRoot + } + + override func tearDown() { + DirectoryHelper.benchmarkBaseDirectoryOverride = nil + if let diskRoot = self.diskRoot { + try? FileManager.default.removeItem(at: diskRoot) + } + super.tearDown() + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift new file mode 100644 index 0000000000..58d8a94e26 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/FixtureServerTests.swift @@ -0,0 +1,202 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class FixtureServerTests: BenchmarkTestCase { + + private let factory = BenchmarkPayloadFactory(paywallCount: 3, workflowCount: 4) + private var server: FixtureServer! + + override func setUp() { + super.setUp() + self.server = FixtureServer(factory: self.factory) + } + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + super.tearDown() + } + + private func request(_ urlString: String, eTag: String? = nil) throws -> URLRequest { + var request = URLRequest(url: try XCTUnwrap(URL(string: urlString))) + if let eTag { + request.setValue(eTag, forHTTPHeaderField: "X-RevenueCat-ETag") + } + return request + } + + func testOfferingsServes200WithETagThen304OnMatch() throws { + let url = "https://api.revenuecat.com/v1/subscribers/user-1/offerings" + + let cold = self.server.response(for: try self.request(url), bodyData: nil) + XCTAssertEqual(cold.statusCode, 200) + XCTAssertEqual(cold.headers["X-RevenueCat-ETag"], FixtureServer.offeringsETag) + XCTAssertEqual(cold.body, self.factory.offeringsData) + + let warm = self.server.response( + for: try self.request(url, eTag: FixtureServer.offeringsETag), + bodyData: nil + ) + XCTAssertEqual(warm.statusCode, 304) + XCTAssertEqual(warm.headers["X-RevenueCat-ETag"], FixtureServer.offeringsETag) + XCTAssertTrue(warm.body.isEmpty) + } + + func testOfferingsFallbackHostPathRoutesIdentically() throws { + let fallback = self.server.response( + for: try self.request("https://api-production.8-lives-cat.io/v1/offerings"), + bodyData: nil + ) + + XCTAssertEqual(fallback.statusCode, 200) + XCTAssertEqual(fallback.body, self.factory.offeringsData) + } + + private func configBody(manifest: String? = nil, prefetchedBlobs: [String]? = nil) throws -> Data { + var body: [String: Any] = ["app_user_id": "u"] + if let manifest { + body["manifest"] = manifest + } + if let prefetchedBlobs { + body["prefetched_blobs"] = prefetchedBlobs + } + return try JSONSerialization.data(withJSONObject: body) + } + + func testConfigServes200ContainerThen204OnMatchingManifestAndBlobs() throws { + let url = "https://api.revenuecat.com/v1/config/app" + + let cold = self.server.response(for: try self.request(url), bodyData: try self.configBody()) + XCTAssertEqual(cold.statusCode, 200) + XCTAssertEqual(cold.body, self.factory.configContainerData) + + // 204 requires the client to prove its config is current AND that it still holds the + // prefetched blobs it was told to cache. + let warm = self.server.response( + for: try self.request(url), + bodyData: try self.configBody( + manifest: self.factory.configManifest, + prefetchedBlobs: Array(self.factory.workflowPrefetchRefs) + ) + ) + XCTAssertEqual(warm.statusCode, 204) + XCTAssertTrue(warm.body.isEmpty) + } + + func testConfigMatchingManifestWithoutCachedBlobProofGets200() throws { + let url = "https://api.revenuecat.com/v1/config/app" + + for prefetchedBlobs in [nil, [], Array(self.factory.workflowPrefetchRefs.dropFirst())] as [[String]?] { + let response = self.server.response( + for: try self.request(url), + bodyData: try self.configBody(manifest: self.factory.configManifest, prefetchedBlobs: prefetchedBlobs) + ) + XCTAssertEqual(response.statusCode, 200, "missing blob proof \(String(describing: prefetchedBlobs))") + } + } + + func testKillSwitchConfigReturns400ButOfferingsStillServe() throws { + let killServer = FixtureServer(factory: self.factory, killSwitchConfig: true) + + let config = killServer.response( + for: try self.request("https://api.revenuecat.com/v1/config/app"), + bodyData: nil + ) + XCTAssertEqual(config.statusCode, 400) + + let offerings = killServer.response( + for: try self.request("https://api.revenuecat.com/v1/subscribers/u/offerings"), + bodyData: nil + ) + XCTAssertEqual(offerings.statusCode, 200) + } + + func testBlobRefRoundTrip() throws { + let ref = try XCTUnwrap(self.factory.allBlobRefs.first) + + let response = self.server.response( + for: try self.request("https://cdn.revenuecat.local/blobs/\(ref)"), + bodyData: nil + ) + + XCTAssertEqual(response.statusCode, 200) + XCTAssertEqual(response.body, self.factory.blobData(forRef: ref)) + } + + func testUnknownPathReturns404() throws { + let response = self.server.response( + for: try self.request("https://api.revenuecat.com/v1/unknown"), + bodyData: nil + ) + + XCTAssertEqual(response.statusCode, 404) + } + + // MARK: - Transport integration + + private func installTransport(lossPercent: Int = 0) { + SimulatedTransportURLProtocol.install( + server: self.server, + profile: .ideal, + loss: LossModel(lossPercent: lossPercent), + seed: 42 + ) + } + + private func fetch(_ urlString: String, timeout: TimeInterval = 10) throws -> (data: Data?, + statusCode: Int?, + error: Error?) { + let url = try XCTUnwrap(URL(string: urlString)) + let session = SimulatedTransportURLProtocol.makeSession() + + let expectation = self.expectation(description: "request completes") + let result = LockedValue<(Data?, Int?, Error?)>() + session.dataTask(with: url) { data, response, error in + result.set((data, (response as? HTTPURLResponse)?.statusCode, error)) + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: timeout) + + let (data, statusCode, error) = result.get() ?? (nil, nil, nil) + return (data, statusCode, error) + } + + func testTransportDeliversFixtureBodyAndRecordsEvent() throws { + self.installTransport() + let ref = try XCTUnwrap(self.factory.allBlobRefs.first) + + let received = try self.fetch("https://cdn.revenuecat.local/blobs/\(ref)") + + XCTAssertNil(received.error) + XCTAssertEqual(received.statusCode, 200) + XCTAssertEqual(received.data, self.factory.blobData(forRef: ref)) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.host, "cdn.revenuecat.local") + XCTAssertEqual(events.first?.kind, .blob) + XCTAssertEqual(events.first?.bytesReceived, self.factory.blobData(forRef: ref)?.count) + XCTAssertTrue(SimulatedTransportURLProtocol.drainEvents().isEmpty, "drain must clear events") + } + + func testTransportInterceptsRealAPIHostsSoNothingLeaks() throws { + self.installTransport() + + let received = try self.fetch("https://api.revenuecat.com/v1/subscribers/u/offerings") + + XCTAssertEqual(received.data, self.factory.offeringsData, "real API hosts must resolve to fixtures") + } + + func testTransportModelsLossAsTimedOutFailure() throws { + self.installTransport(lossPercent: 100) + + let received = try self.fetch("https://api.revenuecat.com/v1/subscribers/u/offerings") + + XCTAssertEqual((received.error as? URLError)?.code, .timedOut) + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertTrue(events.first?.failed == true) + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift new file mode 100644 index 0000000000..a2c84b26d2 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/NetworkProfileTests.swift @@ -0,0 +1,132 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class NetworkProfileTests: BenchmarkTestCase { + + func testProfileLookupByName() { + XCTAssertEqual(NetworkProfile.named("ideal")?.name, "ideal") + XCTAssertEqual(NetworkProfile.named("wifi")?.name, "wifi") + XCTAssertEqual(NetworkProfile.named("lte")?.name, "lte") + XCTAssertNil(NetworkProfile.named("5g")) + } + + func testCDNLatencyIsBelowAPILatencyForRealProfiles() { + for profile in [NetworkProfile.wifi, .lte] { + XCTAssertLessThan(profile.cdnRTTMs.upperBound, profile.apiRTTMs.upperBound, profile.name) + } + } + + func testRTTSamplingIsDeterministicForSameSeed() { + var first = SeededRandom(seed: 9) + var second = SeededRandom(seed: 9) + + let firstSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(for: .offerings, rng: &first) } + let secondSamples = (0..<32).map { _ in NetworkProfile.lte.rttMs(for: .offerings, rng: &second) } + + XCTAssertEqual(firstSamples, secondSamples) + for sample in firstSamples { + XCTAssertTrue(NetworkProfile.lte.apiRTTMs.contains(sample)) + } + } + + func testBlobRequestsSampleFromCDNRange() { + var rng = SeededRandom(seed: 9) + + for _ in 0..<32 { + let sample = NetworkProfile.lte.rttMs(for: .blob, rng: &rng) + XCTAssertTrue(NetworkProfile.lte.cdnRTTMs.contains(sample)) + } + } + + func testRequestKindClassification() throws { + func kind(_ urlString: String) throws -> RequestKind { + return RequestKind(url: try XCTUnwrap(URL(string: urlString))) + } + + XCTAssertEqual(try kind("https://api.revenuecat.com/v1/subscribers/u/offerings"), .offerings) + XCTAssertEqual(try kind("https://api-production.8-lives-cat.io/v1/offerings"), .offerings) + XCTAssertEqual(try kind("https://api.revenuecat.com/v1/config/app"), .config) + XCTAssertEqual(try kind("https://cdn.revenuecat.local/blobs/abc"), .blob) + XCTAssertEqual(try kind("https://d123.cloudfront.net/some/real/cdn/path"), .blob) + } + + func testIdealProfileAddsNoDelay() { + var rng = SeededRandom(seed: 1) + + XCTAssertEqual(NetworkProfile.ideal.rttMs(for: .config, rng: &rng), 0) + XCTAssertEqual(NetworkProfile.ideal.transferTimeMs(forByteCount: 10_000_000), 0) + } + + func testTransferTimeScalesWithBytes() { + let profile = NetworkProfile.lte + + let small = profile.transferTimeMs(forByteCount: 15_000) + let large = profile.transferTimeMs(forByteCount: 1_500_000) + + XCTAssertGreaterThan(large, small * 90) + XCTAssertEqual(profile.transferTimeMs(forByteCount: 1_500_000), 1_000, accuracy: 1) + } + + func testZeroLossAddsNoDelaysAndNeverFails() { + var rng = SeededRandom(seed: 3) + let loss = LossModel(lossPercent: 0) + + for _ in 0..<10_000 { + XCTAssertEqual(loss.chunkRetransmitDelayMs(rttMs: 80, rng: &rng), 0) + XCTAssertFalse(loss.shouldFailRequest(rng: &rng)) + } + } + + func testLossFailureRateMatchesHeuristic() { + var rng = SeededRandom(seed: 4) + let loss = LossModel(lossPercent: 30) + let trials = 10_000 + + let failures = (0.. SimulatedTransportURLProtocol.RequestPlan { + return SimulatedTransportURLProtocol.requestPlan( + key: .init(seed: 42, iteration: iteration, url: target, attempt: attempt), + bodyCount: 100_000, profile: .lte, loss: loss + ) + } + + // Same key produces the identical plan no matter what was computed before it: request + // arrival order (which is scheduler-dependent) must not influence sampling. + let before = plan(url) + _ = plan(otherURL) + _ = plan(otherURL, attempt: 1) + let after = plan(url) + XCTAssertEqual(before.rttMs, after.rttMs) + XCTAssertEqual(before.fails, after.fails) + XCTAssertEqual(before.chunkDelaysMs, after.chunkDelaysMs) + + // Different iterations and attempts get different samples, so distributions stay real. + XCTAssertNotEqual(plan(url, iteration: 2).rttMs, before.rttMs) + XCTAssertNotEqual(plan(url, attempt: 1).rttMs, before.rttMs) + } + + func testLossChunkDelaysAreDeterministicForSameSeed() { + let loss = LossModel(lossPercent: 20) + var first = SeededRandom(seed: 5) + var second = SeededRandom(seed: 5) + + let firstDelays = (0..<256).map { _ in loss.chunkRetransmitDelayMs(rttMs: 80, rng: &first) } + let secondDelays = (0..<256).map { _ in loss.chunkRetransmitDelayMs(rttMs: 80, rng: &second) } + + XCTAssertEqual(firstDelays, secondDelays) + XCTAssertTrue(firstDelays.contains { $0 > 0 }, "20% loss over 256 chunks should add some delay") + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift new file mode 100644 index 0000000000..fa5b5e0386 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/PassthroughTransportTests.swift @@ -0,0 +1,100 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +/// Passthrough (live) mode, verified against an in-test stub backend instead of the real +/// network: requests re-issue through the passthrough sessions and get recorded as events. +final class PassthroughTransportTests: BenchmarkTestCase { + + private var originalAPISession: URLSession! + private var originalBlobSession: URLSession! + + override func setUp() { + super.setUp() + self.originalAPISession = SimulatedTransportURLProtocol.passthroughAPISession + self.originalBlobSession = SimulatedTransportURLProtocol.passthroughBlobSession + + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubBackendURLProtocol.self] + let stubSession = URLSession(configuration: configuration) + SimulatedTransportURLProtocol.passthroughAPISession = stubSession + SimulatedTransportURLProtocol.passthroughBlobSession = stubSession + } + + override func tearDown() { + SimulatedTransportURLProtocol.uninstall() + SimulatedTransportURLProtocol.passthroughAPISession = self.originalAPISession + SimulatedTransportURLProtocol.passthroughBlobSession = self.originalBlobSession + StubBackendURLProtocol.requestCount = 0 + super.tearDown() + } + + func testPassthroughForwardsResponseAndRecordsEvent() throws { + SimulatedTransportURLProtocol.installPassthrough() + let session = SimulatedTransportURLProtocol.makeSession() + let url = try XCTUnwrap(URL(string: "https://api.revenuecat.com/v1/subscribers/u/offerings")) + + let expectation = self.expectation(description: "request completes") + let result = LockedValue<(data: Data?, statusCode: Int?)>() + session.dataTask(with: url) { data, response, _ in + result.set((data, (response as? HTTPURLResponse)?.statusCode)) + expectation.fulfill() + }.resume() + self.wait(for: [expectation], timeout: 5) + + let received = try XCTUnwrap(result.get()) + XCTAssertEqual(received.statusCode, 200) + XCTAssertEqual(received.data, StubBackendURLProtocol.stubBody) + XCTAssertEqual(StubBackendURLProtocol.requestCount, 1, "request must reach the (stub) backend") + + let events = SimulatedTransportURLProtocol.drainEvents() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.statusCode, 200) + XCTAssertEqual(events.first?.bytesReceived, StubBackendURLProtocol.stubBody.count) + XCTAssertEqual(events.first?.failed, false) + } + + func testUninstalledTransportDoesNotClaimRequests() { + SimulatedTransportURLProtocol.uninstall() + + let request = URLRequest(url: URL(fileURLWithPath: "/dev/null")) + XCTAssertFalse(SimulatedTransportURLProtocol.canInit(with: request)) + } + +} + +/// Terminal stub for passthrough tests: answers every request on its session with a canned +/// 200 so no test traffic ever leaves the process. +private final class StubBackendURLProtocol: URLProtocol { + + static let stubBody = Data(#"{"stub":true}"#.utf8) + static var requestCount = 0 + + override class func canInit(with request: URLRequest) -> Bool { + return true + } + + override class func canonicalRequest(for request: URLRequest) -> URLRequest { + return request + } + + override func startLoading() { + Self.requestCount += 1 + guard let url = self.request.url, + let response = HTTPURLResponse( + url: url, + statusCode: 200, + httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + ) else { + self.client?.urlProtocol(self, didFailWithError: URLError(.badURL)) + return + } + self.client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + self.client?.urlProtocol(self, didLoad: Self.stubBody) + self.client?.urlProtocolDidFinishLoading(self) + } + + override func stopLoading() {} + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift b/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift new file mode 100644 index 0000000000..4c7e70e84e --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/Tests/RCContainerEncoderTests.swift @@ -0,0 +1,69 @@ +import XCTest + +@testable import SDKConfigBenchmarkCore + +final class RCContainerEncoderTests: BenchmarkTestCase { + + private let config = Data(#"{"domain":"app","manifest":"m1"}"#.utf8) + private let blobA = Data(#"{"id":"blob-a"}"#.utf8) + private let blobB = Data(String(repeating: "x", count: 300).utf8) + + func testEncodedContainerRoundTripsThroughSDKParser() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: [self.blobA, self.blobB]) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements.count, 3) + XCTAssertEqual(try parsed.elements[0].withDecodedPayloadBytes { Data($0) }, self.config) + XCTAssertEqual(try parsed.elements[1].withDecodedPayloadBytes { Data($0) }, self.blobA) + XCTAssertEqual(try parsed.elements[2].withDecodedPayloadBytes { Data($0) }, self.blobB) + } + + func testEncodedElementChecksumsMatchBlobRefs() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: [self.blobA]) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements[1].checksum, RCContainerEncoder.blobRef(for: self.blobA)) + XCTAssertNotNil(parsed.elementsByChecksum[RCContainerEncoder.blobRef(for: self.blobA)]) + } + + func testBlobRefShape() { + let ref = RCContainerEncoder.blobRef(for: self.blobA) + + XCTAssertEqual(ref.count, 32) + XCTAssertNil(ref.rangeOfCharacter(from: CharacterSet(charactersIn: "+/="))) + } + + func testConfigOnlyContainerParses() throws { + let encoded = RCContainerEncoder.container(config: self.config, contentElements: []) + + let parsed = try RCContainer(data: encoded) + + XCTAssertEqual(parsed.elements.count, 1) + } + + func testSeededRandomIsDeterministic() { + var first = SeededRandom(seed: 7) + var second = SeededRandom(seed: 7) + var other = SeededRandom(seed: 8) + + let firstValues = (0..<8).map { _ in first.next() } + let secondValues = (0..<8).map { _ in second.next() } + let otherValues = (0..<8).map { _ in other.next() } + + XCTAssertEqual(firstValues, secondValues) + XCTAssertNotEqual(firstValues, otherValues) + } + + func testSeededRandomUniformDoubleStaysInRange() { + var rng = SeededRandom(seed: 1) + + for _ in 0..<1_000 { + let value = Double.random(in: 0..<1, using: &rng) + XCTAssertGreaterThanOrEqual(value, 0) + XCTAssertLessThan(value, 1) + } + } + +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/compare.py b/Tests/Benchmarks/SDKConfigBenchmark/compare.py new file mode 100644 index 0000000000..c26eaa68a5 --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/compare.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Render SDK config benchmark JSONL results as a markdown table. + +Usage: + python3 compare.py results.jsonl # one file: summary table + python3 compare.py baseline.jsonl candidate.jsonl # two files: side-by-side with deltas + +Rows are grouped by (mode, scenario, profile, loss_percent). When comparing, groups present +in only one file are shown with the other side empty. +""" + +import json +import sys + + +KEY_FIELDS = ("mode", "transport", "scenario", "profile", "loss_percent", + "paywalls", "workflows", "seed", "iterations", "warmup_discarded", "project_id") +METRICS = ("p50_ms", "p95_ms", "request_count_mean", "bytes_received_mean") + + +def load(path): + rows = {} + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + row = json.loads(line) + key = tuple(row.get(field) for field in KEY_FIELDS) + if key in rows: + raise SystemExit(f"{path}: duplicate row for {dict(zip(KEY_FIELDS, key))}") + rows[key] = row + return rows + + +def fmt(value): + if value is None: + return "-" + if isinstance(value, float): + return f"{value:,.1f}" + return f"{value:,}" + + +def delta(base, cand): + if base in (None, 0) or cand is None: + return "-" + change = (cand - base) / base * 100 + return f"{change:+.0f}%" + + +def print_single(rows): + header = list(KEY_FIELDS) + list(METRICS) + ["errors"] + print("| " + " | ".join(header) + " |") + print("|" + "---|" * len(header)) + invalid = 0 + for key in sorted(rows, key=str): + row = rows[key] + cells = [str(part) for part in key] + cells += [fmt(row.get(metric)) for metric in METRICS] + cells.append(errors_cell(row)) + if post_warmup_errors(row): + invalid += 1 + print("| " + " | ".join(cells) + " |") + return invalid + + +def post_warmup_errors(row): + return row.get("post_warmup_error_count", row.get("error_count")) + + +def errors_cell(row): + """Rows with post-warmup errors are not valid comparison input; make that loud.""" + errors = post_warmup_errors(row) + if errors is None: + return "-" + return f"⚠️ {errors}" if errors else "0" + + +def print_comparison(baseline, candidate, allow_missing=False): + header = list(KEY_FIELDS) + for metric in ("p50_ms", "p95_ms"): + header += [f"{metric} base", f"{metric} cand", "Δ"] + header += ["req base", "req cand", "bytes base", "bytes cand", "err base", "err cand"] + print("| " + " | ".join(header) + " |") + print("|" + "---|" * len(header)) + + invalid = 0 + missing = sorted(set(baseline) ^ set(candidate), key=str) + for key in sorted(set(baseline) | set(candidate), key=str): + base, cand = baseline.get(key, {}), candidate.get(key, {}) + cells = [str(part) for part in key] + for metric in ("p50_ms", "p95_ms"): + cells += [ + fmt(base.get(metric)), + fmt(cand.get(metric)), + delta(base.get(metric), cand.get(metric)), + ] + cells += [ + fmt(base.get("request_count_mean")), + fmt(cand.get("request_count_mean")), + fmt(base.get("bytes_received_mean")), + fmt(cand.get("bytes_received_mean")), + errors_cell(base), + errors_cell(cand), + ] + if any(post_warmup_errors(row) for row in (base, cand) if row): + invalid += 1 + print("| " + " | ".join(cells) + " |") + + if invalid: + print( + f"\n**⚠️ {invalid} row(s) have post-warmup errors; " + "their timing deltas are not valid comparison input.**" + ) + if missing and not allow_missing: + print(f"\n**⚠️ {len(missing)} configuration(s) present in only one file:**") + for key in missing: + side = "candidate" if key in candidate else "baseline" + print(f"- only in {side}: {dict(zip(KEY_FIELDS, key))}") + invalid += len(missing) + return invalid + + +def main(argv): + allow_missing = "--allow-missing" in argv + argv = [arg for arg in argv if arg != "--allow-missing"] + if len(argv) == 2: + invalid = print_single(load(argv[1])) + elif len(argv) == 3: + invalid = print_comparison(load(argv[1]), load(argv[2]), allow_missing=allow_missing) + else: + print(__doc__, file=sys.stderr) + return 2 + # Invalid or incomplete rows must fail automation, not just print a warning. + return 1 if invalid else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh new file mode 100644 index 0000000000..f512a6589e --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Shared API-key resolution for both benchmark tiers (run-matrix.sh and run-app-launch.sh), +# so live runs of the two tiers always resolve the same key for the same project and stay +# comparable. No keys live in source. +# +# Usage (from a script with `set -euo pipefail`): +# source ".../resolve-api-key.sh" +# KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" +# +# Resolution order: the SDK_CONFIG_BENCHMARK_API_KEY environment variable, else mafdet +# (preferring the project's test-store app, whose products actually exist). Exits nonzero +# with a message on stderr when no key can be resolved. + +# Resolves the project id BEFORE any default applies. A custom key via +# SDK_CONFIG_BENCHMARK_API_KEY requires an explicit PROJECT_ID: otherwise rows would carry +# the pinned default project label with another project's key, and live rows from different +# projects would compare as equivalents. +default_benchmark_project_id() { + if [[ -z "${PROJECT_ID:-}" ]]; then + if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then + echo "SDK_CONFIG_BENCHMARK_API_KEY requires an explicit PROJECT_ID so rows are labeled with the key's real project" >&2 + return 1 + fi + printf '5f07e7e3' + else + printf '%s' "$PROJECT_ID" + fi +} + +resolve_benchmark_api_key() { + local project_id="$1" + local key + + if [[ -n "${SDK_CONFIG_BENCHMARK_API_KEY:-}" ]]; then + key="$SDK_CONFIG_BENCHMARK_API_KEY" + else + if ! command -v mafdet >/dev/null; then + echo "set SDK_CONFIG_BENCHMARK_API_KEY or install the mafdet CLI to resolve the project key" >&2 + return 1 + fi + key="$(mafdet app api-keys --project-id "$project_id" 2>/dev/null | python3 -c ' +import json, sys +keys = json.load(sys.stdin) +keys.sort(key=lambda entry: entry.get("app_store_type") != "test_store") +print(keys[0]["key"] if keys else "") +')" + fi + + if [[ -z "$key" ]]; then + echo "Could not resolve an API key for project $project_id" >&2 + return 1 + fi + + printf '%s' "$key" +} diff --git a/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh new file mode 100644 index 0000000000..25e833b78c --- /dev/null +++ b/Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# Runs the SDK config benchmark matrix and emits one JSONL row per configuration to stdout. +# Build logs and progress go to stderr, so redirecting stdout captures clean JSONL: +# +# bash Tests/Benchmarks/SDKConfigBenchmark/run-matrix.sh > baseline.jsonl +# +# Requires the Tuist workspace: tuist install && tuist generate SDKConfigBenchmark +# +# Override the matrix through environment variables: +# TRANSPORT=live # hit the real backend (pinned stress-test project) instead of +# # the simulated transport; forces ideal profile, no loss, and +# # drops the kill-switch mode (cannot force 4xx on production) +# PROJECT_ID= # live only: measure a different RevenueCat project; the key is +# # resolved via mafdet (test-store app preferred) and rows are +# # labeled so cross-project comparisons never mix +# MODES="legacy config config-killswitch" +# SCENARIOS="cold warm" +# PROFILES="ideal lte" +# LOSSES="0" # extra loss sweep applies to config/cold/lte below +# LOSS_SWEEP="10 20 30" +# ITERATIONS=25 WARMUP=3 PAYWALLS=50 WORKFLOWS=100 SEED=42 +# SKIP_BUILD=1 # reuse the previously built binary + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DERIVED_DATA="${DERIVED_DATA:-$REPO_ROOT/.build/sdk-config-benchmark-derived-data}" +BINARY="$DERIVED_DATA/Build/Products/Release/SDKConfigBenchmark" + +TRANSPORT="${TRANSPORT:-simulated}" +if [[ "$TRANSPORT" == "live" ]]; then + MODES="${MODES:-legacy config}" + PROFILES="ideal" + LOSSES="0" + LOSS_SWEEP="" +else + MODES="${MODES:-legacy config config-killswitch}" + PROFILES="${PROFILES:-ideal lte}" + LOSSES="${LOSSES:-0}" + LOSS_SWEEP="${LOSS_SWEEP:-10 20 30}" +fi +SCENARIOS="${SCENARIOS:-cold warm}" +ITERATIONS="${ITERATIONS:-25}" +WARMUP="${WARMUP:-3}" +PAYWALLS="${PAYWALLS:-50}" +WORKFLOWS="${WORKFLOWS:-100}" +SEED="${SEED:-42}" + +if [[ "${SKIP_BUILD:-0}" != "1" ]]; then + echo "Building SDKConfigBenchmark (Release)..." >&2 + xcodebuild -workspace "$REPO_ROOT/RevenueCat-Tuist.xcworkspace" \ + -scheme SDKConfigBenchmark \ + -configuration Release \ + -destination platform=macOS \ + -derivedDataPath "$DERIVED_DATA" \ + build >&2 +fi + +if [[ ! -x "$BINARY" ]]; then + echo "Benchmark binary not found at $BINARY" >&2 + exit 1 +fi + +SDK_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +FAILED_ROWS=0 + +PROJECT_ARGS=() +if [[ "$TRANSPORT" == "live" ]]; then + # No keys live in source: resolve the target project's key at run time (shared with + # the app-launch tier so both tiers always measure the same key for the same project; + # an env-key override requires an explicit PROJECT_ID so rows are labeled correctly). + # shellcheck source=resolve-api-key.sh disable=SC1091 + source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" + PROJECT_ID="$(default_benchmark_project_id)" + RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" + echo "Live target: project $PROJECT_ID" >&2 + PROJECT_ARGS=(--api-key "$RESOLVED_KEY" --project-id "$PROJECT_ID") +fi + +run_row() { + local mode="$1" scenario="$2" profile="$3" loss="$4" + echo "Running transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%..." >&2 + if ! "$BINARY" \ + --transport "$TRANSPORT" \ + --mode "$mode" \ + --scenario "$scenario" \ + --profile "$profile" \ + --loss-percent "$loss" \ + --iterations "$ITERATIONS" \ + --warmup-iterations "$WARMUP" \ + --paywalls "$PAYWALLS" \ + --workflows "$WORKFLOWS" \ + --seed "$SEED" \ + --annotation "sdk_commit=$SDK_COMMIT" \ + ${PROJECT_ARGS[@]+"${PROJECT_ARGS[@]}"}; then + echo "Row FAILED (transport=$TRANSPORT mode=$mode scenario=$scenario profile=$profile loss=$loss%)" >&2 + FAILED_ROWS=$((FAILED_ROWS + 1)) + fi +} + +for mode in $MODES; do + for scenario in $SCENARIOS; do + for profile in $PROFILES; do + for loss in $LOSSES; do + run_row "$mode" "$scenario" "$profile" "$loss" + done + done + done +done + +# Loss sweep: degraded LTE for the two systems' cold starts. Warm scenarios are skipped here +# because loss-induced request failures make the 304/204 verification nondeterministic. +if [[ -n "$LOSS_SWEEP" ]]; then + for loss in $LOSS_SWEEP; do + for mode in legacy config; do + run_row "$mode" cold lte "$loss" + done + done +fi + +if (( FAILED_ROWS > 0 )); then + echo "$FAILED_ROWS row(s) failed or produced invalid timings" >&2 + exit 1 +fi diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift new file mode 100644 index 0000000000..98946bd7fd --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/BenchmarkAppMain.swift @@ -0,0 +1,257 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// BenchmarkAppMain.swift +// +// Created by Facundo Menzella on 9/7/26. + +@_spi(Internal) import RevenueCat +import RevenueCatUI +import SwiftUI + +/// Measures one real SDK launch end to end: configure, first customer info, offerings, +/// paywall appeared. The XCUITest runner relaunches this app once per iteration (a true +/// process cold start), injecting the API key and user via the launch environment: +/// +/// - `BENCH_API_KEY`: RevenueCat public API key (required; never committed to source) +/// - `BENCH_APP_USER_ID`: app user ID for this launch +/// - `--wipe-state` argument: delete all SDK disk state before configuring (cold launch) +@main +struct SDKConfigBenchmarkApp: App { + + private let clock: LaunchClock + @StateObject private var model: LaunchModel + + init() { + // Wipe before anchoring the clock: harness cleanup I/O must never count as SDK + // launch time (it would inflate every cold phase by the previous launch's litter). + if CommandLine.arguments.contains("--wipe-state") { + Self.wipeSDKState() + } + + let clock = LaunchClock() + self.clock = clock + + let environment = ProcessInfo.processInfo.environment + let model = LaunchModel(clock: clock) + self._model = StateObject(wrappedValue: model) + + guard let apiKey = environment["BENCH_API_KEY"], !apiKey.isEmpty else { + model.fail("BENCH_API_KEY missing from launch environment") + return + } + + // The config path (config persisted, each blob stored inline vs downloaded) is only + // observable through the SDK's log stream, so run verbose and parse. The observer + // sits inside the measured window and is NOT perfectly even across variants (the + // config path emits more log lines), but its per-line cost is a few substring scans + // (microseconds), orders of magnitude below the hundreds-of-ms deltas measured. + let observer = model.blobObserver + Purchases.logLevel = .verbose + Purchases.verboseLogHandler = { _, message, _, _, _ in + observer.ingest(message) + } + Purchases.configure( + with: .builder(withAPIKey: apiKey) + .with(appUserID: environment["BENCH_APP_USER_ID"]) + .build() + ) + model.recordConfigured() + // The impression event fires when the paywall CONTENT view appears (resolved paywall + // or explicit fallback), unlike the wrapper's onAppear, which can precede it while a + // loading state shows. It is the headline "user sees the paywall" mark. + Purchases.shared.eventsListener = model.impressionListener + model.startObserving() + } + + var body: some Scene { + WindowGroup { + LaunchView(model: self.model) + } + } + + /// Deletes every place the SDK persists state so the next configure is a true cold start. + private static func wipeSDKState() { + let fileManager = FileManager.default + for directory in [FileManager.SearchPathDirectory.cachesDirectory, .applicationSupportDirectory] { + guard let root = fileManager.urls(for: directory, in: .userDomainMask).first, + let contents = try? fileManager.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + else { continue } + for url in contents { + try? fileManager.removeItem(at: url) + } + } + + if let bundleID = Bundle.main.bundleIdentifier { + UserDefaults.standard.removePersistentDomain(forName: bundleID) + } + let sdkSuite = SDKObservedValues.revenueCatUserDefaultsSuiteName + UserDefaults(suiteName: sdkSuite)?.removePersistentDomain(forName: sdkSuite) + } + +} + +/// Drives the measured phases after configure and publishes the finished sample. +@MainActor +final class LaunchModel: ObservableObject { + + @Published private(set) var offering: Offering? + @Published private(set) var finishedSampleJSON: String? + + let blobObserver: BlobObserver + /// Retained here: the SDK does not keep the events listener alive. + private(set) var impressionListener: PaywallImpressionListener! + + private let clock: LaunchClock + private var sample = LaunchSample() + private var paywallAppeared = false + + init(clock: LaunchClock) { + self.clock = clock + self.blobObserver = BlobObserver(clock: clock) + self.impressionListener = PaywallImpressionListener(clock: clock) { [weak self] elapsedMs in + Task { @MainActor in + self?.recordPaywallImpression(elapsedMs: elapsedMs) + } + } + } + + nonisolated func fail(_ message: String) { + Task { @MainActor in + self.sample.error = message + self.finish() + } + } + + nonisolated func recordConfigured() { + let elapsed = self.clock.elapsedMs() + Task { @MainActor in + self.sample.configuredMs = elapsed + } + } + + nonisolated func startObserving() { + Task { @MainActor in + await self.observeLaunch() + } + } + + private func observeLaunch() async { + async let customerInfo: Void = self.awaitFirstCustomerInfo() + + do { + let offerings = try await Purchases.shared.offerings() + self.sample.offeringsMs = self.clock.elapsedMs() + guard let current = offerings.current else { + throw NSError( + domain: "SDKConfigBenchmarkApp", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "project has no current offering"] + ) + } + self.offering = current + } catch { + self.sample.error = String(describing: error) + self.finish() + } + + await customerInfo + self.finishIfComplete() + } + + private func awaitFirstCustomerInfo() async { + for await _ in Purchases.shared.customerInfoStream { + self.sample.customerInfoMs = self.clock.elapsedMs() + break + } + } + + func recordPaywallAppeared() { + guard !self.paywallAppeared else { return } + self.paywallAppeared = true + self.sample.paywallAppearedMs = self.clock.elapsedMs() + self.finishIfComplete() + } + + private func recordPaywallImpression(elapsedMs: Double) { + guard self.sample.paywallImpressionMs == nil else { return } + self.sample.paywallImpressionMs = elapsedMs + self.finishIfComplete() + } + + private func finishIfComplete() { + guard self.finishedSampleJSON == nil else { return } + let done = self.sample.customerInfoMs != nil + && self.sample.offeringsMs != nil + && self.sample.paywallAppearedMs != nil + && self.sample.paywallImpressionMs != nil + if done { + self.finish() + } + } + + private func finish() { + guard self.finishedSampleJSON == nil else { return } + self.blobObserver.apply(to: &self.sample) + let json = self.sample.jsonString() + self.finishedSampleJSON = json + // Also visible in the unified log for smoke testing without the XCUITest runner. + NSLog("BENCH_SAMPLE %@", json) + } + +} + +/// Stamps the moment the SDK tracks that paywall content actually appeared (via the SDK's +/// internal events listener SPI), unlike the wrapper's `onAppear`, which can fire while a +/// loading state still shows. A classic paywall tracks `paywall_impression`; a workflow +/// paywall tracks `workflows_step_started` when its first step renders. +final class PaywallImpressionListener: EventsListener { + + private let clock: LaunchClock + private let onImpression: (Double) -> Void + + init(clock: LaunchClock, onImpression: @escaping (Double) -> Void) { + self.clock = clock + self.onImpression = onImpression + } + + func onEventTracked(_ event: [String: Any]) { + guard let type = event["type"] as? String, + SDKObservedValues.contentAppearedEventTypes.contains(type) else { return } + self.onImpression(self.clock.elapsedMs()) + } + +} + +struct LaunchView: View { + + @ObservedObject var model: LaunchModel + + var body: some View { + ZStack { + if let offering = self.model.offering { + PaywallView(offering: offering) + .onAppear { + self.model.recordPaywallAppeared() + } + } else { + ProgressView() + } + + if let json = self.model.finishedSampleJSON { + // The runner reads the sample from this element's label. + Text(json) + .font(.system(size: 4)) + .opacity(0.02) + .accessibilityIdentifier("benchmark-result") + } + } + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist b/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist new file mode 100644 index 0000000000..9736be2280 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/Info.plist @@ -0,0 +1,24 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + UILaunchScreen + + + diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift new file mode 100644 index 0000000000..acc507036a --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/App/LaunchMeasurement.swift @@ -0,0 +1,259 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// LaunchMeasurement.swift +// +// Created by Facundo Menzella on 9/7/26. + +import Foundation + +/// One app launch's phase timings, measured from process start (`LaunchClock` creation in +/// the `App` initializer). Encoded as JSON into the `benchmark-result` accessibility element +/// so the XCUITest runner can collect it; also compiled into the benchmark unit-test target. +struct LaunchSample: Codable, Equatable { + + /// Milliseconds until `Purchases.configure` returned. + var configuredMs: Double? + /// Milliseconds until the first `CustomerInfo` was delivered. + var customerInfoMs: Double? + /// Milliseconds until `getOfferings` completed. + var offeringsMs: Double? + /// Milliseconds until the `PaywallView` wrapper mounted (may still be a loading state). + var paywallAppearedMs: Double? + /// Milliseconds until the SDK tracked the paywall impression: the content view (resolved + /// paywall or explicit fallback) actually appeared. This is the headline number. + var paywallImpressionMs: Double? + /// Whether the config endpoint path ran this launch (its refresh was observed in the log). + /// Runtime proof of which SDK variant is actually inside the binary. + var configPathActive: Bool = false + /// Terminal state of the config refresh: `persisted`, `not_modified`, or `failed`. + /// A config-variant launch whose refresh failed silently measures legacy fallback + /// behavior, so the runner requires a success outcome, not just an active path. + var configOutcome: String? + /// Milliseconds until the config endpoint response was persisted (config variant only; + /// observed via the SDK's log stream, nil when the config path never ran). + var configPersistedMs: Double? + /// Milliseconds until the last blob landed in the store (nil when no blobs were stored). + var lastBlobStoredMs: Double? + /// Blobs written to the store this launch, split by how they arrived. + var blobsInline: Int = 0 + var blobsDownloaded: Int = 0 + /// Total bytes of blobs stored this launch. + var blobBytes: Int = 0 + /// Size extremes, for empirically checking the backend's inline-size budget (blobs under + /// the budget should arrive inline; a small blob arriving via CDN is a regression signal). + var maxInlineBlobBytes: Int? + var minDownloadedBlobBytes: Int? + /// Non-nil when the launch could not complete; the runner fails loudly on it. + var error: String? + + func jsonString() -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let data = try? encoder.encode(self), + let json = String(data: data, encoding: .utf8) else { + return "{\"error\":\"encoding-failed\"}" + } + return json + } + + static func decode(from json: String) -> LaunchSample? { + return try? JSONDecoder().decode(LaunchSample.self, from: Data(json.utf8)) + } + +} + +/// SDK values the app copies because the SDK does not export them. Each one is pinned to +/// the real SDK by a unit test in the benchmark suite, so drift over there fails a test +/// instead of silently corrupting rows (a stale suite name would stop `--wipe-state` from +/// wiping and make "cold" rows measure warm behavior; stale event types would time out +/// every launch). +enum SDKObservedValues { + + /// Event types meaning "paywall content actually appeared": a classic paywall tracks + /// `paywall_impression`; a workflow paywall tracks `workflows_step_started`. + static let contentAppearedEventTypes: Set = [ + "paywall_impression", + "workflows_step_started" + ] + + /// The UserDefaults suite the SDK persists into (`UserDefaults.revenueCatSuiteName`, + /// which is private). + static let revenueCatUserDefaultsSuiteName = "com.revenuecat.user_defaults" + +} + +/// One blob landing in the SDK's content-addressed store, observed from the SDK's log stream. +struct BlobStorageEvent: Equatable { + + enum Source: String { + /// Delivered inside the config container response. + case inline + /// Fetched separately from a blob source (CDN). + case downloaded + } + + let source: Source + let byteCount: Int + +} + +/// Extracts config-path progress from the SDK's log messages (the only observable the stock +/// SDK offers an app). Substring-based so logger prefixes (level, emoji) don't matter; the +/// exact phrases are pinned to `RemoteConfigStrings` by unit tests in the benchmark suite. +enum BlobLogParser { + + private static let inlineMarker = "Stored inline remote config blob '" + private static let downloadedMarker = "Stored remote config blob '" + private static let configPersistedMarker = "Persisted remote config for domain '" + private static let configRefreshingMarker = "Refreshing remote config for domain '" + private static let configNotModifiedMarker = "Remote config was not modified" + private static let configRefreshFailedMarker = "Remote config refresh failed" + + static func blobEvent(from message: String) -> BlobStorageEvent? { + let source: BlobStorageEvent.Source + if message.contains(Self.inlineMarker) { + source = .inline + } else if message.contains(Self.downloadedMarker) { + source = .downloaded + } else { + return nil + } + + guard let byteCount = Self.integer(before: " bytes", in: message) else { return nil } + return BlobStorageEvent(source: source, byteCount: byteCount) + } + + static func isConfigPersisted(_ message: String) -> Bool { + return message.contains(Self.configPersistedMarker) + } + + /// Fires on every config-variant launch (cold 200 and warm 204 alike), so it doubles as + /// runtime proof that `ENABLE_REMOTE_CONFIG` is compiled into the SDK being measured. + static func isConfigRefreshStarted(_ message: String) -> Bool { + return message.contains(Self.configRefreshingMarker) + } + + /// The refresh revalidated via a manifest 204 (warm path's terminal success state). + static func isConfigNotModified(_ message: String) -> Bool { + return message.contains(Self.configNotModifiedMarker) + } + + /// The refresh failed and the SDK fell back to cached/legacy behavior. A launch ending + /// here must not pass as a config-path measurement. + static func isConfigRefreshFailed(_ message: String) -> Bool { + return message.contains(Self.configRefreshFailedMarker) + } + + private static func integer(before suffix: String, in message: String) -> Int? { + guard let suffixRange = message.range(of: suffix) else { return nil } + let digits = message[.. Double { + let nanos = DispatchTime.now().uptimeNanoseconds - self.start.uptimeNanoseconds + return Double(nanos) / 1_000_000 + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift new file mode 100644 index 0000000000..0f98c38ef3 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchBenchmarkUITests.swift @@ -0,0 +1,239 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// AppLaunchBenchmarkUITests.swift +// +// Created by Facundo Menzella on 9/7/26. + +import XCTest + +/// Relaunches the benchmark app once per iteration (each launch is a real process cold +/// start), reads the `LaunchSample` the app exposes, and aggregates one JSONL row per +/// scenario, printed as `BENCHMARK_ROW: {...}` for `run-app-launch.sh` to collect. +/// +/// Configured via the runner's environment (pass with a `TEST_RUNNER_` prefix through +/// xcodebuild): `BENCH_API_KEY` (required), `BENCH_MODE_LABEL` (required; identifies the +/// SDK build variant, e.g. `app-launch-legacy`), `BENCH_ITERATIONS`, `BENCH_WARMUP`, +/// `BENCH_PROJECT_ID`. Both tests skip when the required variables are absent, so running +/// the scheme without the script does not produce mislabeled rows. +final class AppLaunchBenchmarkUITests: XCTestCase { + + private struct Configuration { + let apiKey: String + let modeLabel: String + let iterations: Int + let warmup: Int + let projectID: String + let runNonce: String + /// When set (via `BENCH_EXPECT_CONFIG_PATH` = "1"/"0"), every measured launch must + /// report the matching `configPathActive`: runtime proof, from inside the launched + /// binary, that the SDK build variant matches the row's label even on cached builds. + let expectsConfigPath: Bool? + } + + override func setUpWithError() throws { + try super.setUpWithError() + self.continueAfterFailure = false + } + + func testColdLaunches() throws { + let configuration = try self.configuration() + + // A fresh user and wiped state every launch: each iteration is a true cold start. + var retries = 0 + let samples = (0.. Configuration { + let environment = ProcessInfo.processInfo.environment + guard let apiKey = environment["BENCH_API_KEY"], !apiKey.isEmpty else { + throw XCTSkip("BENCH_API_KEY not set; run via run-app-launch.sh") + } + guard let modeLabel = environment["BENCH_MODE_LABEL"], !modeLabel.isEmpty else { + throw XCTSkip("BENCH_MODE_LABEL not set; the row cannot name its SDK build variant") + } + + let iterations = environment["BENCH_ITERATIONS"].flatMap(Int.init) ?? 10 + let warmup = environment["BENCH_WARMUP"].flatMap(Int.init) ?? 2 + guard iterations > 0, warmup >= 0, warmup < iterations else { + throw XCTSkip("BENCH_WARMUP (\(warmup)) must be below BENCH_ITERATIONS (\(iterations))") + } + + return Configuration( + apiKey: apiKey, + modeLabel: modeLabel, + iterations: iterations, + warmup: warmup, + projectID: environment["BENCH_PROJECT_ID"] ?? "5f07e7e3", + runNonce: UUID().uuidString.lowercased().prefix(8).description, + expectsConfigPath: environment["BENCH_EXPECT_CONFIG_PATH"].map { $0 == "1" } + ) + } + + /// One launch, retried once when it produced nothing at all (element never appeared: + /// a simulator hiccup or a >90s stall). At hundreds of iterations a ~1% harness flake + /// is expected; a retry is a fresh, unbiased sample, and the retry COUNT lands in the + /// row so censored launches stay visible. A sample that reports an error is NOT + /// retried: that is a real measured failure. + private func launchCollectingWithRetry( + configuration: Configuration, + appUserID: String, + wipeState: Bool, + retries: inout Int + ) -> LaunchSample? { + if let sample = self.launchAndCollect( + configuration: configuration, appUserID: appUserID, wipeState: wipeState + ) { + return sample + } + retries += 1 + return self.launchAndCollect( + configuration: configuration, appUserID: appUserID, wipeState: wipeState + ) + } + + private func launchAndCollect( + configuration: Configuration, + appUserID: String, + wipeState: Bool + ) -> LaunchSample? { + let app = XCUIApplication() + app.launchEnvironment = [ + "BENCH_API_KEY": configuration.apiKey, + "BENCH_APP_USER_ID": appUserID + ] + app.launchArguments = wipeState ? ["--wipe-state"] : [] + app.launch() + defer { app.terminate() } + + let result = app.staticTexts["benchmark-result"] + guard result.waitForExistence(timeout: 90) else { + return nil + } + return LaunchSample.decode(from: result.label) + } + + private func report( + samples: [LaunchSample?], + scenario: String, + configuration: Configuration, + retries: Int + ) { + // A retried launch is a fresh sample, but the retry budget stays small: pervasive + // retries mean the environment is unstable and the whole round is suspect. + let retryBudget = max(1, configuration.iterations / 50) + XCTAssertLessThanOrEqual( + retries, retryBudget, + "\(retries) launches produced nothing and were retried (budget \(retryBudget)); environment unstable" + ) + + let row = AppLaunchMetrics.row( + mode: configuration.modeLabel, + scenario: scenario, + profile: Self.profile, + projectID: configuration.projectID, + warmupDiscarded: configuration.warmup, + samples: samples, + launchRetries: retries + ) + + // The script greps this prefix out of the xcodebuild log to build the JSONL file. + print("BENCHMARK_ROW: \(row)") + let attachment = XCTAttachment(string: row) + attachment.name = "benchmark-row-\(scenario)" + attachment.lifetime = .keepAlways + self.add(attachment) + + // One definition of "measured sample" (post-warmup, by index) for every assertion, + // matching the window the row's statistics were aggregated from. + let measured = samples.enumerated().filter { $0.offset >= configuration.warmup } + + let postWarmupErrors = measured.compactMap { AppLaunchMetrics.errorMessage(for: $0.element) } + XCTAssertTrue( + postWarmupErrors.isEmpty, + "\(postWarmupErrors.count) measured launch(es) failed; first: \(postWarmupErrors.first ?? "")" + ) + + // Runtime variant proof: the launched binary must have actually run (or not run) + // the config path; a mislabeled row would poison every later comparison. + if let expectsConfigPath = configuration.expectsConfigPath { + let mismatches = measured + .filter { ($0.element?.configPathActive ?? false) != expectsConfigPath } + XCTAssertTrue( + mismatches.isEmpty, + "\(mismatches.count) launch(es) contradict the \(configuration.modeLabel) label " + + "(expected configPathActive == \(expectsConfigPath)); wrong SDK variant in the binary?" + ) + + // An active config path is not enough: a failed refresh silently falls back to + // legacy delivery, so the row would measure the wrong system with clean timings. + // Cold launches must persist a fresh config; warm launches must revalidate via + // the manifest 204 (a fresh persist on warm means the caches were not warm), + // matching the CLI tier's warm validation. + if expectsConfigPath { + let acceptedOutcomes = scenario == "cold" ? ["persisted"] : ["not_modified"] + let badOutcomes = measured + .map { ($0.offset, $0.element?.configOutcome ?? "none") } + .filter { !acceptedOutcomes.contains($0.1) } + XCTAssertTrue( + badOutcomes.isEmpty, + "\(badOutcomes.count) config-variant launch(es) lack a successful config outcome " + + "(accepted: \(acceptedOutcomes)); first: iteration \(badOutcomes.first?.0 ?? -1) " + + "= \(badOutcomes.first?.1 ?? "")" + ) + } + } + } + + private static var profile: String { + #if targetEnvironment(simulator) + return "simulator" + #else + return "device" + #endif + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift new file mode 100644 index 0000000000..a17f5606f5 --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/UITests/AppLaunchMetrics.swift @@ -0,0 +1,150 @@ +// +// Copyright RevenueCat Inc. All Rights Reserved. +// +// Licensed under the MIT License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT +// +// AppLaunchMetrics.swift +// +// Created by Facundo Menzella on 9/7/26. + +import Foundation + +/// Aggregates per-launch `LaunchSample`s into one JSONL row compatible with the CLI +/// benchmark's `compare.py` (same key fields, same nearest-rank percentiles). Compiled into +/// both the XCUITest runner and the benchmark unit-test target. +enum AppLaunchMetrics { + + /// One row per (mode, scenario) configuration. `samples` has one entry per launch in + /// order; `nil` means the launch produced no readable result. Launches with an `error` + /// or a missing phase count as errors and never enter the statistics. The first + /// `warmupDiscarded` entries are excluded from statistics by index. + // swiftlint:disable:next function_body_length + static func row( + mode: String, + scenario: String, + profile: String, + projectID: String, + warmupDiscarded: Int, + samples: [LaunchSample?], + launchRetries: Int = 0 + ) -> String { + let indexed = samples.enumerated() + let errors: [(index: Int, message: String)] = indexed.compactMap { index, sample in + if let message = Self.errorMessage(for: sample) { + return (index, message) + } + return nil + } + let measured: [LaunchSample] = indexed.compactMap { index, sample in + guard index >= warmupDiscarded, + let sample, + Self.errorMessage(for: sample) == nil else { + return nil + } + return sample + } + + var row: [String: Any] = [ + "mode": mode, + "transport": "live", + "scenario": scenario, + "profile": profile, + "loss_percent": 0, + "paywalls": 0, + "workflows": 0, + "seed": 0, + "iterations": samples.count, + "warmup_discarded": warmupDiscarded, + "project_id": projectID, + "measured_iterations": measured.count, + "error_count": errors.count, + "post_warmup_error_count": errors.filter { $0.index >= warmupDiscarded }.count, + // Launches that produced nothing and were relaunched once (censored samples); + // visible so a round's stability can be judged, bounded by the runner. + "launch_retries": launchRetries + ] + + // Headline statistic: `Purchases.configure` + `getOfferings` completed, with or + // without workflows compiled in. This matches the CLI tier's total (offerings + // delivered), so the two tiers answer the same question; the paywall marks stay in + // the row as secondary phase means. + let totals = measured.compactMap(\.offeringsMs).sorted() + if !totals.isEmpty { + row["mean_ms"] = Self.rounded(totals.reduce(0, +) / Double(totals.count)) + row["min_ms"] = Self.rounded(totals[0]) + row["max_ms"] = Self.rounded(totals[totals.count - 1]) + for percentile in [50, 90, 95, 99] { + row["p\(percentile)_ms"] = Self.rounded(Self.percentile(percentile, of: totals)) + } + } + + for (key, values) in [ + ("configured_ms_mean", measured.compactMap(\.configuredMs)), + ("customer_info_ms_mean", measured.compactMap(\.customerInfoMs)), + ("offerings_ms_mean", measured.compactMap(\.offeringsMs)), + ("paywall_appeared_ms_mean", measured.compactMap(\.paywallAppearedMs)), + ("paywall_impression_ms_mean", measured.compactMap(\.paywallImpressionMs)), + ("config_persisted_ms_mean", measured.compactMap(\.configPersistedMs)), + ("last_blob_stored_ms_mean", measured.compactMap(\.lastBlobStoredMs)), + ("blobs_inline_mean", measured.map { Double($0.blobsInline) }), + ("blobs_downloaded_mean", measured.map { Double($0.blobsDownloaded) }), + ("blob_bytes_mean", measured.map { Double($0.blobBytes) }) + ] where !values.isEmpty { + row[key] = Self.rounded(values.reduce(0, +) / Double(values.count)) + } + + // Size extremes across the run: together they bracket the backend's inline-size + // budget (largest blob seen inline vs smallest blob that needed a CDN download). + if let maxInline = measured.compactMap(\.maxInlineBlobBytes).max() { + row["max_inline_blob_bytes"] = maxInline + } + if let minDownloaded = measured.compactMap(\.minDownloadedBlobBytes).min() { + row["min_downloaded_blob_bytes"] = minDownloaded + } + + if let firstError = errors.first { + row["first_error"] = "iteration \(firstError.index): \(firstError.message)" + } + + guard let data = try? JSONSerialization.data(withJSONObject: row, options: [.sortedKeys]), + let json = String(data: data, encoding: .utf8) else { + return "{\"error\":\"row-encoding-failed\"}" + } + return json + } + + /// Non-nil when a launch cannot contribute to the statistics: it failed outright, + /// produced nothing, or is missing a phase (a partially measured launch could otherwise + /// look faster than a complete one). + static func errorMessage(for sample: LaunchSample?) -> String? { + guard let sample else { return "no benchmark-result element" } + if let error = sample.error { return error } + for (phase, value) in [ + ("configuredMs", sample.configuredMs), + ("customerInfoMs", sample.customerInfoMs), + ("offeringsMs", sample.offeringsMs), + ("paywallAppearedMs", sample.paywallAppearedMs), + ("paywallImpressionMs", sample.paywallImpressionMs) + ] where value == nil { + return "sample missing \(phase)" + } + return nil + } + + /// Nearest-rank percentile over an already-sorted array; same formula as the CLI + /// benchmark's `BenchmarkMetrics.percentile` so rows are comparable. + static func percentile(_ percentile: Int, of sorted: [Double]) -> Double { + precondition(!sorted.isEmpty) + let rank = Int((Double(percentile) / 100 * Double(sorted.count)).rounded(.up)) + return sorted[max(0, min(sorted.count - 1, rank - 1))] + } + + private static func rounded(_ value: Double) -> Double { + return (value * 1_000).rounded() / 1_000 + } + +} diff --git a/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh new file mode 100755 index 0000000000..e4cbd3c78c --- /dev/null +++ b/Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# +# App-host tier of the SDK config benchmark: builds SDKConfigBenchmarkApp twice (legacy SDK +# vs config-endpoint SDK) and drives the XCUITest relaunch loop against the live stress-test +# project, emitting one JSONL row per (variant, scenario) to stdout: +# +# bash Tests/TestingApps/SDKConfigBenchmarkApp/run-app-launch.sh > app-launch.jsonl +# +# The legacy/config switch is the ENABLE_REMOTE_CONFIG compile condition, which the SPM-built +# RevenueCat reads from the SWIFT_ACTIVE_COMPILATION_CONDITIONS line of CI.xcconfig (if +# present) or Local.xcconfig (see Package.swift). This script rewrites that file per variant, +# restores it on exit, and builds each variant into its own derived data path. +# +# Environment overrides: +# VARIANTS="legacy config" # SDK build variants to measure +# ITERATIONS=10 WARMUP=2 # app relaunches per scenario / discarded from statistics +# PROJECT_ID= # target RevenueCat project; key resolved via mafdet +# # (test-store app preferred), rows labeled with the id +# SDK_CONFIG_BENCHMARK_API_KEY= # skip mafdet and use this key directly +# DESTINATION="platform=iOS Simulator,id=" # default: first available iPhone +# # simulator; pass a device destination to measure real radio + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DERIVED_DATA_ROOT="${DERIVED_DATA_ROOT:-$REPO_ROOT/.build/sdk-config-benchmark-app}" +VARIANTS="${VARIANTS:-legacy config}" +ITERATIONS="${ITERATIONS:-10}" +WARMUP="${WARMUP:-2}" + +# Key + project resolution shared with the CLI tier's run-matrix.sh (env override requires +# an explicit PROJECT_ID; else mafdet against the pinned project; no keys live in source). +# shellcheck source=../../Benchmarks/SDKConfigBenchmark/resolve-api-key.sh disable=SC1091 +source "$REPO_ROOT/Tests/Benchmarks/SDKConfigBenchmark/resolve-api-key.sh" +PROJECT_ID="$(default_benchmark_project_id)" + +# Package.swift prefers CI.xcconfig over Local.xcconfig, so the variant switch must edit +# whichever file actually wins. +if [[ -f "$REPO_ROOT/CI.xcconfig" ]]; then + XCCONFIG="$REPO_ROOT/CI.xcconfig" +else + XCCONFIG="$REPO_ROOT/Local.xcconfig" +fi + +XCCONFIG_BACKUP="" +if [[ -f "$XCCONFIG" ]]; then + XCCONFIG_BACKUP="$(mktemp)" + cp "$XCCONFIG" "$XCCONFIG_BACKUP" +fi + +VARIANT_MARKER="// sdk-config-benchmark-variant:" + +restore_xcconfig() { + if [[ -n "$XCCONFIG_BACKUP" ]]; then + cp "$XCCONFIG_BACKUP" "$XCCONFIG" + rm -f "$XCCONFIG_BACKUP" + else + rm -f "$XCCONFIG" + fi + sed -i '' "/^${VARIANT_MARKER//\//\\/}/d" "$REPO_ROOT/Package.swift" +} +trap restore_xcconfig EXIT + +RESOLVED_KEY="$(resolve_benchmark_api_key "$PROJECT_ID")" +echo "Live target: project $PROJECT_ID" >&2 + +if [[ -z "${DESTINATION:-}" ]]; then + UDID="$(xcrun simctl list devices available -j | python3 -c ' +import json, sys +devices = json.load(sys.stdin)["devices"] +iphones = [d for ds in devices.values() for d in ds if d["name"].startswith("iPhone")] +print(iphones[0]["udid"] if iphones else "") +')" + if [[ -z "$UDID" ]]; then + echo "No available iPhone simulator; pass DESTINATION explicitly" >&2 + exit 1 + fi + DESTINATION="platform=iOS Simulator,id=$UDID" +fi +echo "Destination: $DESTINATION" >&2 + +echo "Generating workspace..." >&2 +(cd "$REPO_ROOT" && tuist generate --no-open SDKConfigBenchmarkApp SDKConfigBenchmarkAppUITests >&2) + +set_swift_conditions() { + local conditions="$1" + if [[ -f "$XCCONFIG" ]] && grep -q '^SWIFT_ACTIVE_COMPILATION_CONDITIONS' "$XCCONFIG"; then + sed -i '' "s/^SWIFT_ACTIVE_COMPILATION_CONDITIONS.*$/SWIFT_ACTIVE_COMPILATION_CONDITIONS = \$(inherited) $conditions/" "$XCCONFIG" + else + # shellcheck disable=SC2016 # $(inherited) is an xcconfig token, not shell + printf '\nSWIFT_ACTIVE_COMPILATION_CONDITIONS = $(inherited) %s\n' "$conditions" >> "$XCCONFIG" + fi + # SwiftPM caches manifest evaluations by Package.swift CONTENT (mtime is ignored), and + # the xcconfig is not part of the cache key: without a content change, a reused derived + # data path silently builds the previous variant's flags. A marker comment carrying the + # current conditions (removed on exit) forces re-evaluation exactly when they change. + sed -i '' "/^${VARIANT_MARKER//\//\\/}/d" "$REPO_ROOT/Package.swift" + printf '%s %s\n' "$VARIANT_MARKER" "$conditions" >> "$REPO_ROOT/Package.swift" +} + +FAILED_VARIANTS=0 +TOTAL_ROWS=0 + +# Records a variant failure and discards its log. Rows from a failed or unverified variant +# must never reach stdout: the tests print BENCHMARK_ROW before their validity assertions, +# so a leaked row can look clean while measuring the wrong thing. Follow with `continue`. +fail_variant() { + echo "$1" >&2 + FAILED_VARIANTS=$((FAILED_VARIANTS + 1)) + rm -f "$LOG" +} + +for variant in $VARIANTS; do + # BYPASS_SIMULATED_STORE_RELEASE_CHECK: tests build Release (shipping-SDK numbers), and + # live runs use the project's Test Store key, which the SDK otherwise refuses (by + # crashing) in Release builds. This is the SDK's designed opt-out for that guard. + case "$variant" in + legacy) + CONDITIONS="BYPASS_SIMULATED_STORE_RELEASE_CHECK" + EXPECT_CONFIG_PATH=0 + ;; + config) + CONDITIONS="ENABLE_REMOTE_CONFIG BYPASS_SIMULATED_STORE_RELEASE_CHECK" + EXPECT_CONFIG_PATH=1 + ;; + *) echo "Unknown variant $variant (expected legacy or config)" >&2; exit 1 ;; + esac + + echo "=== Variant $variant (conditions: '${CONDITIONS:-none}') ===" >&2 + set_swift_conditions "$CONDITIONS" + + LOG="$(mktemp)" + DERIVED_DATA="$DERIVED_DATA_ROOT/$variant" + if ! env TEST_RUNNER_BENCH_API_KEY="$RESOLVED_KEY" \ + TEST_RUNNER_BENCH_MODE_LABEL="app-launch-$variant" \ + TEST_RUNNER_BENCH_ITERATIONS="$ITERATIONS" \ + TEST_RUNNER_BENCH_WARMUP="$WARMUP" \ + TEST_RUNNER_BENCH_PROJECT_ID="$PROJECT_ID" \ + TEST_RUNNER_BENCH_EXPECT_CONFIG_PATH="$EXPECT_CONFIG_PATH" \ + xcodebuild -workspace "$REPO_ROOT/RevenueCat-Tuist.xcworkspace" \ + -scheme SDKConfigBenchmarkApp \ + -destination "$DESTINATION" \ + -derivedDataPath "$DERIVED_DATA" \ + test > "$LOG" 2>&1; then + echo "Variant $variant FAILED; last log lines:" >&2 + tail -25 "$LOG" >&2 + fail_variant "Variant $variant: xcodebuild test failed" + continue + fi + + # Prove the variant flag reached the SDK compile (guards the manifest-cache hazard). + # Only checkable when this run actually compiled the SDK; cached builds still get + # runtime verification via BENCH_EXPECT_CONFIG_PATH (each launched binary reports + # whether the config path ran, and the tests fail on a mismatch). + if grep -q "SwiftCompile.*RevenueCat" "$LOG"; then + # The rows claim shipping-SDK numbers: refuse Debug-built frameworks. + if ! grep -q "Release-iphone" "$LOG"; then + fail_variant "Variant $variant compiled without a Release configuration; refusing its rows" + continue + fi + HAS_CONFIG_FLAG=0 + grep -q -- "-DENABLE_REMOTE_CONFIG" "$LOG" && HAS_CONFIG_FLAG=1 + if [[ "$HAS_CONFIG_FLAG" != "$EXPECT_CONFIG_PATH" ]]; then + fail_variant "Variant $variant compiled with ENABLE_REMOTE_CONFIG=$HAS_CONFIG_FLAG, expected $EXPECT_CONFIG_PATH; refusing its rows" + continue + fi + else + echo "Warning: SDK not recompiled for variant $variant (cached build); flag not re-verified" >&2 + fi + + ROWS="$(grep -o 'BENCHMARK_ROW: {.*}' "$LOG" | sed 's/^BENCHMARK_ROW: //' | sort -u || true)" + ROW_COUNT=0 + if [[ -n "$ROWS" ]]; then + ROW_COUNT="$(printf '%s\n' "$ROWS" | wc -l | tr -d ' ')" + fi + # Validate the count BEFORE emitting: a partial set means one scenario silently + # produced nothing, and callers capture stdout as the JSONL artifact. + if [[ "$ROW_COUNT" -lt 2 ]]; then + fail_variant "Variant $variant produced $ROW_COUNT row(s), expected 2 (cold + warm); refusing them" + continue + fi + printf '%s\n' "$ROWS" + TOTAL_ROWS=$((TOTAL_ROWS + ROW_COUNT)) + rm -f "$LOG" +done + +if (( FAILED_VARIANTS > 0 )); then + echo "$FAILED_VARIANTS variant check(s) failed" >&2 + exit 1 +fi +echo "Done: $TOTAL_ROWS row(s)" >&2 diff --git a/Tuist/ProjectDescriptionHelpers/Environment.swift b/Tuist/ProjectDescriptionHelpers/Environment.swift index 1dceb01bd1..2043f7b2e6 100644 --- a/Tuist/ProjectDescriptionHelpers/Environment.swift +++ b/Tuist/ProjectDescriptionHelpers/Environment.swift @@ -107,6 +107,19 @@ extension Environment { return value.isEmpty ? nil : value } + /// Returns the API key to inject into the SDKConfigBenchmarkApp scheme's run environment, + /// so the app can be launched directly from Xcode without the XCUITest runner. + /// + /// Example usage: + /// ```bash + /// TUIST_BENCH_API_KEY=$(mafdet app api-keys --project-id 5f07e7e3 | jq -r '.[] | select(.app_store_type == "test_store") | .key') \ + /// tuist generate SDKConfigBenchmarkApp + /// ``` + public static var benchApiKey: String? { + let value = ProcessInfo.processInfo.environment["TUIST_BENCH_API_KEY"] ?? "" + return value.isEmpty ? nil : value + } + /// Returns extra launch arguments to inject into scheme run actions, enabled by default. /// /// Example usage: diff --git a/Workspace.swift b/Workspace.swift index 27bd021419..4acda88bdf 100644 --- a/Workspace.swift +++ b/Workspace.swift @@ -12,7 +12,9 @@ var projects: [Path] = [ "./Projects/APITesters", "./Projects/PaywallValidationTester", "./Projects/BinarySizeTest", - "./Projects/RCTTester" + "./Projects/RCTTester", + "./Projects/SDKConfigBenchmark", + "./Projects/SDKConfigBenchmarkApp" ] // These projects depend on external packages (Nimble, SnapshotTesting, OHHTTPStubs, GoogleMobileAds).