refactor(remote-config): add API sources failover with health check endpoint support

purchases/src/androidTestDefaults/kotlin/com/revenuecat/purchases/integration/BaseIntegrationPurchasesTest.kt

@@ -515,7 +515,7 @@ abstract class BaseIntegrationPurchasesTest : BasePurchasesIntegrationTest() {
                     return "test-storefront"
                 }
             },
-            apiSourceProvider = null,
+            apiSourceFailover = null,
             localeProvider = DefaultLocaleProvider(),
         )
     }

purchases/src/main/kotlin/com/revenuecat/purchases/PurchasesFactory.kt

@@ -29,7 +29,9 @@ import com.revenuecat.purchases.common.events.BackendStoredEvent
 import com.revenuecat.purchases.common.events.EventsManager
 import com.revenuecat.purchases.common.isDeviceProtectedStorageCompat
 import com.revenuecat.purchases.common.log
+import com.revenuecat.purchases.common.networking.APISourceFailover
 import com.revenuecat.purchases.common.networking.ETagManager
+import com.revenuecat.purchases.common.networking.SourceHealthChecker
 import com.revenuecat.purchases.common.offerings.OfferingsCache
 import com.revenuecat.purchases.common.offerings.OfferingsFactory
 import com.revenuecat.purchases.common.offerings.OfferingsManager
@@ -211,14 +213,15 @@ internal class PurchasesFactory(
                 remoteConfigDiskCache?.read()?.topics?.get(it.wireName)
             }
             val apiSourceProvider = DefaultRemoteConfigSourceProvider(remoteConfigTopicStore)
+            val apiSourceFailover = APISourceFailover(appConfig, apiSourceProvider, SourceHealthChecker())
 
             val httpClient = HTTPClient(
                 appConfig,
                 eTagManager,
                 diagnosticsTracker,
                 signingManager,
                 cache,
-                apiSourceProvider,
+                apiSourceFailover,
                 localeProvider = localeProvider,
                 forceServerErrorStrategy = forceServerErrorStrategy,
             )

purchases/src/main/kotlin/com/revenuecat/purchases/common/HTTPClient.kt

@@ -13,6 +13,7 @@ import com.revenuecat.purchases.Store
 import com.revenuecat.purchases.VerificationResult
 import com.revenuecat.purchases.api.BuildConfig
 import com.revenuecat.purchases.common.diagnostics.DiagnosticsTracker
+import com.revenuecat.purchases.common.networking.APISourceFailover
 import com.revenuecat.purchases.common.networking.ConnectionErrorReason
 import com.revenuecat.purchases.common.networking.ETagManager
 import com.revenuecat.purchases.common.networking.Endpoint
@@ -24,7 +25,6 @@ import com.revenuecat.purchases.common.networking.NullPointerReadingErrorStreamE
 import com.revenuecat.purchases.common.networking.RCContainer
 import com.revenuecat.purchases.common.networking.RCContainerFormatException
 import com.revenuecat.purchases.common.networking.RCHTTPStatusCodes
-import com.revenuecat.purchases.common.remoteconfig.RemoteConfigSourceProvider
 import com.revenuecat.purchases.common.verification.SignatureVerificationException
 import com.revenuecat.purchases.common.verification.SignatureVerificationMode
 import com.revenuecat.purchases.common.verification.SigningManager
@@ -71,7 +71,7 @@ internal class HTTPClient(
     private val diagnosticsTrackerIfEnabled: DiagnosticsTracker?,
     val signingManager: SigningManager,
     private val storefrontProvider: StorefrontProvider,
-    private val apiSourceProvider: RemoteConfigSourceProvider?,
+    private val apiSourceFailover: APISourceFailover?,
     private val dateProvider: DateProvider = DefaultDateProvider(),
     private val mapConverter: MapConverter = MapConverter(),
     private val localeProvider: LocaleProvider,
@@ -94,6 +94,10 @@ internal class HTTPClient(
         // Per-element codecs the SDK can decode, advertised so the server never picks one we can't
         // (e.g. brotli/zstd). "gzip" implies "identity" is also acceptable.
         const val RC_FORMAT_ACCEPT_ENCODING = "gzip"
+
+        // Defensive cap on API source attempts within one request, in case the source list is re-armed
+        // (topic rebuild or interval restart) while a request is walking it.
+        const val MAX_API_SOURCE_ATTEMPTS = 5
     }
 
     private val enableExtraRequestLogging = BuildConfig.ENABLE_EXTRA_REQUEST_LOGGING && appConfig.isDebugBuild
@@ -156,7 +160,7 @@ internal class HTTPClient(
      * @throws JSONException Thrown for any JSON errors, not thrown for returned HTTP error codes
      * @throws IOException Thrown for any unexpected errors, not thrown for returned HTTP error codes
      */
-    @Suppress("LongParameterList", "LongMethod", "CyclomaticComplexMethod", "InstanceOfCheckForException")
+    @Suppress("LongParameterList", "LongMethod", "CyclomaticComplexMethod")
     @Throws(JSONException::class, IOException::class)
     fun performRequest(
         baseURL: URL,
@@ -193,8 +197,106 @@ internal class HTTPClient(
 
         val isMainBackend = fallbackURLIndex == 0
 
-        val requestBaseURL = apiSourceURL(endpoint, baseURL, isFallbackAttempt = !isMainBackend) ?: baseURL
+        var source = apiSourceFailover?.currentSource(endpoint, baseURL, isFallbackAttempt = !isMainBackend)
+        var sourceAttempts = 0
+
+        while (true) {
+            sourceAttempts++
+            val outcome = performAttempt(
+                requestBaseURL = source?.url ?: baseURL,
+                isFallbackURL = fallbackURLIndex > 0,
+                isMainBackend = isMainBackend,
+                fallbackAvailable = canUseFallback(),
+                endpoint = endpoint,
+                body = body,
+                postFieldsToSign = postFieldsToSign,
+                requestHeaders = requestHeaders,
+                refreshETag = refreshETag,
+            )
+            if (outcome.canFailOverToNextSource) {
+                val nextSource = sourceToRetryOn(source, sourceAttempts, endpoint)
+                if (nextSource != null) {
+                    source = nextSource
+                    continue
+                }
+            }
+            return when (outcome) {
+                is AttemptOutcome.Failed -> {
+                    if (!canUseFallback()) {
+                        throw outcome.exception
+                    }
+                    // Unlike iOS, we keep failing over on every connection-level IOException here, including
+                    // ones that may be caused by the device being offline. iOS suppresses the host switch on
+                    // device connectivity errors, but Android has no equivalent signal at this layer: a device
+                    // with no connectivity and a host whose DNS fails both surface as UnknownHostException, and
+                    // telling them apart requires a ConnectivityManager check (ACCESS_NETWORK_STATE), which the
+                    // SDK does not currently have.
+                    var fallbackResult = performRequestToFallbackURL()
+                    if (RCHTTPStatusCodes.isServerError(fallbackResult.responseCode) && canUseFallback()) {
+                        fallbackResult = performRequestToFallbackURL()
+                    }
+                    fallbackResult
+                }
+
+                is AttemptOutcome.Completed -> {
+                    val result = outcome.result
+                    when {
+                        result == null -> {
+                            log(LogIntent.WARNING) { NetworkStrings.ETAG_RETRYING_CALL }
+                            performRequest(
+                                baseURL,
+                                endpoint,
+                                body,
+                                postFieldsToSign,
+                                requestHeaders,
+                                refreshETag = true,
+                                fallbackBaseURLs,
+                                fallbackURLIndex,
+                            )
+                        }
+
+                        RCHTTPStatusCodes.isServerError(result.responseCode) && canUseFallback() ->
+                            // Handle server errors with fallback URLs
+                            performRequestToFallbackURL()
+
+                        else -> result
+                    }
+                }
+            }
+        }
+    }
+
+    /** The result of one request attempt against a single host. */
+    private sealed interface AttemptOutcome {
+        /** [result] is null when an ETag cache miss requires retrying with a refreshed ETag. */
+        data class Completed(val result: HTTPResult?) : AttemptOutcome
+
+        data class Failed(val exception: IOException) : AttemptOutcome
 
+        /** Whether this outcome is a failure that API sources may recover from by switching hosts. */
+        val canFailOverToNextSource: Boolean
+            get() = when (this) {
+                is Failed -> true
+                is Completed -> result?.let { RCHTTPStatusCodes.isServerError(it.responseCode) } == true
+            }
+    }
+
+    /**
+     * Performs one request attempt against [requestBaseURL], recording its timeout bookkeeping and
+     * diagnostics. Connection-level failures come back as [AttemptOutcome.Failed] instead of throwing.
+     */
+    @Suppress("LongParameterList", "InstanceOfCheckForException")
+    private fun performAttempt(
+        requestBaseURL: URL,
+        isFallbackURL: Boolean,
+        isMainBackend: Boolean,
+        fallbackAvailable: Boolean,
+        endpoint: Endpoint,
+        body: Map<String, Any?>?,
+        postFieldsToSign: List<Pair<String, String>>?,
+        requestHeaders: Map<String, String>,
+        refreshETag: Boolean,
+    ): AttemptOutcome {
         var callSuccessful = false
         val requestStartTime = dateProvider.now
         var callResult: HTTPResult? = null
@@ -204,7 +306,7 @@ internal class HTTPClient(
         try {
             callResult = performCall(
                 requestBaseURL,
-                fallbackURLIndex > 0,
+                isFallbackURL,
                 endpoint,
                 body,
                 postFieldsToSign,
@@ -218,19 +320,9 @@ internal class HTTPClient(
             }
         } catch (e: IOException) {
             exceptionHit = e
-            if (e is SocketTimeoutException && isMainBackend && canUseFallback()) {
-                requestResult = HTTPTimeoutManager.RequestResult.TIMEOUT_ON_MAIN_BACKEND_FOR_FALLBACK_SUPPORTED_ENDPOINT
-                callResult = performRequestToFallbackURL()
-            } else if (canUseFallback()) {
-                // Unlike iOS, we keep failing over on every connection-level IOException here, including ones
-                // that may be caused by the device being offline. iOS suppresses the host switch on device
-                // connectivity errors, but Android has no equivalent signal at this layer: a device with no
-                // connectivity and a host whose DNS fails both surface as UnknownHostException, and telling
-                // them apart requires a ConnectivityManager check (ACCESS_NETWORK_STATE), which the SDK does
-                // not currently have.
-                callResult = performRequestToFallbackURL()
-            } else {
-                throw e
+            if (e is SocketTimeoutException && isMainBackend && fallbackAvailable) {
+                requestResult =
+                    HTTPTimeoutManager.RequestResult.TIMEOUT_ON_MAIN_BACKEND_FOR_FALLBACK_SUPPORTED_ENDPOINT
             }
         } finally {
             timeoutManager.recordRequestResult(requestResult)
@@ -245,42 +337,26 @@ internal class HTTPClient(
                 connectionException = exceptionHit,
             )
         }
-        if (callResult == null) {
-            log(LogIntent.WARNING) { NetworkStrings.ETAG_RETRYING_CALL }
-            callResult = performRequest(
-                baseURL,
-                endpoint,
-                body,
-                postFieldsToSign,
-                requestHeaders,
-                refreshETag = true,
-                fallbackBaseURLs,
-                fallbackURLIndex,
-            )
-        } else if (RCHTTPStatusCodes.isServerError(callResult.responseCode) && canUseFallback()) {
-            // Handle server errors with fallback URLs
-            callResult = performRequestToFallbackURL()
-        }
-        return callResult
+        return exceptionHit?.let { AttemptOutcome.Failed(it) } ?: AttemptOutcome.Completed(callResult)
     }
 
     /**
-     * The API base URL to use for [endpoint], or null to keep using [baseURL].
-     *
-     * API sources apply only when the `usesRemoteConfigAPISources` dangerous setting is enabled, this is not
-     * an endpoint fallback-host attempt, the endpoint opts in via [Endpoint.usesAPISources], and [baseURL] is
-     * still the default host (a proxy or an overridden base URL pins the host and bypasses API sources).
+     * The next source to retry [endpoint]'s failed request on, or null when the request didn't target an
+     * API [source], the attempt cap is reached, or [APISourceFailover] decides against failing over (the
+     * source's health check passed, or every source is already unhealthy).
      */
-    private fun apiSourceURL(endpoint: Endpoint, baseURL: URL, isFallbackAttempt: Boolean): URL? {
-        val provider = apiSourceProvider ?: return null
-        val eligible = appConfig.usesRemoteConfigAPISources &&
-            !isFallbackAttempt &&
-            endpoint.usesAPISources &&
-            baseURL.toString() == AppConfig.baseUrlString
-        return if (eligible) {
-            provider.currentAPISource()?.url?.let { runCatching { URL(it) }.getOrNull() }
-        } else {
-            null
+    private fun sourceToRetryOn(
+        source: APISourceFailover.ResolvedSource?,
+        sourceAttempts: Int,
+        endpoint: Endpoint,
+    ): APISourceFailover.ResolvedSource? {
+        val decision = source
+            ?.takeIf { sourceAttempts < MAX_API_SOURCE_ATTEMPTS }
+            ?.let { apiSourceFailover?.onRequestFailure(it) }
+        return (decision as? APISourceFailover.FailureDecision.RetryNextSource)?.next?.also {
+            log(LogIntent.DEBUG) {
+                NetworkStrings.RETRYING_CALL_WITH_NEXT_API_SOURCE.format(endpoint.name, it.url)
+            }
         }
     }
 

purchases/src/main/kotlin/com/revenuecat/purchases/common/networking/APISourceFailover.kt

@@ -0,0 +1,89 @@
+package com.revenuecat.purchases.common.networking
+
+import com.revenuecat.purchases.common.AppConfig
+import com.revenuecat.purchases.common.LogIntent
+import com.revenuecat.purchases.common.log
+import com.revenuecat.purchases.common.remoteconfig.RemoteConfigSourceHandle
+import com.revenuecat.purchases.common.remoteconfig.RemoteConfigSourceProvider
+import com.revenuecat.purchases.common.warnLog
+import java.net.MalformedURLException
+import java.net.URL
+
+/**
+ * Decides which API source a request should target and whether a failed request should fail over to
+ * the next source. Only connection-level failures and 5xx responses can trigger a failover, and only
+ * after a health check against the current source fails: a 2xx health response means the source is
+ * healthy (the request failure was something else, e.g. endpoint-specific), so the original error
+ * surfaces without switching hosts. A source whose health check returns non-2xx or cannot complete
+ * is reported unhealthy and the next one takes over. 4xx responses never fail over: they are request
+ * errors, not source problems.
+ */
+internal class APISourceFailover(
+    private val appConfig: AppConfig,
+    private val sourceProvider: RemoteConfigSourceProvider,
+    private val healthChecker: SourceHealthChecker,
+) {
+
+    /** A source handle plus its parsed base URL; only parseable sources are ever handed out. */
+    data class ResolvedSource(val handle: RemoteConfigSourceHandle, val url: URL)
+
+    sealed interface FailureDecision {
+        /** The current source failed its health check; retry the request on [next]. */
+        data class RetryNextSource(val next: ResolvedSource) : FailureDecision
+
+        /** The current source is healthy, so failing over is pointless: surface the original error. */
+        object SourceHealthy : FailureDecision
+
+        /** Every source has been reported unhealthy; surface the original error. */
+        object SourcesExhausted : FailureDecision
+    }
+
+    /**
+     * The source a request to [endpoint] should target, or null to keep using [baseURL]. Sources apply
+     * only when the `usesRemoteConfigAPISources` dangerous setting is enabled, this is not an endpoint
+     * fallback-host attempt, the endpoint opts in via [Endpoint.usesAPISources], and [baseURL] is still
+     * the default host (a proxy or an overridden base URL pins the host and bypasses API sources).
+     */
+    fun currentSource(endpoint: Endpoint, baseURL: URL, isFallbackAttempt: Boolean): ResolvedSource? {
+        val eligible = appConfig.usesRemoteConfigAPISources &&
+            !isFallbackAttempt &&
+            endpoint.usesAPISources &&
+            baseURL.toString() == AppConfig.baseUrlString
+        return if (eligible) currentResolvedSource() else null
+    }
+
+    /**
+     * Decides what to do after a request to [source] failed at the connection level or returned a
+     * 5xx, health-checking the source to tell a source outage apart from anything else. The health
+     * check performs blocking network I/O on the calling thread, so this must always be called from
+     * a background thread (as [com.revenuecat.purchases.common.HTTPClient] requests already are).
+     */
+    fun onRequestFailure(source: ResolvedSource): FailureDecision {
+        if (healthChecker.isHealthy(source.handle.url)) {
+            log(LogIntent.DEBUG) {
+                "API source ${source.handle.url} is healthy despite the request failing; not failing over."
+            }
+            return FailureDecision.SourceHealthy
+        }
+        sourceProvider.reportUnhealthy(source.handle)
+        return currentResolvedSource()?.let { FailureDecision.RetryNextSource(it) }
+            ?: FailureDecision.SourcesExhausted
+    }
+
+    /**
+     * The provider's current source with its URL parsed. Sources with malformed URLs are reported
+     * unhealthy and skipped, so they participate in failover instead of silently pinning requests to
+     * the default host. The walk ends when the provider runs out of sources.
+     */
+    private fun currentResolvedSource(): ResolvedSource? {
+        while (true) {
+            val handle = sourceProvider.currentAPISource() ?: return null
+            try {
+                return ResolvedSource(handle, URL(handle.url))
+            } catch (e: MalformedURLException) {
+                warnLog { "Skipping API source with malformed url ${handle.url}: $e" }
+                sourceProvider.reportUnhealthy(handle)
+            }
+        }
+    }
+}