fix(codex): rotate pool accounts on model capacity

docs-site/src/content/docs/reference/cli/providers-accounts.md

@@ -164,6 +164,10 @@ Direct mode keeps using the caller-owned/native main credential. Usage-based pro
 recovery may later select another eligible Pool account. Those recovery paths remain active when
 usage-based switching is off. OpenCodex replays the conversation after an account change, but the
 provider-side prompt cache may be cold. Unknown providers or ids exit 1.
+Before substantive output, an upstream model-capacity rejection rotates through each eligible Pool
+account once without persisting a cooldown or changing the next request's eligible set. Exhaustion
+returns the first capacity error. Direct mode, exact account selectors, and failures after output do
+not rotate.
 On a **401/403**, App login clears that account's process-local affinity and requires reauthentication.
 On a **429**, opencodex honors `Retry-After`, starts the account cooldown, clears affinity, and may
 rotate the request to another eligible Pool account. These failure transitions remain active with

docs-site/src/content/docs/reference/configuration/providers.md

@@ -150,6 +150,14 @@ alternate account in the same request, even when usage-based proactive switching
 changes preserve and replay the conversation context, but provider-side prompt-cache reuse across
 accounts is not guaranteed and the cache may need to warm again.
 
+Before any text, reasoning, tool call, or other model output reaches the client, a structured
+`server_is_overloaded` / `slow_down` model-capacity rejection (or the standard
+`Selected model is at capacity. Please try a different model.` response) tries each remaining
+eligible Pool account once. This exclusion is request-local: rejected capacity attempts do not write
+account cooldown, health, affinity, or active-selection state. If every account returns capacity, the first response
+is returned; the next request starts with a fresh eligible set. Direct mode and exact account
+selectors never use this rotation, and capacity after substantive output is never replayed.
+
 On a **401/403**, App login clears that account's process-local affinity and requires reauthentication.
 On a **429**, opencodex honors `Retry-After`, starts the account cooldown, clears affinity, and may
 rotate the request to another eligible Pool account. These failure transitions remain active with

src/codex/auth-context.ts

@@ -21,6 +21,7 @@ import {
   tryAcquireCodexQuotaProbeLease,
   tryAcquireCodexQuotaScopeProbeLease,
   pickAlternateCodexAccount,
+  pickAlternateCodexAccountExcluding,
   resolveCodexAccountForThreadDetailed,
 } from "./routing";
 import type { CodexCooldownSource, CodexQuotaScope } from "./routing";
@@ -227,6 +228,8 @@ export function shouldMarkAccountNeedsReauthForCodexAuthFailure(cause: unknown):
 
 export interface ResolveCodexAuthContextOptions {
   excludeAccountId?: string;
+  /** Request-local exclusion set for bounded multi-account recovery. */
+  excludeAccountIds?: ReadonlySet<string>;
   /** Resolve exactly this account without consulting or mutating Pool selection. */
   accountId?: string;
   /** Final native model selected for this request, used to select its quota group. */
@@ -253,7 +256,9 @@ export async function resolveCodexAuthContext(
 ): Promise<CodexAuthContext> {
   const writerGeneration = captureConfigGeneration();
   const fixedAccountId = options.accountId;
-  if (fixedAccountId !== undefined && options.excludeAccountId !== undefined) {
+  const hasExclusions = options.excludeAccountId !== undefined
+    || (options.excludeAccountIds?.size ?? 0) > 0;
+  if (fixedAccountId !== undefined && hasExclusions) {
     throw new Error("Codex auth context cannot select and exclude an account simultaneously");
   }
   // An explicit namespace binding is stronger than the provider's default mode. It must use the
@@ -284,15 +289,25 @@ export async function resolveCodexAuthContext(
     const threadId = headers.get("x-codex-parent-thread-id");
     const resolution = fixedAccountId !== undefined
       ? { status: "selected" as const, accountId: fixedAccountId }
-      : options.excludeAccountId
+      : hasExclusions
       ? (() => {
-          const selected = pickAlternateCodexAccount(
-            config,
-            options.excludeAccountId!,
-            Date.now(),
-            quotaScope,
-            selectionOptions,
-          );
+          const selected = options.excludeAccountIds
+            ? pickAlternateCodexAccountExcluding(
+              config,
+              options.excludeAccountIds,
+              [...options.excludeAccountIds].at(-1)!,
+              Date.now(),
+              quotaScope,
+              selectionOptions,
+              false,
+            )
+            : pickAlternateCodexAccount(
+              config,
+              options.excludeAccountId!,
+              Date.now(),
+              quotaScope,
+              selectionOptions,
+            );
           return selected
             ? { status: "selected" as const, accountId: selected }
             : { status: "none" as const };
@@ -309,7 +324,7 @@ export async function resolveCodexAuthContext(
       // temporary fence rather than misclassifying that credential as invalid.
       // A configured pool retry/exclusion that finds no alternate preserves its
       // ordinary pool-auth failure instead of being mislabeled as a main fence.
-      if (nativeMainTrafficBlocked && !options.excludeAccountId) {
+      if (nativeMainTrafficBlocked && !hasExclusions) {
         throw new CodexMainProfileDrainingError();
       }
       throw new CodexPoolAuthenticationError();

src/codex/routing.ts

@@ -900,16 +900,24 @@ function bindThreadAffinity(
   pruneLruThreadAffinities();
 }
 
+type CodexAccountExclusion = string | ReadonlySet<string> | undefined;
+
+function isExcludedCodexAccount(exclusion: CodexAccountExclusion, accountId: string): boolean {
+  return typeof exclusion === "string"
+    ? exclusion === accountId
+    : exclusion?.has(accountId) === true;
+}
+
 function getEligiblePoolAccounts(
   config: OcxConfig,
-  excludeId?: string,
+  exclusion?: CodexAccountExclusion,
   now = Date.now(),
   quotaScope?: CodexQuotaScope,
   selectionOptions?: CodexAccountUsabilityOptions,
 ): readonly string[] {
   const ids = (config.codexAccounts ?? [])
     .filter(account => isSelectableCodexPoolAccount(account)
-      && account.id !== excludeId
+      && !isExcludedCodexAccount(exclusion, account.id)
       && !isCodexAccountPaused(config, account.id)
       && !isAccountNeedsReauth(account.id))
     .filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
@@ -919,7 +927,7 @@ function getEligiblePoolAccounts(
   // The main Codex account is not stored in config.codexAccounts; include it as a
   // first-class rotation candidate when its read-only token is usable (Option A).
   if (
-    excludeId !== MAIN_CODEX_ACCOUNT_ID
+    !isExcludedCodexAccount(exclusion, MAIN_CODEX_ACCOUNT_ID)
     && !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)
     && !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)
     && getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
@@ -1151,21 +1159,48 @@ export function pickAlternateCodexAccount(
   now = Date.now(),
   quotaScope?: CodexQuotaScope,
   selectionOptions?: CodexAccountUsabilityOptions,
+): string | null {
+  return pickAlternateCodexAccountExcluding(
+    config,
+    new Set([excludeId]),
+    excludeId,
+    now,
+    quotaScope,
+    selectionOptions,
+  );
+}
+
+/** Strategy-aware alternate that never revisits an account already tried by this request. */
+export function pickAlternateCodexAccountExcluding(
+  config: OcxConfig,
+  excludedIds: ReadonlySet<string>,
+  afterId: string,
+  now = Date.now(),
+  quotaScope?: CodexQuotaScope,
+  selectionOptions?: CodexAccountUsabilityOptions,
+  commitRoundRobin = true,
 ): string | null {
   const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
   // The exclusion is passed into eligibility rather than post-filtered off its
   // result: when the excluded account is the only healthy member of the top
   // tier, the tier walk must be free to descend instead of selecting that tier
   // and then handing back an empty list.
   if (strategy === "round-robin") {
-    const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
-    return pickRoundRobinAccount(codexPoolKeyForScope(quotaScope), eligible, stickyLimitForConfig(config));
+    const eligible = getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions);
+    const poolKey = codexPoolKeyForScope(quotaScope);
+    const stickyLimit = stickyLimitForConfig(config);
+    return commitRoundRobin
+      ? pickRoundRobinAccount(poolKey, eligible, stickyLimit)
+      : peekRoundRobinAccount(poolKey, eligible, stickyLimit);
   }
   if (strategy === "fill-first") {
-    const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
-    return pickNextFillFirstCodexAccount(config, excludeId, eligible, now, selectionOptions);
+    const eligible = getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions);
+    return pickNextFillFirstCodexAccount(config, afterId, eligible, now, selectionOptions);
   }
-  return pickLowestUsageCodexAccount(config, excludeId, now, quotaScope, selectionOptions);
+  return pickLowestUsageAmong(
+    config,
+    getEligiblePoolAccounts(config, excludedIds, now, quotaScope, selectionOptions),
+  );
 }
 
 /** Effective active: automatic runtime cursor, else operator/persisted selection. */

src/lib/upstream-retry.ts

@@ -244,6 +244,10 @@ export interface ResetRetryOptions {
 export interface TransientRetryOptions extends ResetRetryOptions {
   /** Test seam: per-attempt slow budget override (defaults to TRANSIENT_RETRY_SLOW_ATTEMPT_MS). */
   slowAttemptMs?: number;
+  /** Optional semantic gate run before retrying a transient HTTP response. */
+  prepareTransientRetry?: (
+    response: Response,
+  ) => Promise<{ response: Response; retry: boolean }> | { response: Response; retry: boolean };
 }
 
 export type UpstreamSendRecovery = "connection-reset" | "transient-5xx";
@@ -367,6 +371,11 @@ export async function fetchWithTransientRetry(
     if (res.ok || !isTransientUpstreamStatus(res.status)) return res;
     if (opts.abortSignal?.aborted) return res;
     if (Date.now() - attemptStart > slowAttemptMs) return res;
+    if (opts.prepareTransientRetry) {
+      const prepared = await opts.prepareTransientRetry(res);
+      res = prepared.response;
+      if (!prepared.retry) return res;
+    }
     console.warn(
       `[upstream-retry] transient ${res.status}${opts.label ? ` (${opts.label})` : ""} — retrying (${attempt + 2}/${attempts})`,
     );