Resume in-flight rebalancer runs before sizing a fresh one

apps/rebalancer/README.md

@@ -47,4 +47,17 @@ bun run start
 
 Passing `--legacy` exits with an error; it cannot start the retired Pendulum/Moonbeam flow.
 
+## Resetting a stuck run
+
+Each Base flow persists its phase in Supabase Storage and resumes it on the next cron run before
+anything new is quoted. If a run can no longer complete (for example its funds were moved back
+manually), reconcile the funds first, then suspend the Render cron job and make sure no
+rebalancer process is running: the script and a live run write the same Supabase object
+without locking. Then inspect and reset the USDC → BRLA → USDC state:
+
+```bash
+bun run reset:usdc-base-state            # prints the persisted state, changes nothing
+bun run reset:usdc-base-state --confirm  # resets it to idle, keeping history
+```
+
 This project was created using `bun init` in bun v1.2.6. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

apps/rebalancer/package.json

@@ -33,6 +33,7 @@
     "dev": "bun run src/index.ts",
     "format": "biome check --write --unsafe --no-errors-on-unmatched",
     "prepare": "husky",
+    "reset:usdc-base-state": "bun run src/scripts/resetUsdcBaseState.ts",
     "serve": "bun dist/index.js",
     "start": "bun run build && bun run serve",
     "test": "bun test",

apps/rebalancer/src/index.ts

@@ -13,7 +13,11 @@ import {
   type RebalancingCostPolicyDecision,
   shouldTriggerOpportunisticUsdcToBrla
 } from "./rebalance/usdc-brla-usdc-base/guards.ts";
-import { checkInitialUsdcBalanceOnBase, compareRoutesUpfront } from "./rebalance/usdc-brla-usdc-base/steps.ts";
+import {
+  checkInitialUsdcBalanceOnBase,
+  compareRoutesUpfront,
+  getUsdcBalanceOnBaseRaw
+} from "./rebalance/usdc-brla-usdc-base/steps.ts";
 import { getBaseNablaCoverageRatio } from "./services/indexer";
 import {
   BrlaToUsdcBaseRebalancePhase,
@@ -268,6 +272,15 @@ async function selectUsdcToBrlaPolicyAmount(coverageDeviationBps: number): Promi
     return { amountUsdcRaw: standardAmountRaw, policyDecision: standardPolicyDecision };
   }
 
+  const baseUsdcRaw = await getUsdcBalanceOnBaseRaw();
+  if (Big(baseUsdcRaw).lt(profitableAmountRaw)) {
+    console.log(
+      `Base USDC balance ${Big(baseUsdcRaw).div(1e6).toFixed(6)} USDC cannot fund the profitable amount ` +
+        `${config.rebalancingProfitableUsdToBrlAmount} USDC. Using standard amount ${standardAmountSelection.amountUsdc} USDC.`
+    );
+    return { amountUsdcRaw: standardAmountRaw, policyDecision: standardPolicyDecision };
+  }
+
   console.log(
     `Evaluating USDC->BRLA rebalance amounts independently: standard ${standardAmountSelection.amountUsdc} USDC, ` +
       `profitable ${config.rebalancingProfitableUsdToBrlAmount} USDC.`
@@ -343,58 +356,65 @@ async function evaluateBrlaToUsdcPolicy(
 }
 
 async function runUsdcToBrla(coverageDeviationBps: number) {
+  const selectedAmount = await selectUsdcToBrlaPolicyAmount(coverageDeviationBps);
+  const policyDecision = selectedAmount.policyDecision;
+  if (!policyDecision.shouldExecute) return;
+  await executeUsdcToBrlaRebalance(selectedAmount.amountUsdcRaw, coverageDeviationBps, policyDecision);
+}
+
+async function runBrlaToUsdc(coverageDeviationBps: number) {
   const config = getConfig();
-  const amountUsdcRaw = toUsdcRaw(manualAmount || config.rebalancingUsdToBrlAmount);
+  const amountUsdcRaw = toUsdcRaw(manualAmount || config.rebalancingBrlToUsdAmount);
 
-  const stateManager = new UsdcBaseStateManager();
-  const state = await stateManager.getState();
-  const isResuming = !forceRestart && state && state.currentPhase !== UsdcBaseRebalancePhase.Idle;
+  const policyDecision = await evaluateBrlaToUsdcPolicy(amountUsdcRaw, coverageDeviationBps);
+  if (!policyDecision.shouldExecute) return;
 
-  if (!isResuming) {
-    const selectedAmount = await selectUsdcToBrlaPolicyAmount(coverageDeviationBps);
-    const policyDecision = selectedAmount.policyDecision;
-    if (!policyDecision.shouldExecute) return;
-    await executeUsdcToBrlaRebalance(selectedAmount.amountUsdcRaw, coverageDeviationBps, policyDecision);
-    return;
-  }
+  const dailyLimitEvaluation = await evaluateCurrentRunDailyLimit(amountUsdcRaw, policyDecision.profitable);
+  if (dailyLimitEvaluation.decision?.shouldSkip) return;
 
-  await rebalanceUsdcBrlaUsdcBase(amountUsdcRaw, forceRestart, forcedRoute);
+  const rebalancerUsdcBalance = await checkInitialUsdcBalanceOnBase(amountUsdcRaw);
+  if (config.rebalancingBrlToUsdMinBalance && rebalancerUsdcBalance.lt(config.rebalancingBrlToUsdMinBalance)) {
+    throw new Error(
+      `Rebalancer USDC balance ${rebalancerUsdcBalance} is below the minimum required balance of ${config.rebalancingBrlToUsdMinBalance} to perform rebalancing.`
+    );
+  }
+  await rebalanceBrlaToUsdcBase(amountUsdcRaw, forceRestart, {
+    config: config.rebalancingCostPolicy,
+    dailyLimitDecision: dailyLimitEvaluation.decision,
+    dailyVolume: dailyLimitEvaluation.dailyVolume,
+    decision: policyDecision.decision,
+    deviationBps: coverageDeviationBps,
+    fallbackRequiresProfit: policyDecision.profitable
+  });
 }
 
-async function runBrlaToUsdc(coverageDeviationBps: number) {
+// A run that died mid-flow holds funds in transit. Resume it before quoting, sizing, or
+// balance-checking a fresh run: those checks assume the in-flight USDC is still in the wallet.
+async function resumeInFlightRebalance(): Promise<boolean> {
+  if (forceRestart) return false;
   const config = getConfig();
-  const amountUsdc = manualAmount || config.rebalancingBrlToUsdAmount;
-  const amountUsdcRaw = multiplyByPowerOfTen(new Big(amountUsdc), 6).toFixed(0, 0);
 
-  const stateManager = new BrlaToUsdcBaseStateManager();
-  const state = await stateManager.getState();
-  const isResuming = !forceRestart && state && state.currentPhase !== BrlaToUsdcBaseRebalancePhase.Idle;
-
-  if (!isResuming) {
-    const policyDecision = await evaluateBrlaToUsdcPolicy(amountUsdcRaw, coverageDeviationBps);
-    if (!policyDecision.shouldExecute) return;
-
-    const dailyLimitEvaluation = await evaluateCurrentRunDailyLimit(amountUsdcRaw, policyDecision.profitable);
-    if (dailyLimitEvaluation.decision?.shouldSkip) return;
+  const usdcBaseState = await new UsdcBaseStateManager().getState();
+  if (usdcBaseState && usdcBaseState.currentPhase !== UsdcBaseRebalancePhase.Idle) {
+    if (config.rebalancingCostPolicy.mode === "dry-run") {
+      console.log(`Dry-run mode: not resuming USDC->BRLA->USDC run at phase ${usdcBaseState.currentPhase}.`);
+      return false;
+    }
+    await rebalanceUsdcBrlaUsdcBase(toUsdcRaw(manualAmount || config.rebalancingUsdToBrlAmount), false, forcedRoute);
+    return true;
+  }
 
-    const rebalancerUsdcBalance = await checkInitialUsdcBalanceOnBase(amountUsdcRaw);
-    if (config.rebalancingBrlToUsdMinBalance && rebalancerUsdcBalance.lt(config.rebalancingBrlToUsdMinBalance)) {
-      throw new Error(
-        `Rebalancer USDC balance ${rebalancerUsdcBalance} is below the minimum required balance of ${config.rebalancingBrlToUsdMinBalance} to perform rebalancing.`
-      );
+  const brlaToUsdcState = await new BrlaToUsdcBaseStateManager().getState();
+  if (brlaToUsdcState && brlaToUsdcState.currentPhase !== BrlaToUsdcBaseRebalancePhase.Idle) {
+    if (config.rebalancingCostPolicy.mode === "dry-run") {
+      console.log(`Dry-run mode: not resuming BRLA->USDC run at phase ${brlaToUsdcState.currentPhase}.`);
+      return false;
     }
-    await rebalanceBrlaToUsdcBase(amountUsdcRaw, forceRestart, {
-      config: config.rebalancingCostPolicy,
-      dailyLimitDecision: dailyLimitEvaluation.decision,
-      dailyVolume: dailyLimitEvaluation.dailyVolume,
-      decision: policyDecision.decision,
-      deviationBps: coverageDeviationBps,
-      fallbackRequiresProfit: policyDecision.profitable
-    });
-    return;
+    await rebalanceBrlaToUsdcBase(toUsdcRaw(manualAmount || config.rebalancingBrlToUsdAmount), false);
+    return true;
   }
 
-  await rebalanceBrlaToUsdcBase(amountUsdcRaw, forceRestart);
+  return false;
 }
 
 async function checkForRebalancing() {
@@ -403,6 +423,8 @@ async function checkForRebalancing() {
 
   if (!coverage) throw new Error("Failed to fetch Base Nabla coverage ratio.");
 
+  if (await resumeInFlightRebalance()) return;
+
   const lowerBound = 1 - config.rebalancingThresholdBrlaToUsdc;
   const upperBound = 1 + config.rebalancingThresholdUsdcToBrla;
 

apps/rebalancer/src/scripts/resetUsdcBaseState.ts

@@ -0,0 +1,25 @@
+// Resets a stuck USDC->BRLA->USDC (Base) rebalance state back to Idle, keeping history.
+// Dry-run by default: prints the persisted state. Pass --confirm to overwrite it.
+// Usage: bun run reset:usdc-base-state [--confirm]
+import { createUsdcBaseRebalanceState, UsdcBaseRebalancePhase, UsdcBaseStateManager } from "../services/stateManager.ts";
+
+const confirm = process.argv.includes("--confirm");
+const stateManager = new UsdcBaseStateManager();
+const state = await stateManager.getState();
+
+console.log("Current USDC->BRLA->USDC (Base) state:", JSON.stringify(state, null, 2));
+
+if (!state || state.currentPhase === UsdcBaseRebalancePhase.Idle) {
+  console.log("State is already idle. Nothing to reset.");
+  process.exit(0);
+}
+
+if (!confirm) {
+  console.log(`State is stuck at phase "${state.currentPhase}". Re-run with --confirm to reset it to idle.`);
+  process.exit(0);
+}
+
+await stateManager.saveState(createUsdcBaseRebalanceState(null, UsdcBaseRebalancePhase.Idle));
+console.log(
+  "State reset to idle. The state printed above is the last record of the abandoned run; keep it if the funds are not fully reconciled yet."
+);

docs/security-spec/07-operations/rebalancer.md

@@ -4,7 +4,7 @@
 
 The rebalancer is a standalone service (`apps/rebalancer/`) that monitors token coverage ratios and automatically moves liquidity across chains when ratios indicate a pool imbalance. Its primary function is ensuring the platform has sufficient tokens to service ramp operations without manual intervention.
 
-The default Base rebalancer is cost-aware. A coverage-ratio breach makes a fresh cron run eligible for evaluation, but execution still depends on the configured urgency band and projected round-trip cost. Mild and moderate imbalances can be skipped when route quotes are unfavorable; severe imbalances tolerate higher configured cost. When coverage is already inside the configured bounds, the USDC → BRLA → USDC flow may still run opportunistically, but only if its projected route cost is below `REBALANCING_OPPORTUNISTIC_USDC_TO_BRLA_MAX_COST_BPS` (default 10 bps). `REBALANCING_HARD_MAX_COST_BPS` remains a hard projected-cost cap in every mode. `REBALANCING_DAILY_BRIDGE_LIMIT_USD` caps non-profitable fresh Base runs, but a quote that projects profit may bypass the daily cap while still being recorded in history after completion. When a separate profitable USDC → BRLA → USDC amount is configured, the flow evaluates that larger amount with its own fresh quotes and executes it only when that larger quote projects profit.
+The default Base rebalancer is cost-aware. A coverage-ratio breach makes a fresh cron run eligible for evaluation, but execution still depends on the configured urgency band and projected round-trip cost. Mild and moderate imbalances can be skipped when route quotes are unfavorable; severe imbalances tolerate higher configured cost. When coverage is already inside the configured bounds, the USDC → BRLA → USDC flow may still run opportunistically, but only if its projected route cost is below `REBALANCING_OPPORTUNISTIC_USDC_TO_BRLA_MAX_COST_BPS` (default 10 bps). `REBALANCING_HARD_MAX_COST_BPS` remains a hard projected-cost cap in every mode. `REBALANCING_DAILY_BRIDGE_LIMIT_USD` caps non-profitable fresh Base runs, but a quote that projects profit may bypass the daily cap while still being recorded in history after completion. When a separate profitable USDC → BRLA → USDC amount is configured, the flow evaluates that larger amount with its own fresh quotes and executes it only when that larger quote projects profit and the rebalancer's Base USDC balance covers it.
 
 **Current implementation:** Two active Base paths plus one dormant compatibility implementation:
 
@@ -41,7 +41,7 @@ bun run start [amount] [--restart] [--route=squidrouter|avenia|nabla-main]
 - `REBALANCING_MAX_COST_BPS_MILD` / `REBALANCING_MAX_COST_BPS_MODERATE` / `REBALANCING_MAX_COST_BPS_SEVERE` — maximum projected round-trip cost per urgency band.
 - `REBALANCING_HARD_MAX_COST_BPS` — final projected-cost ceiling enforced even in `always` mode.
 - `REBALANCING_OPPORTUNISTIC_USDC_TO_BRLA_MAX_COST_BPS` — maximum projected route cost for in-range opportunistic USDC → BRLA → USDC execution (default 10 bps).
-- `REBALANCING_USD_TO_BRL_AMOUNT` / `REBALANCING_PROFITABLE_USD_TO_BRL_AMOUNT` — standard and projected-profitable USDC → BRLA → USDC sizing. The profitable amount defaults to the standard amount when unset and is used only when a fresh quote for that larger size projects profit.
+- `REBALANCING_USD_TO_BRL_AMOUNT` / `REBALANCING_PROFITABLE_USD_TO_BRL_AMOUNT` — standard and projected-profitable USDC → BRLA → USDC sizing. The profitable amount defaults to the standard amount when unset and is used only when the Base USDC balance covers it and a fresh quote for that larger size projects profit.
 
 ---
 
@@ -154,7 +154,7 @@ security review.
 ### Base flow invariants
 
 11. **Daily bridge limit MUST be enforced for paid current runs** — Total requested USDC amount recorded by Base-flow histories per calendar day (UTC), including the amount about to be rebalanced, must not exceed `REBALANCING_DAILY_BRIDGE_LIMIT_USD` for non-profitable fresh Base runs. The limit decision must run after quote/cost-policy evaluation and before fresh state writes or transactions for paid runs. Projected-profitable current runs bypass the cap entirely, but completed profitable runs must still be recorded in history so they count toward later paid-run checks.
-12. **Cost policy MUST run before fresh-run side effects** — For Base flows, route/two-leg quotes and the cost-policy decision must happen before `startNewRebalance`, approvals, swaps, transfers, ticket creation, or history writes. Resumed runs continue the already-started state and do not recompute a fresh skip decision.
+12. **Cost policy MUST run before fresh-run side effects** — For Base flows, route/two-leg quotes and the cost-policy decision must happen before `startNewRebalance`, approvals, swaps, transfers, ticket creation, or history writes. Resumed runs continue the already-started state and do not recompute a fresh skip decision. A non-idle state file is resumed after the coverage read but before any fresh quote, sizing, daily-limit, or Base USDC balance check, including on the opportunistic in-range path, because the in-flight USDC is no longer in the wallet. `dry-run` mode does not resume it either; the run stays paused until an executing mode is restored.
 13. **Severity bands MUST be monotonic** — Moderate deviation must be less than or equal to severe deviation. Mild cost tolerance must be less than or equal to moderate, moderate less than or equal to severe, and severe less than or equal to `REBALANCING_HARD_MAX_COST_BPS`.
 14. **Mild/moderate imbalances MUST be skippable when cost exceeds tolerance** — In `auto` mode, fresh Base rebalances must skip when projected round-trip cost exceeds the configured limit for the current band.
 15. **Opportunistic in-range rebalances MUST stay below the configured projected-cost cap** — When coverage is already inside `[lowerBound, upperBound]`, only USDC → BRLA → USDC may run opportunistically. It must use the normal cost-policy quote, daily-limit/profit decision, Base USDC balance check, route selection, hard max-cost cap, and state machine; it must skip when projected route cost is greater than or equal to `REBALANCING_OPPORTUNISTIC_USDC_TO_BRLA_MAX_COST_BPS` (default 10 bps). If an opportunistic Avenia route later falls back to SquidRouter, the preflight SquidRouter quote must independently satisfy the normal cost policy, the configured opportunistic cap, and the profitable-quote requirement when the original current quote skipped the daily bridge limit because it was projected profitable.