fix(api): auto-recover Axelar stuck-confirm and stop leaked squidRouterPay polling loops

apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts

@@ -0,0 +1,236 @@
+// eslint-disable-next-line import/no-unresolved
+import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test";
+// Captured before mock.module so afterAll can restore the real package —
+// bun module mocks are process-wide and would poison later test files.
+import * as sharedNamespace from "@vortexfi/shared";
+import * as rampServiceNamespace from "../../ramp/ramp.service";
+import * as evmFundingNamespace from "../evm-funding";
+
+// Value copies taken before mock.module runs — the namespaces themselves are
+// live bindings that would reflect the mocks once installed.
+const sharedReal = { ...sharedNamespace };
+const rampServiceReal = { ...rampServiceNamespace };
+const evmFundingReal = { ...evmFundingNamespace };
+
+const Networks = {
+  AssetHub: "assethub",
+  Base: "base",
+  Moonbeam: "moonbeam",
+  Polygon: "polygon"
+} as const;
+
+const FiatToken = {
+  BRL: "BRL",
+  EURC: "EUR"
+} as const;
+
+const RampDirection = {
+  BUY: "BUY",
+  SELL: "SELL"
+} as const;
+
+const SWAP_HASH = "0x31365ff4337000801303097a0494fd97ecc1661ea84fedee801f01825b236f49";
+const EVM_EPHEMERAL_ADDRESS = "0x1111111111111111111111111111111111111111";
+const FUNDER_ADDRESS = "0x2222222222222222222222222222222222222222";
+
+// Queue of axelarscan statuses returned per polling iteration; refilled per test.
+let axelarStatusQueue: unknown[] = [];
+const getStatusAxelarScan = mock(async () => {
+  if (axelarStatusQueue.length > 1) {
+    return axelarStatusQueue.shift();
+  }
+  return axelarStatusQueue[0];
+});
+const getStatus = mock(async () => ({
+  id: "",
+  isGMPTransaction: true,
+  routeStatus: [],
+  squidTransactionStatus: "",
+  status: "ongoing"
+}));
+const recoverAxelarStuckConfirm = mock(async () => "AXELAR_TX_HASH");
+// Never settles on its own; the real implementation rejects on abort, but these
+// tests always resolve via the bridge path of Promise.any.
+const checkEvmBalanceForToken = mock(() => new Promise(() => undefined));
+
+mock.module("@vortexfi/shared", () => ({
+  ...sharedReal,
+  checkEvmBalanceForToken,
+  EvmClientManager: {
+    getInstance: () => ({
+      getClient: () => ({}),
+      getWalletClient: () => ({ account: { address: FUNDER_ADDRESS } })
+    })
+  },
+  FiatToken,
+  getNetworkId: (network: string) => {
+    if (network === Networks.Base) return 8453;
+    if (network === Networks.Polygon) return 137;
+    if (network === Networks.Moonbeam) return 1284;
+    return undefined;
+  },
+  getOnChainTokenDetails: () => ({
+    decimals: 6,
+    erc20AddressSourceChain: "0x3333333333333333333333333333333333333333",
+    isNative: false
+  }),
+  getStatus,
+  getStatusAxelarScan,
+  isAlfredpayToken: () => false,
+  Networks,
+  RampDirection,
+  recoverAxelarStuckConfirm
+}));
+
+mock.module("../evm-funding", () => ({
+  getEvmFundingAccount: () => ({ address: FUNDER_ADDRESS })
+}));
+
+mock.module("../../ramp/ramp.service", () => ({
+  default: {
+    appendErrorLog: mock(async () => undefined)
+  }
+}));
+
+const { default: QuoteTicket } = await import("../../../../models/quoteTicket.model");
+const { SquidRouterPayPhaseHandler } = await import("./squid-router-pay-phase-handler");
+
+const realQuoteTicketFindByPk = QuoteTicket.findByPk;
+
+afterAll(() => {
+  mock.module("@vortexfi/shared", () => ({ ...sharedReal }));
+  mock.module("../evm-funding", () => ({ ...evmFundingReal }));
+  mock.module("../../ramp/ramp.service", () => ({ ...rampServiceReal }));
+  QuoteTicket.findByPk = realQuoteTicketFindByPk;
+});
+
+let quote: {
+  inputCurrency: string;
+  outputCurrency: string;
+  to: string;
+};
+
+QuoteTicket.findByPk = mock(async () => quote as any) as typeof QuoteTicket.findByPk;
+
+function makeState(stateOverrides: Record<string, any> = {}) {
+  const state = {
+    currentPhase: "squidRouterPay",
+    errorLogs: [],
+    get() {
+      const { get: _get, update: _update, ...data } = this;
+      return data;
+    },
+    id: "ramp-1",
+    phaseHistory: [],
+    quoteId: "quote-1",
+    state: {
+      evmEphemeralAddress: EVM_EPHEMERAL_ADDRESS,
+      squidRouterPayTxHash: "0xpay",
+      squidRouterSwapHash: SWAP_HASH,
+      ...stateOverrides
+    },
+    to: Networks.Base,
+    type: RampDirection.BUY,
+    async update(updateData: Record<string, any>) {
+      Object.assign(this, updateData);
+      return this;
+    }
+  };
+  return state as any;
+}
+
+function makeHandler() {
+  const handler = new SquidRouterPayPhaseHandler();
+  // Shrink the real 60s/10s waits so the polling loop runs in test time.
+  (handler as any).initialDelayMs = 10;
+  (handler as any).pollIntervalMs = 10;
+  return handler;
+}
+
+const STUCK_CONFIRM_STATUS = {
+  call: { chain: "base" },
+  confirm_failed: true,
+  id: `${SWAP_HASH}_55_172`,
+  is_insufficient_fee: false,
+  status: "called"
+};
+
+const EXECUTED_STATUS = {
+  id: `${SWAP_HASH}_55_172`,
+  is_insufficient_fee: false,
+  status: "executed"
+};
+
+describe("SquidRouterPayPhaseHandler", () => {
+  beforeEach(() => {
+    axelarStatusQueue = [];
+    getStatus.mockClear();
+    getStatusAxelarScan.mockClear();
+    recoverAxelarStuckConfirm.mockClear();
+    checkEvmBalanceForToken.mockClear();
+    quote = {
+      inputCurrency: FiatToken.BRL,
+      outputCurrency: "USDC",
+      to: Networks.Base
+    };
+  });
+
+  it("recovers a stuck confirm and records the attempt timestamp", async () => {
+    axelarStatusQueue = [STUCK_CONFIRM_STATUS, EXECUTED_STATUS];
+
+    const state = makeState();
+    const updatedState = await makeHandler().execute(state);
+
+    expect(recoverAxelarStuckConfirm).toHaveBeenCalledTimes(1);
+    expect(recoverAxelarStuckConfirm).toHaveBeenCalledWith(SWAP_HASH, "base", undefined);
+    expect(state.state.axelarConfirmRecoveryAt).toBeString();
+    expect(updatedState.currentPhase).toBe("finalSettlementSubsidy");
+  });
+
+  it("respects the cooldown and does not re-broadcast a recent recovery attempt", async () => {
+    axelarStatusQueue = [STUCK_CONFIRM_STATUS, STUCK_CONFIRM_STATUS, EXECUTED_STATUS];
+
+    const state = makeState({ axelarConfirmRecoveryAt: new Date().toISOString() });
+    await makeHandler().execute(state);
+
+    expect(recoverAxelarStuckConfirm).not.toHaveBeenCalled();
+  });
+
+  it("does not attempt recovery while the confirm poll has not failed", async () => {
+    axelarStatusQueue = [
+      { ...STUCK_CONFIRM_STATUS, confirm_failed: false },
+      EXECUTED_STATUS
+    ];
+
+    const state = makeState();
+    await makeHandler().execute(state);
+
+    expect(recoverAxelarStuckConfirm).not.toHaveBeenCalled();
+  });
+
+  it("stops polling when the processor aborts the execution", async () => {
+    // Regression test for the retry storm: abandoned executions must unwind on abort
+    // instead of polling the status APIs forever.
+    axelarStatusQueue = [STUCK_CONFIRM_STATUS];
+    quote = {
+      inputCurrency: FiatToken.EURC,
+      outputCurrency: "USDC",
+      to: Networks.AssetHub
+    };
+
+    const abortController = new AbortController();
+    const state = makeState({ axelarConfirmRecoveryAt: new Date().toISOString() });
+
+    const execution = makeHandler().execute(state, abortController.signal);
+    // Let the loop run a few iterations before aborting.
+    await new Promise(resolve => setTimeout(resolve, 100));
+    abortController.abort(new Error("Phase execution timed out"));
+
+    await expect(execution).rejects.toThrow();
+    expect(getStatus.mock.calls.length).toBeGreaterThan(0);
+
+    const callsAtAbort = getStatus.mock.calls.length;
+    await new Promise(resolve => setTimeout(resolve, 150));
+    expect(getStatus.mock.calls.length).toBe(callsAtAbort);
+  });
+});

apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts

@@ -17,7 +17,9 @@ import {
   OnChainToken,
   RampDirection,
   RampPhase,
-  SquidRouterPayResponse
+  recoverAxelarStuckConfirm,
+  SquidRouterPayResponse,
+  sleep
 } from "@vortexfi/shared";
 import Big from "big.js";
 import { createWalletClient, encodeFunctionData, Hash, PublicClient } from "viem";
@@ -41,6 +43,9 @@ const BALANCE_POLLING_TIME_MS = 10000;
 // of otherwise successful bridge operations.
 const EVM_BALANCE_CHECK_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes
 const DEFAULT_SQUIDROUTER_GAS_ESTIMATE = "1600000"; // Estimate used to calculate part of the gas fee for SquidRouter transactions.
+// Minimum time between Axelar stuck-confirm recovery broadcasts for the same ramp. A new
+// validator poll needs a few minutes to complete, so re-broadcasting sooner is pure noise.
+const AXELAR_CONFIRM_RECOVERY_COOLDOWN_MS = 10 * 60 * 1000;
 /**
  * Handler for the squidRouter pay phase. Checks the status of the Axelar bridge and pays on native GLMR fee.
  */
@@ -51,6 +56,9 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
   private moonbeamWalletClient: ReturnType<typeof createWalletClient>;
   private polygonWalletClient: ReturnType<typeof createWalletClient>;
   private baseWalletClient: ReturnType<typeof createWalletClient>;
+  // Instance fields (not module constants) so tests can shrink the waits.
+  private initialDelayMs = SQUIDROUTER_INITIAL_DELAY_MS;
+  private pollIntervalMs = AXELAR_POLLING_INTERVAL_MS;
 
   constructor() {
     super();
@@ -77,7 +85,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
    * @param state The current ramp state
    * @returns The updated ramp state
    */
-  protected async executePhase(state: RampState): Promise<RampState> {
+  protected async executePhase(state: RampState, signal?: AbortSignal): Promise<RampState> {
     const quote = await QuoteTicket.findByPk(state.quoteId);
     if (!quote) {
       throw new Error("Quote not found for the given state");
@@ -98,7 +106,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
       }
 
       // Enter check status loop
-      await this.checkStatus(state, bridgeCallHash, quote);
+      await this.checkStatus(state, bridgeCallHash, quote, signal);
 
       if (state.to === Networks.AssetHub) {
         return this.transitionToNextPhase(state, "moonbeamToPendulum");
@@ -117,13 +125,13 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
    * If the bridge reports success, we consider it a success.
    * Only if both fail (timeout) we throw.
    */
-  private async checkStatus(state: RampState, swapHash: string, quote: QuoteTicket): Promise<void> {
+  private async checkStatus(state: RampState, swapHash: string, quote: QuoteTicket, signal?: AbortSignal): Promise<void> {
     // If the destination is not an EVM network, skip the EVM balance optimization and rely on bridge status only.
     if (quote.to === Networks.AssetHub) {
       logger.info("SquidRouterPayPhaseHandler: Destination network is non-EVM; skipping EVM balance check optimization.", {
         toNetwork: quote.to
       });
-      await this.checkBridgeStatus(state, swapHash, quote);
+      await this.checkBridgeStatus(state, swapHash, quote, signal);
       return;
     }
 
@@ -141,6 +149,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
           chain: toChain,
           intervalMs: BALANCE_POLLING_TIME_MS,
           ownerAddress: ephemeralAddress,
+          signal,
           timeoutMs: EVM_BALANCE_CHECK_TIMEOUT_MS,
           tokenDetails: outTokenDetails
         });
@@ -156,7 +165,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
     }
 
     // Wrap both promises to prevent unhandled rejections after one succeeds
-    const bridgeCheckPromise = this.checkBridgeStatus(state, swapHash, quote).catch(err => {
+    const bridgeCheckPromise = this.checkBridgeStatus(state, swapHash, quote, signal).catch(err => {
       // Re-throw to preserve the error for Promise.any
       throw err;
     });
@@ -199,11 +208,14 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
    * Gets the status of the Axelar bridge
    * @param txHash The swap (bridgeCall) transaction hash
    */
-  private async checkBridgeStatus(state: RampState, swapHash: string, quote: QuoteTicket): Promise<void> {
+  private async checkBridgeStatus(state: RampState, swapHash: string, quote: QuoteTicket, signal?: AbortSignal): Promise<void> {
     let isExecuted = false;
     let payTxHash: string | undefined = state.state.squidRouterPayTxHash;
 
-    await new Promise(resolve => setTimeout(resolve, SQUIDROUTER_INITIAL_DELAY_MS));
+    // The signal-aware sleeps make abandoned executions unwind when the processor
+    // times out this phase; without them every timed-out execution left an immortal
+    // polling loop behind, and they piled up against the SquidRouter rate limit.
+    await sleep(this.initialDelayMs, signal);
 
     while (!isExecuted) {
       try {
@@ -256,6 +268,8 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
             await state.update({
               state: { ...state.state, squidRouterPayTxHash: payTxHash }
             });
+          } else if (axelarScanStatus.status === "called" && axelarScanStatus.confirm_failed) {
+            await this.maybeRecoverStuckConfirm(state, swapHash, axelarScanStatus.call?.chain, signal);
           }
         } else {
           logger.info("SquidRouterPayPhaseHandler: Same-chain transaction detected. Skipping Axelar check.");
@@ -266,7 +280,53 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler {
         );
       }
 
-      await new Promise(resolve => setTimeout(resolve, AXELAR_POLLING_INTERVAL_MS));
+      await sleep(this.pollIntervalMs, signal);
+    }
+  }
+
+  /**
+   * Axelar's relayer does not retry a failed validator confirmation poll, so a transfer
+   * whose poll failed stays in status "called" forever. Ask Axelar's recovery signing
+   * service for a new ConfirmGatewayTx and broadcast it, which restarts the poll.
+   * Attempts are rate-limited via a timestamp persisted in the ramp state, and failures
+   * are swallowed so the status loop keeps polling and retries after the cooldown.
+   */
+  private async maybeRecoverStuckConfirm(
+    state: RampState,
+    swapHash: string,
+    sourceChain: string | undefined,
+    signal?: AbortSignal
+  ): Promise<void> {
+    // An unparseable persisted timestamp yields NaN; treat it as "never attempted" so
+    // the comparison below stays well-defined (NaN comparisons are always false).
+    const parsedLastAttempt = state.state.axelarConfirmRecoveryAt ? new Date(state.state.axelarConfirmRecoveryAt).getTime() : 0;
+    const lastAttempt = Number.isFinite(parsedLastAttempt) ? parsedLastAttempt : 0;
+    if (Date.now() - lastAttempt < AXELAR_CONFIRM_RECOVERY_COOLDOWN_MS) {
+      return;
+    }
+
+    if (!sourceChain) {
+      logger.warn(
+        `SquidRouterPayPhaseHandler: Confirm poll failed for ${swapHash} but Axelar status has no source chain; cannot attempt recovery.`
+      );
+      return;
+    }
+
+    // Persist the attempt timestamp before broadcasting so a failing relayer is not
+    // hammered on every 10s poll iteration.
+    await state.update({
+      state: { ...state.state, axelarConfirmRecoveryAt: new Date().toISOString() }
+    });
+
+    try {
+      const axelarTxHash = await recoverAxelarStuckConfirm(swapHash, sourceChain, signal);
+      logger.info(
+        `SquidRouterPayPhaseHandler: Confirm poll failed for ${swapHash}; broadcast recovery ConfirmGatewayTx ${axelarTxHash} on Axelar.`
+      );
+    } catch (error) {
+      logger.warn(
+        `SquidRouterPayPhaseHandler: Axelar stuck-confirm recovery attempt failed for ${swapHash}: ${error instanceof Error ? error.message : String(error)}`
+      );
     }
   }